From 47801b58de20d4fe66be7e6b8586a4be8b89a55b Mon Sep 17 00:00:00 2001 From: SandroMaglione Date: Tue, 17 Nov 2026 18:35:11 +1300 Subject: [PATCH] Keyboard walk: presses on several pages, a ready signal, waitFor steps, and a guard for a page that stops answering - focus-dropped presses on up to four pages at once; a press that did not keep focus is run again alone, and that result is the one reported. - A loaded screen is walked once it stops changing (no request in flight, same menu, focus, and controls for 400 ms), 1.5 s at the latest. - A route step waitFor waits up to 15 s for a named control to show. - A page that stops answering for 20 s is closed and reported as a page-error, so the walk fails instead of hanging. Co-Authored-By: Claude Fable 5.2 --- .changeset/devtools-keyboard-walk.md | 2 +- packages/devtools/README.md | 19 +- packages/devtools/src/cli.ts | 5 +- .../devtools/src/internal/keyboardPage.ts | 17 +- .../devtools/src/internal/keyboardWalk.ts | 206 ++++++++++++++---- packages/devtools/test/KeyboardWalk.test.ts | 43 +++- .../test/fixtures/keyboard/late-control.html | 12 + .../test/fixtures/keyboard/stuck.html | 9 + 8 files changed, 260 insertions(+), 53 deletions(-) create mode 100644 packages/devtools/test/fixtures/keyboard/late-control.html create mode 100644 packages/devtools/test/fixtures/keyboard/stuck.html diff --git a/.changeset/devtools-keyboard-walk.md b/.changeset/devtools-keyboard-walk.md index 4bf8df6a..9888253e 100644 --- a/.changeset/devtools-keyboard-walk.md +++ b/.changeset/devtools-keyboard-walk.md @@ -2,4 +2,4 @@ "@typeonce/ritmo-devtools": minor --- -`ritmo-devtools keyboard [--url ] [--route ] [--json]` plays the game's page by keyboard in Chromium and fails on what breaks keyboard play: a menu open with focus outside it, Tab leaving an open menu, a control with no accessible name, Enter or Space not pressing the focused control, a press that drops focus to ``, and a visible control the keyboard cannot reach. Each failure names the check, the screen, the control (role, name, selector), what was seen, and the fix; a check it cannot decide is printed as not checked, never as passed, and a screen on which nothing could be checked fails. Without `--url` it serves the working directory with the game's own Vite config. A route file lists the screens to visit and how to reach them; without one, the start page is the only screen. It presses controls for real to check focus, so a route marks what must not be pressed. It uses the optional `playwright` and `vite` peers `verify` already uses, at the same versions. +`ritmo-devtools keyboard [--url ] [--route ] [--json]` plays the game's page by keyboard in Chromium and fails on what breaks keyboard play: a menu open with focus outside it, Tab leaving an open menu, a control with no accessible name, Enter or Space not pressing the focused control, a press that drops focus to ``, and a visible control the keyboard cannot reach. Each failure names the check, the screen, the control (role, name, selector), what was seen, and the fix; a check it cannot decide is printed as not checked, never as passed, and a screen on which nothing could be checked fails. Without `--url` it serves the working directory with the game's own Vite config. A route file lists the screens to visit and how to reach them (key presses, clicks, `waitFor` a named control, fixed waits); without one, the start page is the only screen. A loaded screen is walked as soon as it stops changing, and the real presses run on up to four pages at once, with any press that did not keep focus run again alone before it is reported. It presses controls for real to check focus, so a route marks what must not be pressed. It uses the optional `playwright` and `vite` peers `verify` already uses, at the same versions. diff --git a/packages/devtools/README.md b/packages/devtools/README.md index c155029e..6e41db2b 100644 --- a/packages/devtools/README.md +++ b/packages/devtools/README.md @@ -573,7 +573,9 @@ would; a custom control (a `role="switch"`) counts as pressed when its `aria-checked`, `aria-pressed`, `aria-expanded`, or `aria-selected` changes. `focus-dropped` presses each control for real, on a fresh load of its screen, so it does what a player's press does: mark what must not be pressed -in the route. A link to another site is never pressed for real. +in the route. A link to another site is never pressed for real. The presses +run on up to four pages at once; one that did not keep focus is run again +with no other page open, and that second result is the one reported. Without `--route` the start page is one screen. A route is a JSON file: @@ -582,18 +584,25 @@ Without `--route` the start page is one screen. A route is a JSON file: "screens": [ { "name": "title" }, { "name": "play", "path": "?seed=7", "steps": [{ "click": { "role": "button", "name": "Play" } }, { "press": "s" }, { "wait": 800 }] }, - { "name": "end", "path": "?moment=end", "steps": [{ "wait": 5000 }], "noPress": [{ "name": "Start over", "role": "button" }] } + { "name": "end", "path": "?moment=end", "steps": [{ "waitFor": { "role": "button", "name": "Start over" } }], "noPress": [{ "name": "Start over", "role": "button" }] } ] } ``` - `path` is appended to the URL. `steps` run in order after the page settles: `press` takes a key as Playwright names it, `click` a role and - the accessible name the report prints, `wait` milliseconds. + the accessible name the report prints, `waitFor` the same and waits up to + 15 seconds for that control to show, `wait` milliseconds. +- A loaded page has settled when no request is in flight and its menu, + focus, and visible controls have not changed for 400 ms, or after 1.5 + seconds at the latest. A screen that shows its controls later than that, + or after a quiet moment (a splash, a fade), needs a `waitFor` step; + prefer it to `wait`, which is paid again for every control pressed. - `noPress` is `true` (press nothing on that screen) or a list of controls by name and optional role. -- A step that cannot run, or an error the page throws, is a failure, and - that screen's checks are not checked. A screen on which no check could be +- A step that cannot run, an error the page throws, or a page that stops + answering for 20 seconds is a failure, and that screen's checks are not + checked. A screen on which no check could be decided is a failure too (`nothing-checked`): the route did not reach the game's controls, or the page has none the walk can see. diff --git a/packages/devtools/src/cli.ts b/packages/devtools/src/cli.ts index b0cfee78..a06d9252 100755 --- a/packages/devtools/src/cli.ts +++ b/packages/devtools/src/cli.ts @@ -78,6 +78,7 @@ * @docGroup devtools */ import { readFileSync } from "node:fs" +import { availableParallelism } from "node:os" import { resolve } from "node:path" import { pathToFileURL } from "node:url" @@ -335,7 +336,9 @@ const keyboard = async (args: ReadonlyArray): Promise => { return 2 } } - const report = await KeyboardWalk.walk(browser, { url, route }) + // Half the cores, four pages at most: each page runs the whole game. + const pages = Math.max(1, Math.min(4, Math.floor(availableParallelism() / 2))) + const report = await KeyboardWalk.walk(browser, { url, route, pages }) console.log(json ? JSON.stringify(report, null, 2) : KeyboardWalk.format(report)) return report.ok ? 0 : 1 } finally { diff --git a/packages/devtools/src/internal/keyboardPage.ts b/packages/devtools/src/internal/keyboardPage.ts index e2c6f874..86601f2c 100644 --- a/packages/devtools/src/internal/keyboardPage.ts +++ b/packages/devtools/src/internal/keyboardPage.ts @@ -7,6 +7,9 @@ * It installs `__ritmoKeyboardWalk` with: * * - `state()`: the focused element and the open menu, each described; + * - `signature()`: the page's clock and one string naming the open menu, the + * focused element, and every visible control, to tell when a screen has + * stopped changing; * - `controls(scope)`: the visible, enabled controls inside the menu (or the * page), described; * - `groupStops(control, stops)`: the Tab stops in the smallest element that @@ -108,6 +111,8 @@ export const pageScript = String.raw`(() => { ...customRoles.map((role) => '[role="' + role + '"]'), '[role="slider"]', '[role="spinbutton"]', '[role="combobox"]', '[role="textbox"]' ].join(", ") + const visibleControls = (root) => [...root.querySelectorAll(controlSelector)] + .filter((control) => !control.disabled && control.closest("[inert]") === null && visible(control)) // A custom control (a switch, a toggle) may act on keydown with no click: a change of its state attributes is a press too. const stateOf = (element) => element === null ? "" : ["aria-checked", "aria-pressed", "aria-expanded", "aria-selected"].map((name) => element.getAttribute(name) ?? "").join("|") let armed = null @@ -139,9 +144,15 @@ export const pageScript = String.raw`(() => { controls: (scope) => { const root = scope === null ? document.body : element(scope) if (root === null) return [] - return [...root.querySelectorAll(controlSelector)] - .filter((control) => !control.disabled && control.closest("[inert]") === null && visible(control)) - .map(describe) + return visibleControls(root).map(describe) + }, + signature: () => { + const menu = openMenu() + const active = focused() + return { + now: performance.now(), + key: [menu, active, ...(document.body === null ? [] : visibleControls(document.body))].map((element) => element === null ? "-" : idOf(element)).join(" ") + } }, groupStops: (control, stops) => { const target = element(control) diff --git a/packages/devtools/src/internal/keyboardWalk.ts b/packages/devtools/src/internal/keyboardWalk.ts index 2776130f..64cb8a0f 100644 --- a/packages/devtools/src/internal/keyboardWalk.ts +++ b/packages/devtools/src/internal/keyboardWalk.ts @@ -16,6 +16,17 @@ * - `unreachable-control`: a visible control neither Tab nor the arrows of * its group reach. * + * A loaded screen is walked once it has stopped changing: no request in + * flight, and the same menu, focus, and visible controls for 400 ms, or + * `settle` at the latest. `focus-dropped` presses on several pages at once + * (`pages`); a press that did not keep focus is run again with no other page + * open, and that second result is the one reported, so a failure never comes + * from a busy machine. + * + * A page that stops answering for `stall` milliseconds (its main thread is + * busy, or its renderer hung) is closed and reported as a `page-error`, so + * the walk fails instead of waiting for ever. + * * A screen on which no check could be decided is a failure too * (`nothing-checked`): nothing checked is never an exit code of 0. * @@ -42,6 +53,7 @@ export interface ControlName { export type Step = | { readonly press: string } | { readonly click: { readonly role: string; readonly name: string } } + | { readonly waitFor: { readonly role: string; readonly name: string } } | { readonly wait: number } /** One screen of the walk: where it starts, how to reach it, and what not to press. */ @@ -81,7 +93,7 @@ const parseControlName = (value: unknown, at: string): Result.Result => { if (!isRecord(value) || Object.keys(value).length !== 1) { - return Result.failure(`${at} must be one of { "press": "Escape" }, { "click": { "role": "button", "name": "Play" } }, { "wait": 500 }`) + return Result.failure(`${at} must be one of { "press": "Escape" }, { "click": { "role": "button", "name": "Play" } }, { "waitFor": { "role": "button", "name": "Play" } }, { "wait": 500 }`) } if ("press" in value) { return nonEmpty(value["press"]) ? Result.success({ press: value["press"] }) : Result.failure(`${at}.press must be a key name, as Playwright's keyboard.press takes it ("Enter", "Escape", "a")`) @@ -92,16 +104,18 @@ const parseStep = (value: unknown, at: string): Result.Result => { ? Result.success({ wait }) : Result.failure(`${at}.wait must be a whole number of milliseconds from 0 to 60000`) } - if ("click" in value) { - const click = value["click"] - if (!isRecord(click)) return Result.failure(`${at}.click must be an object { "role": string, "name": string }`) - const extra = onlyKeys(click, ["role", "name"], `${at}.click`) + for (const kind of ["click", "waitFor"] as const) { + if (!(kind in value)) continue + const named = value[kind] + if (!isRecord(named)) return Result.failure(`${at}.${kind} must be an object { "role": string, "name": string }`) + const extra = onlyKeys(named, ["role", "name"], `${at}.${kind}`) if (extra !== undefined) return Result.failure(extra) - if (!nonEmpty(click["role"])) return Result.failure(`${at}.click.role must be a non-empty string: an ARIA role such as "button"`) - if (!nonEmpty(click["name"])) return Result.failure(`${at}.click.name must be a non-empty string: the control's accessible name`) - return Result.success({ click: { role: click["role"], name: click["name"] } }) + if (!nonEmpty(named["role"])) return Result.failure(`${at}.${kind}.role must be a non-empty string: an ARIA role such as "button"`) + if (!nonEmpty(named["name"])) return Result.failure(`${at}.${kind}.name must be a non-empty string: the control's accessible name`) + const control = { role: named["role"], name: named["name"] } + return Result.success(kind === "click" ? { click: control } : { waitFor: control }) } - return Result.failure(`${at} must be one of { "press": ... }, { "click": ... }, { "wait": ... }; ${Object.keys(value)[0]} is not a step`) + return Result.failure(`${at} must be one of { "press": ... }, { "click": ... }, { "waitFor": ... }, { "wait": ... }; ${Object.keys(value)[0]} is not a step`) } const parseScreen = (value: unknown, at: string): Result.Result => { @@ -209,8 +223,12 @@ export const fixes: Record = { export interface Options { readonly url: string readonly route: Route - /** Milliseconds after a page load before the route's steps run (default 1500). */ + /** The most milliseconds a loaded page gets to stop changing before the route's steps run (default 1500). */ readonly settle?: number + /** How many pages press controls at once for `focus-dropped` (default 1). */ + readonly pages?: number + /** How many milliseconds a page may go without answering before it is closed and reported (default 20000). */ + readonly stall?: number } interface Described { @@ -255,7 +273,7 @@ export const walk = async (browser: PlaywrightTypes.Browser, options: Options): const failures: Array = [] const outcomes: Array = [] for (const screen of options.route.screens) { - const result = await walkScreen(browser, options.url, screen, settle) + const result = await walkScreen(browser, { url: options.url, settle, pages: Math.max(1, options.pages ?? 1), stall: options.stall ?? 20_000 }, screen) failures.push(...result.failures) outcomes.push(...result.outcomes) } @@ -265,16 +283,52 @@ export const walk = async (browser: PlaywrightTypes.Browser, options: Options): interface Opened { readonly page: PlaywrightTypes.Page readonly errors: Array + /** Requests in flight, and a count that moves whenever one starts or ends. */ + readonly network: { inFlight: number; activity: number } + /** True once the page stopped answering and was closed for it. */ + readonly stalled: () => boolean readonly state: () => Promise readonly control: (described: Described) => Promise readonly close: () => Promise } -const open = async (browser: PlaywrightTypes.Browser): Promise => { +/** What every page of one walk shares. */ +interface Walk { + readonly url: string + readonly settle: number + readonly pages: number + readonly stall: number +} + +const stalledSaw = (stall: number): string => `the page stopped answering for ${stall / 1000} seconds: its main thread is busy, or its renderer hung` + +/** Whether the page runs a script within `most` milliseconds. A page that is closed or navigating has answered. */ +const answers = (page: PlaywrightTypes.Page, most: number): Promise => + new Promise((resolve) => { + // oxlint-disable-next-line ritmo/use-clock-for-time -- a tool on the developer's machine, not a simulation: it times a page that no longer answers, so no page timer or tick can do it + const timer = setTimeout(() => resolve(false), most) + void page.evaluate(() => true).then(() => true, () => true).then((answered) => { + clearTimeout(timer) + resolve(answered) + }) + }) + +const open = async (browser: PlaywrightTypes.Browser, stall: number): Promise => { const page = await browser.newPage({ viewport: { width: 1280, height: 800 } }) await page.addInitScript(pageScript) const errors: Array = [] page.on("pageerror", (error) => errors.push(error.message)) + const network = { inFlight: 0, activity: 0 } + page.on("request", () => { + network.inFlight++ + network.activity++ + }) + const ended = (): void => { + network.inFlight = Math.max(0, network.inFlight - 1) + network.activity++ + } + page.on("requestfinished", ended) + page.on("requestfailed", ended) const cdp = await page.context().newCDPSession(page) const state = () => page.evaluate(() => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as { state: () => PageState }).state()) const control = async (described: Described): Promise => { @@ -291,7 +345,20 @@ const open = async (browser: PlaywrightTypes.Browser): Promise => { return { role: "", name: "", selector: described.selector } } } - return { page, errors, state, control, close: () => page.close() } + // Closing a page that stopped answering rejects every call still waiting on it, so the walk moves on. + let stalled = false + const watch = async (): Promise => { + while (!page.isClosed()) { + if (!(await answers(page, stall))) { + stalled = !page.isClosed() + await page.close().catch(() => undefined) + return + } + await page.waitForTimeout(1000).catch(() => undefined) + } + } + void watch() + return { page, errors, network, stalled: () => stalled, state, control, close: () => page.close().catch(() => undefined) } } /** Opens the screen: loads its URL, lets it settle, and runs its steps. A failed step is returned as its message. */ @@ -302,14 +369,17 @@ const reach = async (opened: Opened, url: string, screen: Screen, settle: number } catch (error) { return `the page did not load: ${error instanceof Error ? error.message : String(error)}` } - await page.waitForTimeout(settle) + await settled(opened, settle) for (const [index, step] of screen.steps.entries()) { const at = `steps[${index}]` try { if ("press" in step) await page.keyboard.press(step.press) else if ("wait" in step) await page.waitForTimeout(step.wait) - else { - const target = await find(opened, step.click) + else if ("waitFor" in step) { + const target = await find(opened, step.waitFor, 15_000) + if (target === undefined) return `${at}: no visible ${step.waitFor.role} named ${JSON.stringify(step.waitFor.name)} showed within 15 seconds (names are the ones this report prints)` + } else { + const target = await find(opened, step.click, 5000) if (target === undefined) return `${at}: no visible ${step.click.role} named ${JSON.stringify(step.click.name)} to click (names are the ones this report prints)` await page.locator(target.selector).first().click({ timeout: 5000 }) } @@ -324,20 +394,61 @@ const reach = async (opened: Opened, url: string, screen: Screen, settle: number /** * The first visible element with this role and accessible name, by the names * this report prints (Chromium's accessibility tree), so a name copied from a - * report finds its control. Waits up to five seconds for it to show. + * report finds its control. Waits up to about `most` milliseconds for it to show. */ -const find = async (opened: Opened, wanted: { readonly role: string; readonly name: string }): Promise => { - for (let tries = 0; tries < 10; tries++) { +const find = async (opened: Opened, wanted: { readonly role: string; readonly name: string }, most: number): Promise => { + for (let waited = 0; waited <= most; waited += 150) { const candidates = await call(opened.page, "controls", null) as ReadonlyArray for (const candidate of candidates) { const control = await opened.control(candidate) if (control.role === wanted.role && control.name === wanted.name) return candidate } - await opened.page.waitForTimeout(500) + await opened.page.waitForTimeout(150) } return undefined } +/** + * Waits for a loaded page to stop changing: no request in flight, and the + * same menu, focus, and visible controls for 400 ms of the page's own clock, + * or `most` milliseconds at the latest. A page that never goes quiet (it + * polls a server, or keeps redrawing its controls) waits the whole `most`. + */ +const settled = async (opened: Opened, most: number): Promise => { + const quiet = 400 + let start: number | undefined + let since = 0 + let seen: string | undefined + // The count bounds a page whose clock cannot be read (it is navigating). + for (let polls = 0; polls * 50 <= most * 2; polls++) { + try { + const at = await opened.page.evaluate(() => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as { signature: () => { now: number; key: string } }).signature()) + const key = `${at.key} ${opened.network.activity}` + if (start === undefined || at.now < since) start = since = at.now + if (key !== seen || opened.network.inFlight > 0) since = at.now + seen = key + if (at.now - since >= quiet || at.now - start >= most) return + } catch { + seen = undefined + } + await opened.page.waitForTimeout(50) + } +} + +/** Runs `run` on every item, at most `most` at once; the results keep the items' order. */ +const inParallel = async (items: ReadonlyArray, most: number, run: (item: A) => Promise): Promise> => { + const results: Array = [] + let next = 0 + const worker = async (): Promise => { + while (next < items.length) { + const index = next++ + results[index] = await run(items[index]!) + } + } + await Promise.all(Array.from({ length: Math.min(most, items.length) }, worker)) + return results +} + const call = (page: PlaywrightTypes.Page, name: string, argument: A): Promise => page.evaluate( ({ name, argument }) => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as Record unknown>)[name]!(argument), @@ -361,7 +472,7 @@ interface ScreenResult { readonly outcomes: ReadonlyArray } -const walkScreen = async (browser: PlaywrightTypes.Browser, url: string, screen: Screen, settle: number): Promise => { +const walkScreen = async (browser: PlaywrightTypes.Browser, { url, settle, pages, stall }: Walk, screen: Screen): Promise => { const failures: Array = [] const outcomes: Array = [] const fail = (check: Failure["check"], saw: string, control?: Control): void => { @@ -369,9 +480,16 @@ const walkScreen = async (browser: PlaywrightTypes.Browser, url: string, screen: } const skipAll = (reason: string): ScreenResult => ({ failures, outcomes: checks.map((check) => notChecked(check, screen.name, reason)) }) - const opened = await open(browser) + const opened = await open(browser, stall) + // What was seen before a page stopped answering is dropped with it: the report says one thing about the screen. + const stalled = (): ScreenResult => { + failures.length = 0 + fail("page-error", stalledSaw(stall)) + return skipAll("the page stopped answering") + } try { const reached = await reach(opened, url, screen, settle) + if (opened.stalled()) return stalled() if (reached !== undefined) { fail("route-step", reached) return skipAll(`the route did not reach this screen (${reached})`) @@ -521,25 +639,30 @@ const walkScreen = async (browser: PlaywrightTypes.Browser, url: string, screen: const skip = screen.noPress const results = { decided: 0, undecided: [] as Array, failed: 0 } const candidates = stops.filter((stop) => stop.native && stop.keys.length > 0) + const skipped = (stop: Described): string | undefined => + matches(controls.get(stop.id)!, skip) + ? "the route marks it not to be pressed" + : stop.external + ? "a link to another site is not pressed for real" + : failedIds.has(stop.id) + ? "Enter or Space does not press it (key-not-pressed)" + : undefined + const pressed = await inParallel(candidates.filter((stop) => skipped(stop) === undefined), pages, (stop) => pressForReal(browser, { url, settle, pages, stall }, screen, stop)) + let at = 0 for (const stop of candidates) { const control = controls.get(stop.id)! - if (matches(control, skip)) { - results.undecided.push({ control, reason: "the route marks it not to be pressed" }) - continue - } - if (stop.external) { - results.undecided.push({ control, reason: "a link to another site is not pressed for real" }) + const reason = skipped(stop) + if (reason !== undefined) { + results.undecided.push({ control, reason }) continue } - if (failedIds.has(stop.id)) { - results.undecided.push({ control, reason: "Enter or Space does not press it (key-not-pressed)" }) - continue - } - const outcome = await pressForReal(browser, url, screen, settle, stop) + let outcome = pressed[at++]! + // A press that shared the machine with others and did not keep focus is run again alone: only that result is reported. + if (pages > 1 && outcome._tag !== "Kept") outcome = await pressForReal(browser, { url, settle, pages, stall }, screen, stop) if (outcome._tag === "Undecided") results.undecided.push({ control, reason: outcome.reason }) else if (outcome._tag === "PageError") { - fail("page-error", `the page threw when ${keyName(outcome.key)} pressed it: ${outcome.message}`, control) - results.undecided.push({ control, reason: "the page threw when it was pressed" }) + fail("page-error", outcome.saw, control) + results.undecided.push({ control, reason: outcome.reason }) } else { results.decided++ if (outcome._tag === "Dropped") { @@ -558,6 +681,9 @@ const walkScreen = async (browser: PlaywrightTypes.Browser, url: string, screen: fail("nothing-checked", "no check could be decided on this screen: no menu is open, and it shows no control") } return { failures, outcomes: checks.map((check) => outcomes.find((outcome) => outcome.check === check)!) } + } catch (error) { + if (!opened.stalled()) throw error + return stalled() } finally { await opened.close() } @@ -585,14 +711,15 @@ const probe = async (opened: Opened, stop: number, most: number): Promise => { - const opened = await open(browser) +const pressForReal = async (browser: PlaywrightTypes.Browser, { url, settle, stall }: Walk, screen: Screen, stop: Described): Promise => { + const opened = await open(browser, stall) try { const reached = await reach(opened, url, screen, settle) + if (opened.stalled()) return { _tag: "PageError", saw: stalledSaw(stall), reason: "the page stopped answering" } if (reached !== undefined) return { _tag: "Undecided", reason: `the route did not reach the screen again (${reached})` } let at = await opened.state() for (let tabs = 0; at.active?.selector !== stop.selector && tabs < 120; tabs++) { @@ -612,12 +739,13 @@ const pressForReal = async (browser: PlaywrightTypes.Browser, url: string, scree opened.page.off("popup", onPopup) await Promise.all(popups.map((popup) => popup.close().catch(() => undefined))) if (popups.length > 0) return { _tag: "Undecided", reason: "the press opened another page" } - if (opened.errors.length > before) return { _tag: "PageError", key, message: opened.errors[before]! } + if (opened.errors.length > before) return { _tag: "PageError", saw: `the page threw when ${keyName(key)} pressed it: ${opened.errors[before]!}`, reason: "the page threw when it was pressed" } const after = await opened.state() if (after.url.replace(/#.*$/, "") !== at.url.replace(/#.*$/, "")) return { _tag: "Undecided", reason: "the press left the page" } if (!result.pressed) return { _tag: "Undecided", reason: `${keyName(key)} did not press it after a reload` } return after.active === null ? { _tag: "Dropped", key } : { _tag: "Kept", key } } catch (error) { + if (opened.stalled()) return { _tag: "PageError", saw: stalledSaw(stall), reason: "the page stopped answering" } return { _tag: "Undecided", reason: `the press failed: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}` } } finally { await opened.close() diff --git a/packages/devtools/test/KeyboardWalk.test.ts b/packages/devtools/test/KeyboardWalk.test.ts index 7d59ef67..1a165e94 100644 --- a/packages/devtools/test/KeyboardWalk.test.ts +++ b/packages/devtools/test/KeyboardWalk.test.ts @@ -32,12 +32,12 @@ describe("keyboard route", () => { it("keeps every kind of step and both kinds of noPress", () => { const parsed = KeyboardWalk.parseRoute({ screens: [ - { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, + { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { waitFor: { role: "button", name: "Resume" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, { name: "end", noPress: true } ] }) expect(parsed.ok && parsed.value.screens).toEqual([ - { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, + { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { waitFor: { role: "button", name: "Resume" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, { name: "end", path: "", steps: [], noPress: true } ]) }) @@ -54,6 +54,7 @@ describe("keyboard route", () => { expect(error({ screens: [{ name: "a" }, { name: "a" }] })).toBe("route.screens[1].name \"a\" is already the name of another screen") expect(error({ screens: [{ name: "a", url: "/" }] })).toBe("route.screens[0].url is not a field (expected name, path, steps, noPress)") expect(error({ screens: [{ name: "a", steps: [{ click: { role: "button" } }] }] })).toBe("route.screens[0].steps[0].click.name must be a non-empty string: the control's accessible name") + expect(error({ screens: [{ name: "a", steps: [{ waitFor: { name: "Play" } }] }] })).toBe("route.screens[0].steps[0].waitFor.role must be a non-empty string: an ARIA role such as \"button\"") expect(error({ screens: [{ name: "a", steps: [{ wait: 1.5 }] }] })).toBe("route.screens[0].steps[0].wait must be a whole number of milliseconds from 0 to 60000") expect(error({ screens: [{ name: "a", steps: [{ press: "a", wait: 1 }] }] })).toMatch(/^route\.screens\[0\]\.steps\[0\] must be one of/) expect(error({ screens: [{ name: "a", steps: [{ type: "abc" }] }] })).toMatch(/type is not a step$/) @@ -114,8 +115,8 @@ describe.skipIf(skipped !== undefined)("ritmo-devtools keyboard", () => { await new Promise((done) => server?.close(done)) }) - const walk = (screens: ReadonlyArray & { readonly name: string }>) => - KeyboardWalk.walk(browser, { url: base, route: { screens: screens.map((screen) => ({ path: "", steps: [], noPress: [], ...screen })) }, settle: 100 }) + const walk = (screens: ReadonlyArray & { readonly name: string }>, options: { readonly settle?: number; readonly pages?: number; readonly stall?: number } = {}) => + KeyboardWalk.walk(browser, { url: base, route: { screens: screens.map((screen) => ({ path: "", steps: [], noPress: [], ...screen })) }, settle: 100, ...options }) const statuses = (report: KeyboardWalkModule.Report, screen: string) => Object.fromEntries(report.outcomes.filter((outcome) => outcome.screen === screen).map((outcome) => [outcome.check, outcome.status])) @@ -204,6 +205,40 @@ describe.skipIf(skipped !== undefined)("ritmo-devtools keyboard", () => { expect(KeyboardWalk.format(report).split("\n").at(-1)).toMatch(/: 1 failure \(1 from the route or the page, not from a check\); checks 0 failed, 0 passed, 6 not checked on 1 screen$/) }, 120_000) + it("gives the same report when it presses on several pages at once", async () => { + const screens = [{ name: "play" }, { name: "paused", path: "#paused" }, { name: "dropped", path: "focus-dropped.html" }] + const alone = await walk(screens) + const together = await walk(screens, { pages: 3 }) + expect(together).toEqual(alone) + expect(together.failures.map((failure) => [failure.screen, failure.check])).toEqual([["dropped", "focus-dropped"]]) + }, 180_000) + + it("waits for a screen to stop changing, and for the control a waitFor step names", async () => { + const report = await walk([ + // Shown 300 ms after the load: the page has not settled until it is there. + { name: "soon", path: "late-control.html" }, + // Shown after the page went quiet: only a waitFor step reaches it. + { name: "late", path: "late-control.html?after=2500", steps: [{ waitFor: { role: "button", name: "Options" } }] }, + { name: "missed", path: "late-control.html?after=2500" } + ], { settle: 1500 }) + expect(report.failures).toEqual([]) + const reached = (screen: string) => report.outcomes.find((outcome) => outcome.screen === screen && outcome.check === "unnamed-control")?.decided + expect([reached("soon"), reached("late"), reached("missed")]).toEqual([2, 2, 1]) + }, 180_000) + + it("reports a page that stops answering, and does not wait for it", async () => { + const report = await walk([{ name: "stuck", path: "stuck.html" }, { name: "play" }], { stall: 2000, pages: 2 }) + expect(report.failures).toEqual([{ + check: "page-error", + screen: "stuck", + saw: "the page stopped answering for 2 seconds: its main thread is busy, or its renderer hung", + fix: KeyboardWalk.fixes["page-error"] + }]) + expect(report.outcomes.filter((outcome) => outcome.screen === "stuck").every((outcome) => outcome.status === "not-checked")).toBe(true) + // The next screen is walked as usual. + expect(statuses(report, "play")["focus-dropped"]).toBe("passed") + }, 120_000) + it("does not press what the route marks", async () => { const report = await walk([{ name: "some", noPress: [{ name: "Menu", role: "button" }] }, { name: "none", noPress: true }]) expect(report.outcomes.find((outcome) => outcome.screen === "some" && outcome.check === "focus-dropped")).toMatchObject({ diff --git a/packages/devtools/test/fixtures/keyboard/late-control.html b/packages/devtools/test/fixtures/keyboard/late-control.html new file mode 100644 index 00000000..1c93b0ff --- /dev/null +++ b/packages/devtools/test/fixtures/keyboard/late-control.html @@ -0,0 +1,12 @@ + + +a control that shows after the page loads + + diff --git a/packages/devtools/test/fixtures/keyboard/stuck.html b/packages/devtools/test/fixtures/keyboard/stuck.html new file mode 100644 index 00000000..3810428e --- /dev/null +++ b/packages/devtools/test/fixtures/keyboard/stuck.html @@ -0,0 +1,9 @@ + + +the page stops answering + +