From 4a2f24819d72d705e1f8fda82666e6c797f74565 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 19:58:39 +0000 Subject: [PATCH 1/6] devtools: share the route and screen opening between walks The layout audit reads the same route file as the keyboard walk and reaches screens the same way, so the route parser and the code that opens a screen move out of keyboardWalk.ts. The route gains a type step (to fill a name field on the way to a screen) and the layout audit's fields (viewports, minTarget, allow), which the keyboard walk ignores. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- packages/devtools/src/cli.ts | 5 +- .../devtools/src/internal/keyboardWalk.ts | 236 +-------------- packages/devtools/src/internal/pageOpen.ts | 142 +++++++++ packages/devtools/src/internal/route.ts | 273 ++++++++++++++++++ packages/devtools/test/KeyboardWalk.test.ts | 47 +-- packages/devtools/test/Route.test.ts | 77 +++++ 6 files changed, 502 insertions(+), 278 deletions(-) create mode 100644 packages/devtools/src/internal/pageOpen.ts create mode 100644 packages/devtools/src/internal/route.ts create mode 100644 packages/devtools/test/Route.test.ts diff --git a/packages/devtools/src/cli.ts b/packages/devtools/src/cli.ts index b0cfee78..cce2fa28 100755 --- a/packages/devtools/src/cli.ts +++ b/packages/devtools/src/cli.ts @@ -84,6 +84,7 @@ import { pathToFileURL } from "node:url" import * as DependencyScan from "./internal/dependencyScan.ts" import * as KeyboardWalk from "./internal/keyboardWalk.ts" import * as PackageVersions from "./internal/packageVersions.ts" +import * as Route from "./internal/route.ts" import * as SimulationCoverage from "./internal/simulationCoverage.ts" import * as SkillLink from "./internal/skillLink.ts" import * as Lint from "./Lint.ts" @@ -272,7 +273,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { console.error(`--url must be an http(s) URL of the game's page, such as http://localhost:5173/; got ${url}`) return 2 } - let route = KeyboardWalk.startRoute + let route = Route.startRoute if (routeFile !== undefined) { let data: unknown try { @@ -281,7 +282,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { console.error(`${routeFile} could not be read as JSON: ${error instanceof Error ? error.message : String(error)}`) return 2 } - const parsed = KeyboardWalk.parseRoute(data) + const parsed = Route.parseRoute(data) if (!parsed.ok) { console.error(`${routeFile} is not a keyboard route: ${parsed.error}`) return 2 diff --git a/packages/devtools/src/internal/keyboardWalk.ts b/packages/devtools/src/internal/keyboardWalk.ts index 2776130f..92a8a8e0 100644 --- a/packages/devtools/src/internal/keyboardWalk.ts +++ b/packages/devtools/src/internal/keyboardWalk.ts @@ -28,138 +28,15 @@ */ import type * as PlaywrightTypes from "playwright" -import * as Result from "@typeonce/ritmo/Result" - -import { pageScript } from "./keyboardPage.ts" - -/** A control a route names: by accessible name, and by role when given. */ -export interface ControlName { - readonly name: string - readonly role?: string -} - -/** One thing a route does to reach a screen. */ -export type Step = - | { readonly press: string } - | { readonly click: { readonly role: string; readonly name: string } } - | { readonly wait: number } - -/** One screen of the walk: where it starts, how to reach it, and what not to press. */ -export interface Screen { - readonly name: string - /** Appended to the URL: `/`, `?map=8`, `#settings`. */ - readonly path: string - readonly steps: ReadonlyArray - /** `true` presses nothing on this screen (no `focus-dropped`); a list skips those controls. */ - readonly noPress: true | ReadonlyArray -} - -export interface Route { - readonly screens: ReadonlyArray -} - -/** The route of a game that writes none: the start page, as one screen. */ -export const startRoute: Route = { screens: [{ name: "start", path: "", steps: [], noPress: [] }] } - -const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) - -const onlyKeys = (value: Record, keys: ReadonlyArray, at: string): string | undefined => { - const extra = Object.keys(value).find((key) => !keys.includes(key)) - return extra === undefined ? undefined : `${at}.${extra} is not a field (expected ${keys.join(", ")})` -} - -const nonEmpty = (value: unknown): value is string => typeof value === "string" && value.trim() !== "" - -const parseControlName = (value: unknown, at: string): Result.Result => { - if (!isRecord(value)) return Result.failure(`${at} must be an object { "name": string, "role"?: string }`) - const extra = onlyKeys(value, ["name", "role"], at) - if (extra !== undefined) return Result.failure(extra) - if (!nonEmpty(value["name"])) return Result.failure(`${at}.name must be a non-empty string: the control's accessible name`) - if (value["role"] !== undefined && !nonEmpty(value["role"])) return Result.failure(`${at}.role must be a non-empty string when given: an ARIA role such as "button"`) - return Result.success(value["role"] === undefined ? { name: value["name"] } : { name: value["name"], role: value["role"] }) -} - -const parseStep = (value: unknown, at: string): Result.Result => { - if (!isRecord(value) || Object.keys(value).length !== 1) { - return Result.failure(`${at} must be one of { "press": "Escape" }, { "click": { "role": "button", "name": "Play" } }, { "wait": 500 }`) - } - if ("press" in value) { - return nonEmpty(value["press"]) ? Result.success({ press: value["press"] }) : Result.failure(`${at}.press must be a key name, as Playwright's keyboard.press takes it ("Enter", "Escape", "a")`) - } - if ("wait" in value) { - const wait = value["wait"] - return typeof wait === "number" && Number.isInteger(wait) && wait >= 0 && wait <= 60_000 - ? Result.success({ wait }) - : Result.failure(`${at}.wait must be a whole number of milliseconds from 0 to 60000`) - } - if ("click" in value) { - const click = value["click"] - if (!isRecord(click)) return Result.failure(`${at}.click must be an object { "role": string, "name": string }`) - const extra = onlyKeys(click, ["role", "name"], `${at}.click`) - if (extra !== undefined) return Result.failure(extra) - if (!nonEmpty(click["role"])) return Result.failure(`${at}.click.role must be a non-empty string: an ARIA role such as "button"`) - if (!nonEmpty(click["name"])) return Result.failure(`${at}.click.name must be a non-empty string: the control's accessible name`) - return Result.success({ click: { role: click["role"], name: click["name"] } }) - } - return Result.failure(`${at} must be one of { "press": ... }, { "click": ... }, { "wait": ... }; ${Object.keys(value)[0]} is not a step`) -} - -const parseScreen = (value: unknown, at: string): Result.Result => { - if (!isRecord(value)) return Result.failure(`${at} must be an object { "name", "path"?, "steps"?, "noPress"? }`) - const extra = onlyKeys(value, ["name", "path", "steps", "noPress"], at) - if (extra !== undefined) return Result.failure(extra) - if (!nonEmpty(value["name"])) return Result.failure(`${at}.name must be a non-empty string`) - const path = value["path"] ?? "" - if (typeof path !== "string") return Result.failure(`${at}.path must be a string appended to the URL ("/", "?map=8")`) - const rawSteps = value["steps"] ?? [] - if (!Array.isArray(rawSteps)) return Result.failure(`${at}.steps must be an array`) - const steps: Array = [] - for (const [index, raw] of rawSteps.entries()) { - const step = parseStep(raw, `${at}.steps[${index}]`) - if (!step.ok) return step - steps.push(step.value) - } - const rawNoPress = value["noPress"] ?? [] - if (rawNoPress === true) return Result.success({ name: value["name"], path, steps, noPress: true }) - if (!Array.isArray(rawNoPress)) return Result.failure(`${at}.noPress must be true (press nothing on this screen) or an array of { "name", "role"? }`) - const noPress: Array = [] - for (const [index, raw] of rawNoPress.entries()) { - const control = parseControlName(raw, `${at}.noPress[${index}]`) - if (!control.ok) return control - noPress.push(control.value) - } - return Result.success({ name: value["name"], path, steps, noPress }) -} - -/** Validates a route file's JSON; the error names the field that is wrong. */ -export const parseRoute = (value: unknown): Result.Result => { - if (!isRecord(value)) return Result.failure(`the route must be an object { "screens": [...] }`) - const extra = onlyKeys(value, ["screens"], "route") - if (extra !== undefined) return Result.failure(extra) - const raw = value["screens"] - if (!Array.isArray(raw) || raw.length === 0) return Result.failure(`route.screens must be a non-empty array`) - const screens: Array = [] - for (const [index, entry] of raw.entries()) { - const screen = parseScreen(entry, `route.screens[${index}]`) - if (!screen.ok) return screen - if (screens.some((other) => other.name === screen.value.name)) return Result.failure(`route.screens[${index}].name "${screen.value.name}" is already the name of another screen`) - screens.push(screen.value) - } - return Result.success({ screens }) -} +import type { Control, Described, Opened, PageState } from "./pageOpen.ts" +import { call, open, reach } from "./pageOpen.ts" +import type { ControlName, Route, Screen } from "./route.ts" /** The six checks. */ export type CheckId = "focus-outside-menu" | "tab-leaves-menu" | "unnamed-control" | "key-not-pressed" | "focus-dropped" | "unreachable-control" export const checks: ReadonlyArray = ["focus-outside-menu", "tab-leaves-menu", "unnamed-control", "key-not-pressed", "focus-dropped", "unreachable-control"] -/** A control as the report names it: its role and name from the accessibility tree, and a CSS selector that finds it. */ -export interface Control { - readonly role: string - readonly name: string - readonly selector: string -} - /** One failure: which check, on which screen, which control, what the walk saw, and what to change. */ export interface Failure { readonly check: CheckId | "route-step" | "page-error" | "nothing-checked" @@ -213,24 +90,6 @@ export interface Options { readonly settle?: number } -interface Described { - readonly id: number - readonly tag: string - readonly selector: string - readonly keys: ReadonlyArray - readonly native: boolean - /** A link to another site: pressing it for real would leave the game, so it is not pressed. */ - readonly external: boolean - readonly canvas: boolean -} - -interface PageState { - readonly active: Described | null - readonly menu: Described | null - readonly inMenu: boolean - readonly url: string -} - interface PressResult { readonly pressed: boolean readonly keydownPrevented: boolean | null @@ -239,13 +98,6 @@ interface PressResult { const keyName = (key: string): string => key === " " ? "Space" : key -/** Joins a screen's path to the base URL, keeping the base's own path. */ -const screenUrl = (base: string, path: string): string => { - if (path === "") return base - if (path.startsWith("?") || path.startsWith("#")) return base.replace(/[?#].*$/, "") + path - return new URL(path, base).href -} - const describeControl = (control: Control): string => `${control.role === "" ? "element" : control.role} ${control.name === "" ? "(no name)" : JSON.stringify(control.name)} at ${control.selector}` @@ -262,88 +114,6 @@ export const walk = async (browser: PlaywrightTypes.Browser, options: Options): return { ok: failures.length === 0, url: options.url, screens: options.route.screens.map((screen) => screen.name), failures, outcomes } } -interface Opened { - readonly page: PlaywrightTypes.Page - readonly errors: Array - readonly state: () => Promise - readonly control: (described: Described) => Promise - readonly close: () => Promise -} - -const open = async (browser: PlaywrightTypes.Browser): Promise => { - const page = await browser.newPage({ viewport: { width: 1280, height: 800 } }) - await page.addInitScript(pageScript) - const errors: Array = [] - page.on("pageerror", (error) => errors.push(error.message)) - const cdp = await page.context().newCDPSession(page) - const state = () => page.evaluate(() => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as { state: () => PageState }).state()) - const control = async (described: Described): Promise => { - try { - const evaluated = await cdp.send("Runtime.evaluate", { expression: `__ritmoKeyboardWalk.element(${described.id})`, objectGroup: "ritmo-keyboard" }) - const objectId = evaluated.result.objectId - if (objectId === undefined) return { role: "", name: "", selector: described.selector } - const tree = await cdp.send("Accessibility.getPartialAXTree", { objectId, fetchRelatives: false }) - const node = tree.nodes[0] - const role = node?.role?.value - const name = node?.name?.value - return { role: typeof role === "string" ? role : "", name: typeof name === "string" ? name.trim() : "", selector: described.selector } - } catch { - return { role: "", name: "", selector: described.selector } - } - } - return { page, errors, state, control, close: () => page.close() } -} - -/** Opens the screen: loads its URL, lets it settle, and runs its steps. A failed step is returned as its message. */ -const reach = async (opened: Opened, url: string, screen: Screen, settle: number): Promise => { - const { page } = opened - try { - await page.goto(screenUrl(url, screen.path), { waitUntil: "load" }) - } catch (error) { - return `the page did not load: ${error instanceof Error ? error.message : String(error)}` - } - await page.waitForTimeout(settle) - for (const [index, step] of screen.steps.entries()) { - const at = `steps[${index}]` - try { - if ("press" in step) await page.keyboard.press(step.press) - else if ("wait" in step) await page.waitForTimeout(step.wait) - else { - const target = await find(opened, step.click) - if (target === undefined) return `${at}: no visible ${step.click.role} named ${JSON.stringify(step.click.name)} to click (names are the ones this report prints)` - await page.locator(target.selector).first().click({ timeout: 5000 }) - } - } catch (error) { - return `${at} failed: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}` - } - await page.waitForTimeout(300) - } - return undefined -} - -/** - * The first visible element with this role and accessible name, by the names - * this report prints (Chromium's accessibility tree), so a name copied from a - * report finds its control. Waits up to five seconds for it to show. - */ -const find = async (opened: Opened, wanted: { readonly role: string; readonly name: string }): Promise => { - for (let tries = 0; tries < 10; tries++) { - const candidates = await call(opened.page, "controls", null) as ReadonlyArray - for (const candidate of candidates) { - const control = await opened.control(candidate) - if (control.role === wanted.role && control.name === wanted.name) return candidate - } - await opened.page.waitForTimeout(500) - } - return undefined -} - -const call = (page: PlaywrightTypes.Page, name: string, argument: A): Promise => - page.evaluate( - ({ name, argument }) => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as Record unknown>)[name]!(argument), - { name, argument } - ) - const press = async (opened: Opened, key: string, stop: boolean, wait: number): Promise => { await call(opened.page, "arm", stop) await opened.page.keyboard.press(key) diff --git a/packages/devtools/src/internal/pageOpen.ts b/packages/devtools/src/internal/pageOpen.ts new file mode 100644 index 00000000..34481461 --- /dev/null +++ b/packages/devtools/src/internal/pageOpen.ts @@ -0,0 +1,142 @@ +/** + * Opening one screen of a route in Chromium, for `ritmo-devtools keyboard` + * and `ritmo-devtools layout`: a page at a viewport, the keyboard walk's page + * script installed (it finds controls by the names a report prints), the + * route's steps run, and errors the page throws collected. + * + * @internal + */ +import type * as PlaywrightTypes from "playwright" + +import { pageScript } from "./keyboardPage.ts" +import type { Insets, Screen } from "./route.ts" +import { screenUrl } from "./route.ts" + +/** A control as the report names it: its role and name from the accessibility tree, and a CSS selector that finds it. */ +export interface Control { + readonly role: string + readonly name: string + readonly selector: string +} + +export interface Described { + readonly id: number + readonly tag: string + readonly selector: string + readonly keys: ReadonlyArray + readonly native: boolean + /** A link to another site: pressing it for real would leave the game, so it is not pressed. */ + readonly external: boolean + readonly canvas: boolean +} + +export interface PageState { + readonly active: Described | null + readonly menu: Described | null + readonly inMenu: boolean + readonly url: string +} + +export interface Opened { + readonly page: PlaywrightTypes.Page + readonly errors: Array + readonly state: () => Promise + readonly control: (described: Described) => Promise + readonly close: () => Promise +} + +/** How to open a page: its size, a touch screen or not, what the device keeps for itself, and more scripts to install. */ +export interface OpenOptions { + readonly width: number + readonly height: number + readonly touch: boolean + readonly insets?: Insets + readonly reducedMotion?: boolean + readonly scripts?: ReadonlyArray +} + +const desktop: OpenOptions = { width: 1280, height: 800, touch: false } + +export const open = async (browser: PlaywrightTypes.Browser, options: OpenOptions = desktop): Promise => { + const context = await browser.newContext({ + viewport: { width: options.width, height: options.height }, + hasTouch: options.touch, + isMobile: options.touch, + reducedMotion: options.reducedMotion === true ? "reduce" : "no-preference" + }) + const page = await context.newPage() + await page.addInitScript(pageScript) + for (const script of options.scripts ?? []) await page.addInitScript(script) + const errors: Array = [] + page.on("pageerror", (error) => errors.push(error.message)) + const cdp = await context.newCDPSession(page) + // The device's own space (a notch, a home bar): what CSS reads as env(safe-area-inset-*). + if (options.insets !== undefined) await cdp.send("Emulation.setSafeAreaInsetsOverride", { insets: options.insets }) + const state = () => page.evaluate(() => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as { state: () => PageState }).state()) + const control = async (described: Described): Promise => { + try { + const evaluated = await cdp.send("Runtime.evaluate", { expression: `__ritmoKeyboardWalk.element(${described.id})`, objectGroup: "ritmo-keyboard" }) + const objectId = evaluated.result.objectId + if (objectId === undefined) return { role: "", name: "", selector: described.selector } + const tree = await cdp.send("Accessibility.getPartialAXTree", { objectId, fetchRelatives: false }) + const node = tree.nodes[0] + const role = node?.role?.value + const name = node?.name?.value + return { role: typeof role === "string" ? role : "", name: typeof name === "string" ? name.trim() : "", selector: described.selector } + } catch { + return { role: "", name: "", selector: described.selector } + } + } + return { page, errors, state, control, close: () => context.close() } +} + +/** Opens the screen: loads its URL, lets it settle, and runs its steps. A failed step is returned as its message. */ +export const reach = async (opened: Opened, url: string, screen: Screen, settle: number): Promise => { + const { page } = opened + try { + await page.goto(screenUrl(url, screen.path), { waitUntil: "load" }) + } catch (error) { + return `the page did not load: ${error instanceof Error ? error.message : String(error)}` + } + await page.waitForTimeout(settle) + for (const [index, step] of screen.steps.entries()) { + const at = `steps[${index}]` + try { + if ("press" in step) await page.keyboard.press(step.press) + else if ("type" in step) await page.keyboard.type(step.type) + else if ("wait" in step) await page.waitForTimeout(step.wait) + else { + const target = await find(opened, step.click) + if (target === undefined) return `${at}: no visible ${step.click.role} named ${JSON.stringify(step.click.name)} to click (names are the ones this report prints)` + await page.locator(target.selector).first().click({ timeout: 5000 }) + } + } catch (error) { + return `${at} failed: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}` + } + await page.waitForTimeout(300) + } + return undefined +} + +/** + * The first visible element with this role and accessible name, by the names + * this report prints (Chromium's accessibility tree), so a name copied from a + * report finds its control. Waits up to five seconds for it to show. + */ +export const find = async (opened: Opened, wanted: { readonly role: string; readonly name: string }): Promise => { + for (let tries = 0; tries < 10; tries++) { + const candidates = await call(opened.page, "controls", null) as ReadonlyArray + for (const candidate of candidates) { + const control = await opened.control(candidate) + if (control.role === wanted.role && control.name === wanted.name) return candidate + } + await opened.page.waitForTimeout(500) + } + return undefined +} + +export const call = (page: PlaywrightTypes.Page, name: string, argument: A): Promise => + page.evaluate( + ({ name, argument }) => (Reflect.get(globalThis, "__ritmoKeyboardWalk") as Record unknown>)[name]!(argument), + { name, argument } + ) diff --git a/packages/devtools/src/internal/route.ts b/packages/devtools/src/internal/route.ts new file mode 100644 index 00000000..295a234a --- /dev/null +++ b/packages/devtools/src/internal/route.ts @@ -0,0 +1,273 @@ +/** + * The route file `ritmo-devtools keyboard` and `ritmo-devtools layout` both + * read: the screens of a game's page, how to reach each one, and what each + * walk needs to know about them. Each walk ignores the fields that belong to + * the other (`noPress` is the keyboard walk's; `viewports`, `minTarget`, and + * `allow` are the layout audit's), so one route serves both. + * + * @internal + */ +import * as Result from "@typeonce/ritmo/Result" + +/** A control a route names: by accessible name, and by role when given. */ +export interface ControlName { + readonly name: string + readonly role?: string +} + +/** One thing a route does to reach a screen. */ +export type Step = + | { readonly press: string } + | { readonly type: string } + | { readonly click: { readonly role: string; readonly name: string } } + | { readonly wait: number } + +/** The layout audit's checks that a route may allow, by selector. */ +export type LayoutCheckId = + | "overlap" + | "cut-text" + | "off-screen" + | "safe-area" + | "small-target" + | "layout-shift" + | "truncated-text" + | "scroll-edge" + +export const layoutChecks: ReadonlyArray = ["overlap", "cut-text", "off-screen", "safe-area", "small-target", "layout-shift", "truncated-text", "scroll-edge"] + +const isLayoutCheck = (value: unknown): value is LayoutCheckId => typeof value === "string" && (layoutChecks as ReadonlyArray).includes(value) + +/** + * A finding the game means: the layout audit does not report a finding of + * `check` whose element (or the other element of an overlap) matches + * `selector` or is inside an element that does. + */ +export interface Allow { + readonly check: LayoutCheckId + readonly selector: string +} + +/** One screen of the walk: where it starts, how to reach it, what not to press, and what is meant. */ +export interface Screen { + readonly name: string + /** Appended to the URL: `/`, `?map=8`, `#settings`. */ + readonly path: string + readonly steps: ReadonlyArray + /** `true` presses nothing on this screen (no `focus-dropped`); a list skips those controls. */ + readonly noPress: true | ReadonlyArray + /** Layout findings the game means on this screen (with the route's own `allow`). */ + readonly allow: ReadonlyArray +} + +/** Space the device keeps for itself on each side (a notch, a home bar), in CSS pixels. */ +export interface Insets { + readonly top: number + readonly right: number + readonly bottom: number + readonly left: number +} + +/** A screen size the layout audit measures every screen at. */ +export interface Viewport { + readonly name: string + readonly width: number + readonly height: number + /** A touch screen (a phone or a tablet): `hasTouch` and a mobile viewport. */ + readonly touch: boolean + readonly insets: Insets +} + +export interface Route { + readonly screens: ReadonlyArray + /** The sizes the layout audit measures at (`defaultViewports` when the route names none). */ + readonly viewports: ReadonlyArray + /** The smallest tap area, in CSS pixels, either side (44 by default). */ + readonly minTarget: number + /** Layout findings the game means on every screen. */ + readonly allow: ReadonlyArray +} + +const noInsets: Insets = { top: 0, right: 0, bottom: 0, left: 0 } + +/** + * A small phone and a phone with a notch, held sideways and upright, and a + * desktop window. The notch insets are an iPhone's (the camera on one side + * when sideways, at the top when upright, and the home bar at the bottom). + */ +export const defaultViewports: ReadonlyArray = [ + { name: "small-phone-landscape", width: 667, height: 375, touch: true, insets: noInsets }, + { name: "phone-landscape", width: 844, height: 390, touch: true, insets: { top: 0, right: 47, bottom: 21, left: 47 } }, + { name: "phone-portrait", width: 390, height: 844, touch: true, insets: { top: 47, right: 0, bottom: 34, left: 0 } }, + { name: "desktop", width: 1280, height: 800, touch: false, insets: noInsets } +] + +/** The route of a game that writes none: the start page, as one screen. */ +export const startRoute: Route = { + screens: [{ name: "start", path: "", steps: [], noPress: [], allow: [] }], + viewports: defaultViewports, + minTarget: 44, + allow: [] +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) + +const onlyKeys = (value: Record, keys: ReadonlyArray, at: string): string | undefined => { + const extra = Object.keys(value).find((key) => !keys.includes(key)) + return extra === undefined ? undefined : `${at}.${extra} is not a field (expected ${keys.join(", ")})` +} + +const nonEmpty = (value: unknown): value is string => typeof value === "string" && value.trim() !== "" + +const wholeIn = (value: unknown, least: number, most: number): value is number => + typeof value === "number" && Number.isInteger(value) && value >= least && value <= most + +const parseControlName = (value: unknown, at: string): Result.Result => { + if (!isRecord(value)) return Result.failure(`${at} must be an object { "name": string, "role"?: string }`) + const extra = onlyKeys(value, ["name", "role"], at) + if (extra !== undefined) return Result.failure(extra) + if (!nonEmpty(value["name"])) return Result.failure(`${at}.name must be a non-empty string: the control's accessible name`) + if (value["role"] !== undefined && !nonEmpty(value["role"])) return Result.failure(`${at}.role must be a non-empty string when given: an ARIA role such as "button"`) + return Result.success(value["role"] === undefined ? { name: value["name"] } : { name: value["name"], role: value["role"] }) +} + +const stepKinds = `{ "press": "Escape" }, { "type": "Ada" }, { "click": { "role": "button", "name": "Play" } }, { "wait": 500 }` + +const parseStep = (value: unknown, at: string): Result.Result => { + if (!isRecord(value) || Object.keys(value).length !== 1) return Result.failure(`${at} must be one of ${stepKinds}`) + if ("press" in value) { + return nonEmpty(value["press"]) ? Result.success({ press: value["press"] }) : Result.failure(`${at}.press must be a key name, as Playwright's keyboard.press takes it ("Enter", "Escape", "a")`) + } + if ("type" in value) { + return typeof value["type"] === "string" && value["type"] !== "" ? Result.success({ type: value["type"] }) : Result.failure(`${at}.type must be the text to type into the focused field ("Ada")`) + } + if ("wait" in value) { + const wait = value["wait"] + return wholeIn(wait, 0, 60_000) ? Result.success({ wait }) : Result.failure(`${at}.wait must be a whole number of milliseconds from 0 to 60000`) + } + if ("click" in value) { + const click = value["click"] + if (!isRecord(click)) return Result.failure(`${at}.click must be an object { "role": string, "name": string }`) + const extra = onlyKeys(click, ["role", "name"], `${at}.click`) + if (extra !== undefined) return Result.failure(extra) + if (!nonEmpty(click["role"])) return Result.failure(`${at}.click.role must be a non-empty string: an ARIA role such as "button"`) + if (!nonEmpty(click["name"])) return Result.failure(`${at}.click.name must be a non-empty string: the control's accessible name`) + return Result.success({ click: { role: click["role"], name: click["name"] } }) + } + return Result.failure(`${at} must be one of ${stepKinds}; ${Object.keys(value)[0]} is not a step`) +} + +const parseAllow = (value: unknown, at: string): Result.Result, string> => { + const raw = value ?? [] + if (!Array.isArray(raw)) return Result.failure(`${at} must be an array of { "check", "selector" }`) + const allow: Array = [] + for (const [index, entry] of raw.entries()) { + const where = `${at}[${index}]` + if (!isRecord(entry)) return Result.failure(`${where} must be an object { "check": string, "selector": string }`) + const extra = onlyKeys(entry, ["check", "selector"], where) + if (extra !== undefined) return Result.failure(extra) + if (!isLayoutCheck(entry["check"])) return Result.failure(`${where}.check must be one of ${layoutChecks.join(", ")}`) + if (!nonEmpty(entry["selector"])) return Result.failure(`${where}.selector must be a CSS selector of the element the finding names, such as ".card .badge"`) + allow.push({ check: entry["check"], selector: entry["selector"] }) + } + return Result.success(allow) +} + +const parseScreen = (value: unknown, at: string): Result.Result => { + if (!isRecord(value)) return Result.failure(`${at} must be an object { "name", "path"?, "steps"?, "noPress"?, "allow"? }`) + const extra = onlyKeys(value, ["name", "path", "steps", "noPress", "allow"], at) + if (extra !== undefined) return Result.failure(extra) + if (!nonEmpty(value["name"])) return Result.failure(`${at}.name must be a non-empty string`) + const path = value["path"] ?? "" + if (typeof path !== "string") return Result.failure(`${at}.path must be a string appended to the URL ("/", "?map=8")`) + const rawSteps = value["steps"] ?? [] + if (!Array.isArray(rawSteps)) return Result.failure(`${at}.steps must be an array`) + const steps: Array = [] + for (const [index, raw] of rawSteps.entries()) { + const step = parseStep(raw, `${at}.steps[${index}]`) + if (!step.ok) return step + steps.push(step.value) + } + const allow = parseAllow(value["allow"], `${at}.allow`) + if (!allow.ok) return allow + const rawNoPress = value["noPress"] ?? [] + if (rawNoPress === true) return Result.success({ name: value["name"], path, steps, noPress: true, allow: allow.value }) + if (!Array.isArray(rawNoPress)) return Result.failure(`${at}.noPress must be true (press nothing on this screen) or an array of { "name", "role"? }`) + const noPress: Array = [] + for (const [index, raw] of rawNoPress.entries()) { + const control = parseControlName(raw, `${at}.noPress[${index}]`) + if (!control.ok) return control + noPress.push(control.value) + } + return Result.success({ name: value["name"], path, steps, noPress, allow: allow.value }) +} + +const parseViewport = (value: unknown, at: string): Result.Result => { + if (!isRecord(value)) return Result.failure(`${at} must be an object { "name", "width", "height", "touch"?, "insets"? }`) + const extra = onlyKeys(value, ["name", "width", "height", "touch", "insets"], at) + if (extra !== undefined) return Result.failure(extra) + if (!nonEmpty(value["name"])) return Result.failure(`${at}.name must be a non-empty string`) + if (!wholeIn(value["width"], 200, 4000)) return Result.failure(`${at}.width must be a whole number of CSS pixels from 200 to 4000`) + if (!wholeIn(value["height"], 200, 4000)) return Result.failure(`${at}.height must be a whole number of CSS pixels from 200 to 4000`) + const touch = value["touch"] ?? false + if (typeof touch !== "boolean") return Result.failure(`${at}.touch must be true (a phone or tablet) or false`) + const rawInsets = value["insets"] ?? noInsets + if (!isRecord(rawInsets)) return Result.failure(`${at}.insets must be an object { "top", "right", "bottom", "left" } of CSS pixels`) + const insetsExtra = onlyKeys(rawInsets, ["top", "right", "bottom", "left"], `${at}.insets`) + if (insetsExtra !== undefined) return Result.failure(insetsExtra) + const sides = ["top", "right", "bottom", "left"] as const + for (const side of sides) { + if (!wholeIn(rawInsets[side] ?? 0, 0, 200)) return Result.failure(`${at}.insets.${side} must be a whole number of CSS pixels from 0 to 200`) + } + const inset = (side: (typeof sides)[number]): number => { + const given = rawInsets[side] + return typeof given === "number" ? given : 0 + } + return Result.success({ + name: value["name"], + width: value["width"], + height: value["height"], + touch, + insets: { top: inset("top"), right: inset("right"), bottom: inset("bottom"), left: inset("left") } + }) +} + +/** Validates a route file's JSON; the error names the field that is wrong. */ +export const parseRoute = (value: unknown): Result.Result => { + if (!isRecord(value)) return Result.failure(`the route must be an object { "screens": [...] }`) + const extra = onlyKeys(value, ["screens", "viewports", "minTarget", "allow"], "route") + if (extra !== undefined) return Result.failure(extra) + const raw = value["screens"] + if (!Array.isArray(raw) || raw.length === 0) return Result.failure(`route.screens must be a non-empty array`) + const screens: Array = [] + for (const [index, entry] of raw.entries()) { + const screen = parseScreen(entry, `route.screens[${index}]`) + if (!screen.ok) return screen + if (screens.some((other) => other.name === screen.value.name)) return Result.failure(`route.screens[${index}].name "${screen.value.name}" is already the name of another screen`) + screens.push(screen.value) + } + let viewports = defaultViewports + if (value["viewports"] !== undefined) { + const rawViewports = value["viewports"] + if (!Array.isArray(rawViewports) || rawViewports.length === 0) return Result.failure(`route.viewports must be a non-empty array of { "name", "width", "height", "touch"?, "insets"? }`) + const parsed: Array = [] + for (const [index, entry] of rawViewports.entries()) { + const viewport = parseViewport(entry, `route.viewports[${index}]`) + if (!viewport.ok) return viewport + if (parsed.some((other) => other.name === viewport.value.name)) return Result.failure(`route.viewports[${index}].name "${viewport.value.name}" is already the name of another viewport`) + parsed.push(viewport.value) + } + viewports = parsed + } + const minTarget = value["minTarget"] ?? 44 + if (!wholeIn(minTarget, 1, 200)) return Result.failure(`route.minTarget must be a whole number of CSS pixels from 1 to 200 (the smallest tap area either side)`) + const allow = parseAllow(value["allow"], "route.allow") + if (!allow.ok) return allow + return Result.success({ screens, viewports, minTarget, allow: allow.value }) +} + +/** Joins a screen's path to the base URL, keeping the base's own path. */ +export const screenUrl = (base: string, path: string): string => { + if (path === "") return base + if (path.startsWith("?") || path.startsWith("#")) return base.replace(/[?#].*$/, "") + path + return new URL(path, base).href +} diff --git a/packages/devtools/test/KeyboardWalk.test.ts b/packages/devtools/test/KeyboardWalk.test.ts index 7d59ef67..bd9c2407 100644 --- a/packages/devtools/test/KeyboardWalk.test.ts +++ b/packages/devtools/test/KeyboardWalk.test.ts @@ -7,9 +7,11 @@ import { fileURLToPath } from "node:url" import { afterAll, beforeAll, describe, expect, it } from "vitest" import type * as KeyboardWalkModule from "../src/internal/keyboardWalk.ts" +import type * as RouteModule from "../src/internal/route.ts" // The walk is internal to the CLI (the package exports no `internal/*`): load its source to call it with a short settle time. const KeyboardWalk = await import(new URL("../src/internal/keyboardWalk.ts", import.meta.url).href) as typeof KeyboardWalkModule +const Route = await import(new URL("../src/internal/route.ts", import.meta.url).href) as typeof RouteModule const cli = fileURLToPath(new URL("../src/cli.ts", import.meta.url)) const fixtures = fileURLToPath(new URL("./fixtures/keyboard/", import.meta.url)) @@ -21,47 +23,6 @@ const run = (args: ReadonlyArray, cwd: string) => }) }) -describe("keyboard route", () => { - it("fills in a screen's defaults", () => { - expect(KeyboardWalk.parseRoute({ screens: [{ name: "title" }] })).toEqual({ - ok: true, - value: { screens: [{ name: "title", path: "", steps: [], noPress: [] }] } - }) - }) - - it("keeps every kind of step and both kinds of noPress", () => { - const parsed = KeyboardWalk.parseRoute({ - screens: [ - { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, - { name: "end", noPress: true } - ] - }) - expect(parsed.ok && parsed.value.screens).toEqual([ - { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, - { name: "end", path: "", steps: [], noPress: true } - ]) - }) - - it("names the field that is wrong", () => { - const error = (value: unknown) => { - const parsed = KeyboardWalk.parseRoute(value) - return parsed.ok ? undefined : parsed.error - } - expect(error([])).toBe("the route must be an object { \"screens\": [...] }") - expect(error({ screens: [] })).toBe("route.screens must be a non-empty array") - expect(error({ screens: [{ name: "a" }], extra: 1 })).toBe("route.extra is not a field (expected screens)") - expect(error({ screens: [{ name: "" }] })).toBe("route.screens[0].name must be a non-empty string") - expect(error({ screens: [{ name: "a" }, { name: "a" }] })).toBe("route.screens[1].name \"a\" is already the name of another screen") - expect(error({ screens: [{ name: "a", url: "/" }] })).toBe("route.screens[0].url is not a field (expected name, path, steps, noPress)") - expect(error({ screens: [{ name: "a", steps: [{ click: { role: "button" } }] }] })).toBe("route.screens[0].steps[0].click.name must be a non-empty string: the control's accessible name") - expect(error({ screens: [{ name: "a", steps: [{ wait: 1.5 }] }] })).toBe("route.screens[0].steps[0].wait must be a whole number of milliseconds from 0 to 60000") - expect(error({ screens: [{ name: "a", steps: [{ press: "a", wait: 1 }] }] })).toMatch(/^route\.screens\[0\]\.steps\[0\] must be one of/) - expect(error({ screens: [{ name: "a", steps: [{ type: "abc" }] }] })).toMatch(/type is not a step$/) - expect(error({ screens: [{ name: "a", noPress: [{ role: "button" }] }] })).toBe("route.screens[0].noPress[0].name must be a non-empty string: the control's accessible name") - expect(error({ screens: [{ name: "a", noPress: false }] })).toBe("route.screens[0].noPress must be true (press nothing on this screen) or an array of { \"name\", \"role\"? }") - }) -}) - describe("ritmo-devtools keyboard arguments", () => { it("refuses a bad route or URL with exit code 2", async () => { const dir = mkdtempSync(join(tmpdir(), "ritmo-keyboard-")) @@ -114,8 +75,8 @@ describe.skipIf(skipped !== undefined)("ritmo-devtools keyboard", () => { await new Promise((done) => server?.close(done)) }) - const walk = (screens: ReadonlyArray & { readonly name: string }>) => - KeyboardWalk.walk(browser, { url: base, route: { screens: screens.map((screen) => ({ path: "", steps: [], noPress: [], ...screen })) }, settle: 100 }) + const walk = (screens: ReadonlyArray & { readonly name: string }>) => + KeyboardWalk.walk(browser, { url: base, route: { ...Route.startRoute, screens: screens.map((screen) => ({ path: "", steps: [], noPress: [], allow: [], ...screen })) }, settle: 100 }) const statuses = (report: KeyboardWalkModule.Report, screen: string) => Object.fromEntries(report.outcomes.filter((outcome) => outcome.screen === screen).map((outcome) => [outcome.check, outcome.status])) diff --git a/packages/devtools/test/Route.test.ts b/packages/devtools/test/Route.test.ts new file mode 100644 index 00000000..2bfe64e3 --- /dev/null +++ b/packages/devtools/test/Route.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest" + +import type * as RouteModule from "../src/internal/route.ts" + +// The route is internal to the CLI (the package exports no `internal/*`): load its source. +const Route = await import(new URL("../src/internal/route.ts", import.meta.url).href) as typeof RouteModule + +const error = (value: unknown) => { + const parsed = Route.parseRoute(value) + return parsed.ok ? undefined : parsed.error +} + +describe("route", () => { + it("fills in a screen's and the route's defaults", () => { + expect(Route.parseRoute({ screens: [{ name: "title" }] })).toEqual({ + ok: true, + value: { screens: [{ name: "title", path: "", steps: [], noPress: [], allow: [] }], viewports: Route.defaultViewports, minTarget: 44, allow: [] } + }) + }) + + it("keeps every kind of step and both kinds of noPress", () => { + const parsed = Route.parseRoute({ + screens: [ + { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { type: "Ada" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }] }, + { name: "end", noPress: true } + ] + }) + expect(parsed.ok && parsed.value.screens).toEqual([ + { name: "play", path: "?map=8", steps: [{ press: "Escape" }, { type: "Ada" }, { click: { role: "button", name: "Play" } }, { wait: 500 }], noPress: [{ name: "Start over", role: "button" }], allow: [] }, + { name: "end", path: "", steps: [], noPress: true, allow: [] } + ]) + }) + + it("keeps the layout audit's fields", () => { + const parsed = Route.parseRoute({ + screens: [{ name: "home", allow: [{ check: "overlap", selector: ".card .badge" }] }], + viewports: [{ name: "phone", width: 844, height: 390, touch: true, insets: { left: 47, right: 47 } }, { name: "desktop", width: 1280, height: 800 }], + minTarget: 42, + allow: [{ check: "truncated-text", selector: ".name" }] + }) + expect(parsed).toEqual({ + ok: true, + value: { + screens: [{ name: "home", path: "", steps: [], noPress: [], allow: [{ check: "overlap", selector: ".card .badge" }] }], + viewports: [ + { name: "phone", width: 844, height: 390, touch: true, insets: { top: 0, right: 47, bottom: 0, left: 47 } }, + { name: "desktop", width: 1280, height: 800, touch: false, insets: { top: 0, right: 0, bottom: 0, left: 0 } } + ], + minTarget: 42, + allow: [{ check: "truncated-text", selector: ".name" }] + } + }) + }) + + it("names the field that is wrong", () => { + expect(error([])).toBe("the route must be an object { \"screens\": [...] }") + expect(error({ screens: [] })).toBe("route.screens must be a non-empty array") + expect(error({ screens: [{ name: "a" }], extra: 1 })).toBe("route.extra is not a field (expected screens, viewports, minTarget, allow)") + expect(error({ screens: [{ name: "" }] })).toBe("route.screens[0].name must be a non-empty string") + expect(error({ screens: [{ name: "a" }, { name: "a" }] })).toBe("route.screens[1].name \"a\" is already the name of another screen") + expect(error({ screens: [{ name: "a", url: "/" }] })).toBe("route.screens[0].url is not a field (expected name, path, steps, noPress, allow)") + expect(error({ screens: [{ name: "a", steps: [{ click: { role: "button" } }] }] })).toBe("route.screens[0].steps[0].click.name must be a non-empty string: the control's accessible name") + expect(error({ screens: [{ name: "a", steps: [{ wait: 1.5 }] }] })).toBe("route.screens[0].steps[0].wait must be a whole number of milliseconds from 0 to 60000") + expect(error({ screens: [{ name: "a", steps: [{ press: "a", wait: 1 }] }] })).toMatch(/^route\.screens\[0\]\.steps\[0\] must be one of/) + expect(error({ screens: [{ name: "a", steps: [{ drag: "abc" }] }] })).toMatch(/drag is not a step$/) + expect(error({ screens: [{ name: "a", steps: [{ type: "" }] }] })).toBe("route.screens[0].steps[0].type must be the text to type into the focused field (\"Ada\")") + expect(error({ screens: [{ name: "a", noPress: [{ role: "button" }] }] })).toBe("route.screens[0].noPress[0].name must be a non-empty string: the control's accessible name") + expect(error({ screens: [{ name: "a", noPress: false }] })).toBe("route.screens[0].noPress must be true (press nothing on this screen) or an array of { \"name\", \"role\"? }") + expect(error({ screens: [{ name: "a", allow: [{ check: "overlaps", selector: ".a" }] }] })).toBe(`route.screens[0].allow[0].check must be one of ${Route.layoutChecks.join(", ")}`) + expect(error({ screens: [{ name: "a", allow: [{ check: "overlap" }] }] })).toBe("route.screens[0].allow[0].selector must be a CSS selector of the element the finding names, such as \".card .badge\"") + expect(error({ screens: [{ name: "a" }], viewports: [] })).toBe("route.viewports must be a non-empty array of { \"name\", \"width\", \"height\", \"touch\"?, \"insets\"? }") + expect(error({ screens: [{ name: "a" }], viewports: [{ name: "p", width: 100, height: 400 }] })).toBe("route.viewports[0].width must be a whole number of CSS pixels from 200 to 4000") + expect(error({ screens: [{ name: "a" }], viewports: [{ name: "p", width: 400, height: 400, insets: { left: -1 } }] })).toBe("route.viewports[0].insets.left must be a whole number of CSS pixels from 0 to 200") + expect(error({ screens: [{ name: "a" }], viewports: [{ name: "p", width: 400, height: 400 }, { name: "p", width: 500, height: 400 }] })).toBe("route.viewports[1].name \"p\" is already the name of another viewport") + expect(error({ screens: [{ name: "a" }], minTarget: 0 })).toBe("route.minTarget must be a whole number of CSS pixels from 1 to 200 (the smallest tap area either side)") + }) +}) From e9858c4c019bbebb6c4be6d3a3442dfa7f92e618 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 20:04:38 +0000 Subject: [PATCH 2/6] browser: Touch lets ritmo-devtools layout read its layout The Touch buttons are drawn on the canvas, so a page audit cannot see them. While attached, the target carries a function returning the layout at its current size (under a Symbol.for key), and dispose() removes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- packages/browser/src/Touch.ts | 11 +++++++++++ packages/browser/test/Touch.test.ts | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/packages/browser/src/Touch.ts b/packages/browser/src/Touch.ts index 00e090cd..d8618e3a 100644 --- a/packages/browser/src/Touch.ts +++ b/packages/browser/src/Touch.ts @@ -43,6 +43,12 @@ * press opens is cancelled while a tracked finger is down. `dispose()` * restores the styles. * + * While attached, the target also carries a function that returns the + * layout at its current size, under `Symbol.for("@typeonce/ritmo-browser/Touch.layout")`: + * `ritmo-devtools layout` reads it to check the buttons the canvas does not + * show to the page (too small to tap, under the notch, under a panel). + * `dispose()` removes it. + * * @module Touch * @docGroup browser * @@ -77,6 +83,9 @@ import { anyUsed } from "./internal/used.ts" import type { ActionState } from "./internal/state.ts" import type { Vector } from "./internal/vectors.ts" +/** Where `ritmo-devtools layout` reads the layout of an attached target (see the module header). */ +const layoutKey = Symbol.for("@typeonce/ritmo-browser/Touch.layout") + /** A position in CSS pixels relative to the element's top-left corner. */ export interface Position { readonly x: number @@ -551,6 +560,7 @@ export const actions = [property, style.getPropertyValue(property)] as const) for (const property of noBrowserGestures) style?.setProperty(property, "none") + Reflect.set(target, layoutKey, () => layout(sizeOf())) target.addEventListener("pointerdown", onDown) target.addEventListener("pointermove", onMove) target.addEventListener("pointerup", onUp) @@ -610,6 +620,7 @@ export const actions = { expect(withMouse.snapshot().actions.attack.held).toBe(true) }) + it("lets ritmo-devtools layout read the layout while attached", () => { + const fake = makeTarget() + const key = Symbol.for("@typeonce/ritmo-browser/Touch.layout") + const touch = Touch.actions(fake.target, fake.blurHost, layout) + const read = Reflect.get(fake.target, key) + expect(typeof read === "function" ? read() : undefined).toEqual(layout(size)) + touch.dispose() + expect(Reflect.has(fake.target, key)).toBe(false) + }) + it("cancels the long-press context menu only while a tracked finger is down", () => { const fake = makeTarget() Touch.actions(fake.target, fake.blurHost, layout) From 4ab656ff95cfe32d182714ac1ed8efea2ee917e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 20:13:28 +0000 Subject: [PATCH 3/6] devtools: ritmo-devtools layout audits a game's page at each screen size Opens each screen of the route in Chromium at each viewport (phones held both ways, with an iPhone's notch and home bar, and a desktop by default) and reports what is broken in the layout the player sees. Errors fail the run: overlap, cut-text, off-screen, safe-area, small-target, layout-shift, never-settled. Warnings never fail, because a game may mean them: truncated-text (an ellipsis or clamp), scroll-edge, and what breaks only with every text 40% longer. A route's allow leaves out what the game means. The rules come from a prototype run on spirit-call: a control that draws nothing (a touch area over the stage) is not an overlap, something behind a scrim is not under the scrim's text, a box that spans the screen cuts at the screen's edge, items in a scrolling area are judged by it, and a tap area is measured by hit tests so a ::after extension counts. Touch buttons are read from the target Touch.actions marks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- packages/devtools/src/cli.ts | 67 ++- packages/devtools/src/internal/layoutPage.ts | 512 ++++++++++++++++++ packages/devtools/src/internal/layoutWalk.ts | 348 ++++++++++++ packages/devtools/test/LayoutWalk.test.ts | 204 +++++++ .../devtools/test/fixtures/layout/clean.html | 31 ++ .../test/fixtures/layout/cut-text.html | 8 + .../test/fixtures/layout/layout-shift.html | 9 + .../test/fixtures/layout/longer-text.html | 8 + .../test/fixtures/layout/never-settled.html | 10 + .../test/fixtures/layout/off-screen.html | 8 + .../test/fixtures/layout/overlap.html | 14 + .../test/fixtures/layout/safe-area.html | 10 + .../test/fixtures/layout/scroll-edge.html | 9 + .../test/fixtures/layout/small-target.html | 8 + .../devtools/test/fixtures/layout/touch.html | 18 + .../test/fixtures/layout/truncated-text.html | 8 + 16 files changed, 1257 insertions(+), 15 deletions(-) create mode 100644 packages/devtools/src/internal/layoutPage.ts create mode 100644 packages/devtools/src/internal/layoutWalk.ts create mode 100644 packages/devtools/test/LayoutWalk.test.ts create mode 100644 packages/devtools/test/fixtures/layout/clean.html create mode 100644 packages/devtools/test/fixtures/layout/cut-text.html create mode 100644 packages/devtools/test/fixtures/layout/layout-shift.html create mode 100644 packages/devtools/test/fixtures/layout/longer-text.html create mode 100644 packages/devtools/test/fixtures/layout/never-settled.html create mode 100644 packages/devtools/test/fixtures/layout/off-screen.html create mode 100644 packages/devtools/test/fixtures/layout/overlap.html create mode 100644 packages/devtools/test/fixtures/layout/safe-area.html create mode 100644 packages/devtools/test/fixtures/layout/scroll-edge.html create mode 100644 packages/devtools/test/fixtures/layout/small-target.html create mode 100644 packages/devtools/test/fixtures/layout/touch.html create mode 100644 packages/devtools/test/fixtures/layout/truncated-text.html diff --git a/packages/devtools/src/cli.ts b/packages/devtools/src/cli.ts index cce2fa28..045323e1 100755 --- a/packages/devtools/src/cli.ts +++ b/packages/devtools/src/cli.ts @@ -6,6 +6,7 @@ * ritmo-devtools verify [--engines node,chromium,webkit,firefox] [--json] * ritmo-devtools scan [--json] * ritmo-devtools keyboard [--url ] [--route ] [--json] + * ritmo-devtools layout [--url ] [--route ] [--screenshots ] [--json] * * `lint` imports `` (a module exporting `lintTarget`, see the `Lint` module), * prints every lint of its schedules with its fix, and exits with 1 when one @@ -66,11 +67,22 @@ * Chromium, and `vite` when there is no `--url`; a missing one is printed * with its install command. * + * `layout` opens each screen of the same route in Chromium at each of the + * route's viewports (phones held both ways and a desktop by default) and + * reports what is broken in the layout: things drawn over each other, text + * cut by a box, anything past the screen's edge or in the device's own + * space, taps that land on too small an area, and layout that moves while + * nothing happens. Those fail. Text shortened with an ellipsis, controls half + * hidden at a scroll edge, and what breaks with every text 40% longer are + * warnings, which never fail. `--screenshots ` saves each screen with + * its findings boxed and numbered. It needs what `keyboard` needs. + * * Exit codes: 0 ok (and every `scan` that ran), 1 lint warning, uncovered * module, mismatched versions, an engine that differs or did not run, or a - * keyboard failure, 2 usage error (for `scan` also an unreadable entry, an - * import that does not resolve, or a missing tool; for `keyboard` an invalid - * route, a missing tool, or a dev server that did not start). + * keyboard failure, or a layout error, 2 usage error (for `scan` also an + * unreadable entry, an import that does not resolve, or a missing tool; for + * `keyboard` and `layout` an invalid route, a missing tool, or a dev server + * that did not start). * Requires Node 22.15 or later for `module.registerHooks`; on older Node it * prints that coverage was not checked. * @@ -83,6 +95,7 @@ import { pathToFileURL } from "node:url" import * as DependencyScan from "./internal/dependencyScan.ts" import * as KeyboardWalk from "./internal/keyboardWalk.ts" +import * as LayoutWalk from "./internal/layoutWalk.ts" import * as PackageVersions from "./internal/packageVersions.ts" import * as Route from "./internal/route.ts" import * as SimulationCoverage from "./internal/simulationCoverage.ts" @@ -96,10 +109,12 @@ const usage = [ " ritmo-devtools verify [--engines node,chromium,webkit,firefox] [--json]", " ritmo-devtools scan [--json]", " ritmo-devtools keyboard [--url ] [--route ] [--json]", + " ritmo-devtools layout [--url ] [--route ] [--screenshots ] [--json]", " lint: exports lintTarget: () => { runtime, schedules, describe?, hostEvents? } (see the Lint module)", " verify: exports replayTarget: (playback) => { runtime, schedules, hostClock } (see the Verify module)", " scan: is the simulation's entry module; every file it imports, node_modules included, is checked for Math, clock, and randomness calls", - " keyboard: plays the page at --url (or the working directory served with its Vite config) by keyboard in Chromium; --route is a JSON file of screens" + " keyboard: plays the page at --url (or the working directory served with its Vite config) by keyboard in Chromium; --route is a JSON file of screens", + " layout: measures each screen of the route at each of its viewports in Chromium; --screenshots writes each screen with its findings boxed" ].join("\n") const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null @@ -120,6 +135,7 @@ const main = async (args: ReadonlyArray): Promise => { if (command === "verify") return verify(rest) if (command === "scan") return scan(rest) if (command === "keyboard") return keyboard(rest) + if (command === "layout") return layout(rest) console.error(usage) return 2 } @@ -250,9 +266,22 @@ const flag = (args: ReadonlyArray, index: number, name: string): { reado return undefined } -const keyboard = async (args: ReadonlyArray): Promise => { +/** The walks over a game's page: which one, and the flags only it takes. */ +type PageCommand = "keyboard" | "layout" + +const keyboard = (args: ReadonlyArray): Promise => pageWalk("keyboard", args) + +const layout = (args: ReadonlyArray): Promise => pageWalk("layout", args) + +/** + * Reads `--url`, `--route`, `--json` (and `layout`'s `--screenshots`), + * launches Chromium, serves the working directory with the game's Vite + * config when there is no `--url`, and runs the walk. + */ +const pageWalk = async (command: PageCommand, args: ReadonlyArray): Promise => { let url: string | undefined let routeFile: string | undefined + let screenshots: string | undefined let json = false for (let index = 0; index < args.length;) { if (args[index] === "--json") { @@ -260,13 +289,16 @@ const keyboard = async (args: ReadonlyArray): Promise => { index++ continue } - const given = flag(args, index, "--url") ?? flag(args, index, "--route") - if (given === undefined || given.value === undefined || given.value === "") { + const named = ["--url", "--route", ...(command === "layout" ? ["--screenshots"] : [])] + const name = named.find((candidate) => flag(args, index, candidate) !== undefined) + const given = name === undefined ? undefined : flag(args, index, name) + if (name === undefined || given === undefined || given.value === undefined || given.value === "") { console.error(usage) return 2 } - if (args[index]!.startsWith("--url")) url = given.value - else routeFile = given.value + if (name === "--url") url = given.value + else if (name === "--route") routeFile = given.value + else screenshots = resolve(given.value) index += given.taken } if (url !== undefined && !/^https?:\/\//.test(url)) { @@ -284,7 +316,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { } const parsed = Route.parseRoute(data) if (!parsed.ok) { - console.error(`${routeFile} is not a keyboard route: ${parsed.error}`) + console.error(`${routeFile} is not a ${command} route: ${parsed.error}`) return 2 } route = parsed.value @@ -293,7 +325,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { try { playwright = await import("playwright") } catch { - console.error("keyboard needs playwright, which is not installed: npm install --save-dev playwright && npx playwright install chromium") + console.error(`${command} needs playwright, which is not installed: npm install --save-dev playwright && npx playwright install chromium`) return 2 } let browser: Awaited> @@ -302,7 +334,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { } catch (error) { const message = error instanceof Error ? error.message : String(error) console.error(/Executable doesn't exist|playwright install/i.test(message) - ? "keyboard needs Chromium, which is not downloaded: npx playwright install chromium" + ? `${command} needs Chromium, which is not downloaded: npx playwright install chromium` : `Chromium did not launch: ${message.split("\n")[0]}`) return 2 } @@ -313,7 +345,7 @@ const keyboard = async (args: ReadonlyArray): Promise => { try { vite = await import("vite") } catch { - console.error("keyboard serves the game with its own Vite config and needs vite, which is not installed: npm install --save-dev vite (or pass --url of a running dev server)") + console.error(`${command} serves the game with its own Vite config and needs vite, which is not installed: npm install --save-dev vite (or pass --url of a running dev server)`) return 2 } try { @@ -336,8 +368,13 @@ const keyboard = async (args: ReadonlyArray): Promise => { return 2 } } - const report = await KeyboardWalk.walk(browser, { url, route }) - console.log(json ? JSON.stringify(report, null, 2) : KeyboardWalk.format(report)) + if (command === "keyboard") { + const report = await KeyboardWalk.walk(browser, { url, route }) + console.log(json ? JSON.stringify(report, null, 2) : KeyboardWalk.format(report)) + return report.ok ? 0 : 1 + } + const report = await LayoutWalk.walk(browser, screenshots === undefined ? { url, route } : { url, route, screenshots }) + console.log(json ? JSON.stringify(report, null, 2) : LayoutWalk.format(report)) return report.ok ? 0 : 1 } finally { await browser.close().catch(() => undefined) diff --git a/packages/devtools/src/internal/layoutPage.ts b/packages/devtools/src/internal/layoutPage.ts new file mode 100644 index 00000000..b0639849 --- /dev/null +++ b/packages/devtools/src/internal/layoutPage.ts @@ -0,0 +1,512 @@ +/** + * The page side of `ritmo-devtools layout`: a script installed before the + * game's own scripts on every page the audit opens. It runs in the browser, + * so it is plain JavaScript in a string: Node never runs it, and the package + * config has no DOM types. + * + * It installs `__ritmoLayout` with: + * + * - `snapshot()`: every measured element's layout box, ignoring transforms, + * for the walk to tell when the screen holds still (a transform animation + * is motion, not layout, so a pulsing button still settles); + * - `audit(options)`: the findings of each check over what the page shows + * now (the open modal only, when there is one), and how many items it + * measured; + * - `steady()` and `shifts(allow)`: Chromium's own layout-shift entries + * recorded since `steady()`, with the elements that moved; + * - `pad()`: makes every text on the page about 40% longer, and keeps it so + * as the game renders new text, for the longer-text pass; + * - `mark(boxes)` and `unmark()`: numbered boxes over the findings, in the + * top layer, for a screenshot. + * + * What it measures: each line of visible text (`Range.getClientRects`, the + * text itself rather than the box around it), each visible control (the + * elements the keyboard walk calls controls), and each button of a + * `Touch.actions` layout (read from the element it is attached to). Boxes + * come in viewport pixels after every transform, so a UI scaled as one piece + * is measured as it is shown. + * + * It changes nothing on the game's page except in `pad()` (a pass of its + * own, on its own page load), and while `audit` runs, when a style that + * makes every element answer hit tests lets it see what is drawn on top + * (removed before it returns). + */ +export const pageScript = String.raw`(() => { + if (globalThis.__ritmoLayout) return + const touchKey = Symbol.for("@typeonce/ritmo-browser/Touch.layout") + const controlSelector = [ + "button", "a[href]", "input:not([type=hidden])", "select", "textarea", "summary", + '[role="button"]', '[role="link"]', '[role="tab"]', '[role="switch"]', '[role="checkbox"]', '[role="radio"]', + '[role="menuitem"]', '[role="menuitemcheckbox"]', '[role="menuitemradio"]', '[role="option"]', '[role="treeitem"]', + '[role="slider"]', '[role="spinbutton"]', '[role="combobox"]', '[role="textbox"]' + ].join(", ") + const marksId = "__ritmo_layout_marks" + + // Chromium's layout-shift entries, from the first paint on. + const shiftLog = [] + try { + new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + if (entry.hadRecentInput) continue + shiftLog.push({ time: entry.startTime, value: entry.value, nodes: (entry.sources || []).map((source) => source.node).filter((node) => node) }) + } + }).observe({ type: "layout-shift", buffered: true }) + } catch (error) {} + let steadyFrom = Infinity + + const shown = (element) => typeof element.checkVisibility === "function" && element.checkVisibility({ visibilityProperty: true, opacityProperty: true }) === true + const transparent = (color) => color === "transparent" || /rgba\([^)]*,\s*0\)$/.test(color) || /\/\s*0\)$/.test(color) + + const selector = (element) => { + const parts = [] + let at = element + while (at !== null && at.nodeType === 1 && at !== document.documentElement) { + if (at.id !== "" && document.querySelectorAll("#" + CSS.escape(at.id)).length === 1) { + parts.unshift("#" + CSS.escape(at.id)) + break + } + let part = at.localName + const classes = [...at.classList].filter((name) => !name.startsWith("__")).slice(0, 2) + if (classes.length > 0) part += classes.map((name) => "." + CSS.escape(name)).join("") + const parent = at.parentElement + if (parent !== null) { + const same = [...parent.children].filter((child) => child.localName === at.localName && classes.every((name) => child.classList.contains(name))) + if (same.length > 1) part += ":nth-of-type(" + ([...parent.children].filter((child) => child.localName === at.localName).indexOf(at) + 1) + ")" + } + parts.unshift(part) + if (parts.length >= 4) break + at = parent + } + return parts.join(" > ") + } + + const box = (rect) => ({ l: rect.left, t: rect.top, r: rect.right, b: rect.bottom }) + const meet = (a, b) => ({ l: Math.max(a.l, b.l), t: Math.max(a.t, b.t), r: Math.min(a.r, b.r), b: Math.min(a.b, b.b) }) + const width = (a) => Math.max(0, a.r - a.l) + const height = (a) => Math.max(0, a.b - a.t) + const area = (a) => width(a) * height(a) + const contains = (outer, inner) => outer.l <= inner.l + 1 && outer.t <= inner.t + 1 && outer.r >= inner.r - 1 && outer.b >= inner.b - 1 + const round = (a) => ({ l: Math.round(a.l), t: Math.round(a.t), r: Math.round(a.r), b: Math.round(a.b) }) + + const clipsOverflow = (style) => style.overflowX !== "visible" || style.overflowY !== "visible" + const scrolls = (style) => /auto|scroll/.test(style.overflowX) || /auto|scroll/.test(style.overflowY) + const makesContainingBlock = (style, fixed) => + (!fixed && style.position !== "static") || style.transform !== "none" || style.perspective !== "none" || + style.filter !== "none" || /paint|layout|strict|content/.test(style.contain) || /transform/.test(style.willChange) + /** The ancestor whose box a positioned element is placed in (null: the viewport). */ + const containingBlock = (element, fixed) => { + for (let at = element.parentElement; at !== null && at !== document.documentElement; at = at.parentElement) { + if (makesContainingBlock(getComputedStyle(at), fixed)) return at + } + return null + } + /** + * The ancestors that clip what an element draws (its text, with self), in + * order: overflow clips a descendant only when the descendant's containing + * block is inside the clipping element, so an absolutely placed badge + * escapes a parent that hides overflow, as it does on screen. + */ + const clippers = (element, self) => { + const out = [] + let escapeTo + let at = element + while (at !== null && at !== document.documentElement) { + if (escapeTo !== undefined && at === escapeTo) escapeTo = undefined + const style = getComputedStyle(at) + // The body's overflow belongs to the viewport when the root's is visible: it clips nothing of its own. + const propagated = at === document.body && getComputedStyle(document.documentElement).overflow === "visible" + if (escapeTo === undefined && (at !== element || self) && !propagated && clipsOverflow(style)) { + out.push({ element: at, box: box(at.getBoundingClientRect()), scroll: scrolls(style) }) + } + if (style.position === "fixed" || style.position === "absolute") { + const block = containingBlock(at, style.position === "fixed") + if (block === null) break + if (escapeTo === undefined) escapeTo = block + } + at = at.parentElement + } + return out + } + const nearestScroller = (clips) => clips.find((clip) => clip.scroll) + + const painted = (element) => { + const style = getComputedStyle(element) + if (/^(img|svg|canvas|video|input|select|textarea|progress|meter)$/.test(element.localName)) return true + if (!transparent(style.backgroundColor) || style.backgroundImage !== "none" || style.boxShadow !== "none") return true + if (style.backdropFilter !== undefined && style.backdropFilter !== "none") return true + const borders = ["Top", "Right", "Bottom", "Left"].some((side) => parseFloat(style["border" + side + "Width"]) > 0 && !transparent(style["border" + side + "Color"])) + return borders + } + + const scope = () => { + const modals = [...document.querySelectorAll("dialog:modal")] + if (modals.length > 0) return modals[modals.length - 1] + const shownModals = [...document.querySelectorAll('[aria-modal="true"]')].filter(shown) + return shownModals.length > 0 ? shownModals[shownModals.length - 1] : document.body + } + + const textOf = (element) => (element.getAttribute("aria-label") || element.textContent || element.getAttribute("title") || element.getAttribute("placeholder") || "").replace(/\s+/g, " ").trim() + + /** Everything the audit measures: lines of text, controls, and the buttons of a Touch layout. */ + const items = () => { + const root = scope() + const out = [] + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT) + for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) { + if (node.nodeValue.trim() === "") continue + const parent = node.parentElement + if (parent === null || parent.closest("script, style, noscript, template, #" + marksId) !== null || !shown(parent)) continue + if (transparent(getComputedStyle(parent).color)) continue + const control = parent.closest(controlSelector) + const range = document.createRange() + range.selectNodeContents(node) + for (const rect of range.getClientRects()) { + if (rect.width < 1 || rect.height < 1) continue + out.push({ kind: "text", element: parent, owner: control !== null && root.contains(control) ? control : parent, rect: box(rect), text: node.nodeValue.replace(/\s+/g, " ").trim() }) + } + } + for (const control of root.querySelectorAll(controlSelector)) { + if (control.disabled || control.closest("[inert]") !== null || !shown(control)) continue + const rect = control.getBoundingClientRect() + if (rect.width < 1 || rect.height < 1) continue + out.push({ kind: "control", element: control, owner: control, rect: box(rect), text: textOf(control), painted: painted(control) || control.querySelector("img, svg, canvas, video") !== null }) + } + for (const element of document.querySelectorAll("canvas, [style*='touch-action']")) { + const read = element[touchKey] + if (typeof read !== "function" || !shown(element)) continue + const origin = element.getBoundingClientRect() + let layout + try { layout = read() } catch (error) { continue } + for (const [name, zone] of Object.entries(layout.buttons || {})) { + if (zone === null || zone === undefined) continue + const x = origin.left + zone.center.x + const y = origin.top + zone.center.y + out.push({ kind: "touch", element, owner: element, rect: { l: x - zone.radius, t: y - zone.radius, r: x + zone.radius, b: y + zone.radius }, text: name, centre: { x, y }, radius: zone.radius }) + } + } + return out + } + + /** The layout box of an element, transforms left out, in page pixels. */ + const layoutBox = (element) => { + let left = 0 + let top = 0 + for (let at = element; at !== null && at instanceof HTMLElement; at = at.offsetParent) { + left += at.offsetLeft + top += at.offsetTop + } + return element instanceof HTMLElement ? [left, top, element.offsetWidth, element.offsetHeight] : [0, 0, 0, 0] + } + + const snapshot = () => { + const seen = new Set() + const out = [] + for (const item of items()) { + if (seen.has(item.element)) continue + seen.add(item.element) + out.push([selector(item.element), ...layoutBox(item.element), item.kind === "text" ? item.text.length : 0]) + } + return JSON.stringify(out) + } + + const describe = (item) => ({ kind: item.kind, selector: selector(item.element), text: item.text.slice(0, 60) }) + const allowed = (allow, check, elements) => + allow.some((entry) => entry.check === check && elements.some((element) => { + try { return element !== null && element.closest(entry.selector) !== null } catch (error) { return false } + })) + + const related = (owner, element) => element === owner || owner.contains(element) + + /** + * Whether two owners really show over each other at the points of their + * shared box: both hit there, and nothing drawn sits above both (a scrim + * over a hidden button, a panel over the stage). Every element answers hit + * tests while this runs, so text in a pointer-events: none layer counts. + */ + const shownTogether = (a, b, boxA, boxB, shared) => { + const points = [] + for (const fx of [0.2, 0.5, 0.8]) for (const fy of [0.2, 0.5, 0.8]) points.push([shared.l + width(shared) * fx, shared.t + height(shared) * fy]) + let top = null + let count = 0 + for (const [x, y] of points) { + const stack = document.elementsFromPoint(x, y) + const hitA = stack.findIndex((element) => related(a, element)) + const hitB = stack.findIndex((element) => related(b, element)) + if (hitA === -1 || hitB === -1) continue + const first = Math.min(hitA, hitB) + const covered = stack.slice(0, first).some((element) => element.id !== marksId && painted(element)) + if (covered) continue + // A layer the upper one sits on (a scrim, a panel) that hides the whole lower one: the lower is behind it, not under the upper. + const [upper, lower, lowerBox] = hitA < hitB ? [a, b, boxB] : [b, a, boxA] + const layered = stack.slice(0, Math.max(hitA, hitB)).some((element) => element !== upper && !related(lower, element) && element.id !== marksId && painted(element) && contains(box(element.getBoundingClientRect()), lowerBox)) + if (layered) continue + count++ + if (top === null) top = hitA < hitB ? "a" : "b" + } + return { count, top } + } + + const tapExtent = (item, most) => { + const x0 = item.rect.l + width(item.rect) / 2 + const y0 = item.rect.t + height(item.rect) / 2 + const hits = (x, y) => { + if (x < 0 || y < 0 || x >= innerWidth || y >= innerHeight) return false + const found = document.elementFromPoint(x, y) + return found !== null && related(item.element, found) + } + if (!hits(x0, y0)) return { width: 0, height: 0, box: null } + const run = (dx, dy) => { + let steps = 0 + while (steps < most && hits(x0 + dx * (steps + 1), y0 + dy * (steps + 1))) steps++ + return steps + } + const left = run(-1, 0), right = run(1, 0), up = run(0, -1), down = run(0, 1) + return { width: left + right + 1, height: up + down + 1, box: { l: x0 - left, t: y0 - up, r: x0 + right + 1, b: y0 + down + 1 } } + } + + const audit = (options) => { + const insets = options.insets + const minTarget = options.minTarget + const allow = options.allow + const checks = new Set(options.checks) + // The screen's size, not the page's: on a phone, a page wider than the screen widens its layout and is zoomed out to fit. + const screenWidth = options.width + const screenHeight = options.height + const viewport = { l: 0, t: 0, r: screenWidth, b: screenHeight } + const safe = { l: insets.left, t: insets.top, r: screenWidth - insets.right, b: screenHeight - insets.bottom } + const pageScrolls = document.documentElement.scrollHeight > screenHeight + 1 + const all = items() + const findings = [] + let skipped = 0 + const add = (check, elements, finding) => { + if (allowed(allow, check, elements)) { skipped++; return } + findings.push({ check, ...finding }) + } + for (const item of all) { + item.clips = item.kind === "touch" ? [] : clippers(item.element, item.kind === "text") + item.scroller = nearestScroller(item.clips) + item.visible = item.clips.reduce((rect, clip) => meet(rect, clip.box), item.rect) + } + + // overlap: two things drawn over each other, neither part of the other. + if (checks.has("overlap")) { + const force = document.createElement("style") + force.textContent = "*, *::before, *::after { pointer-events: auto !important; }" + document.documentElement.appendChild(force) + try { + const drawn = all.filter((item) => item.kind === "text" || (item.kind === "control" && item.painted)) + const reported = new Set() + for (let i = 0; i < drawn.length; i++) for (let j = i + 1; j < drawn.length; j++) { + const a = drawn[i], b = drawn[j] + if (a.owner === b.owner || a.owner.contains(b.owner) || b.owner.contains(a.owner)) continue + const shared = meet(a.visible, b.visible) + if (width(shared) <= 2 || height(shared) <= 2) continue + const key = [selector(a.owner), selector(b.owner)].sort().join("|") + if (reported.has(key)) continue + const seen = shownTogether(a.owner, b.owner, a.visible, b.visible, shared) + if (seen.count === 0) continue + reported.add(key) + const over = seen.top === "a" ? a : b + const under = seen.top === "a" ? b : a + add("overlap", [a.owner, b.owner], { + element: describe(under), + other: describe(over), + box: round(shared), + saw: "they overlap by " + Math.round(width(shared)) + " x " + Math.round(height(shared)) + " px, and the " + over.kind + " is drawn over the " + under.kind + (under.kind === "control" ? ", so a tap there reaches the wrong thing" : "") + }) + } + // A Touch button under something the page draws, or two Touch buttons that share space. + const touches = all.filter((item) => item.kind === "touch") + for (const touch of touches) { + const found = document.elementFromPoint(touch.centre.x, touch.centre.y) + if (found !== null && !related(touch.element, found) && found.id !== marksId && (painted(found) || textOf(found) !== "")) { + add("overlap", [touch.element, found], { element: describe(touch), other: { kind: "control", selector: selector(found), text: textOf(found).slice(0, 60) }, box: round(touch.rect), saw: "the page draws over the touch button " + JSON.stringify(touch.text) + " at its centre" }) + } + } + for (let i = 0; i < touches.length; i++) for (let j = i + 1; j < touches.length; j++) { + const a = touches[i], b = touches[j] + if (Math.hypot(a.centre.x - b.centre.x, a.centre.y - b.centre.y) < a.radius + b.radius - 2) { + add("overlap", [a.element], { element: describe(a), other: describe(b), box: round(meet(a.rect, b.rect)), saw: "touch buttons " + JSON.stringify(a.text) + " and " + JSON.stringify(b.text) + " share space, so a thumb between them presses either" }) + } + } + } finally { + force.remove() + } + } + + // cut-text and truncated-text: text a clipping box hides part of. + const cut = new Set() + for (const item of all) { + if (item.kind !== "text" || cut.has(item.element)) continue + if (area(item.visible) < 1) continue + for (const clip of item.clips) { + if (clip.scroll) break + const inside = meet(item.rect, clip.box) + if (width(item.rect) - width(inside) <= 1 && height(item.rect) - height(inside) <= 1) continue + cut.add(item.element) + const style = getComputedStyle(clip.element) + const meant = style.textOverflow === "ellipsis" || (style.webkitLineClamp !== undefined && style.webkitLineClamp !== "none") + if (meant && checks.has("truncated-text")) { + add("truncated-text", [item.element, clip.element], { element: describe(item), box: round(item.visible), saw: JSON.stringify(item.text.slice(0, 60)) + " is shortened with an ellipsis: " + Math.round(width(inside)) + " of its " + Math.round(width(item.rect)) + " px show" }) + } else if (!meant && checks.has("cut-text")) { + add("cut-text", [item.element, clip.element], { element: describe(item), box: round(item.visible), saw: JSON.stringify(item.text.slice(0, 60)) + " is cut by the edge of " + selector(clip.element) + " (" + Math.round(width(inside)) + " x " + Math.round(height(inside)) + " of " + Math.round(width(item.rect)) + " x " + Math.round(height(item.rect)) + " px show), with nothing to say it is cut" }) + } + break + } + } + if (checks.has("truncated-text")) { + for (const element of scope().querySelectorAll("*")) { + if (cut.has(element) || !shown(element)) continue + const style = getComputedStyle(element) + if (style.webkitLineClamp === undefined || style.webkitLineClamp === "none" || element.scrollHeight <= element.clientHeight + 1) continue + cut.add(element) + add("truncated-text", [element], { element: { kind: "text", selector: selector(element), text: textOf(element).slice(0, 60) }, box: round(box(element.getBoundingClientRect())), saw: "the text is clamped to " + style.webkitLineClamp + " lines and the rest is hidden" }) + } + } + + // off-screen, safe-area, scroll-edge: where the visible part of each item is. + const scrollEdge = new Set() + const edgeOf = (item, saw) => { + const key = item.scroller.element + if (scrollEdge.has(key)) return + scrollEdge.add(key) + add("scroll-edge", [item.owner, item.scroller.element], { element: describe(item), box: round(item.visible), saw }) + } + const placed = new Set() + // A control's own box first: its text is judged only when the box is fine. + for (const item of [...all].sort((a, b) => (a.kind === "text" ? 1 : 0) - (b.kind === "text" ? 1 : 0))) { + if (item.kind === "touch") { + const out = item.rect.l < -1 || item.rect.t < -1 || item.rect.r > screenWidth + 1 || item.rect.b > screenHeight + 1 + if (out && checks.has("off-screen")) add("off-screen", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " reaches past the edge of the screen" }) + else if (!out && checks.has("safe-area") && area(meet(item.rect, safe)) < area(item.rect) - 1) add("safe-area", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " reaches into the space the device keeps (a notch, a home bar)" }) + continue + } + if (placed.has(item.owner) || area(item.visible) < 1) continue + // A control that draws nothing (a touch area over the stage) has nothing to cut; its text is measured on its own. + if (item.kind === "control" && !item.painted) continue + // A box that spans the whole screen (the game's root, the body) cuts at the screen's edge: that is off-screen, not a cut. + const shownPart = item.clips.filter((clip) => !contains(clip.box, viewport)).reduce((rect, clip) => meet(rect, clip.box), item.rect) + if (area(shownPart) > area(item.visible) + 1) item.visible = shownPart + const onScreen = meet(item.visible, viewport) + if (area(onScreen) < 1) continue + if (item.kind === "control" && item.scroller !== undefined && checks.has("scroll-edge")) { + const shownShare = area(item.visible) / Math.max(1, area(item.rect)) + if (shownShare > 0.1 && shownShare < 0.9) { + edgeOf(item, "only " + Math.round(shownShare * 100) + "% of it shows at the edge of a scrolling area") + } + } + const past = area(onScreen) < area(item.visible) - 1 + if (past) { + placed.add(item.owner) + if (item.scroller !== undefined) { + if (checks.has("scroll-edge")) edgeOf(item, "its scrolling area runs past the edge of the screen") + } else if (pageScrolls && item.visible.t >= 0 && item.visible.l >= 0 && item.visible.r <= screenWidth + 1) { + // Below the fold of a page that scrolls: not cut. + } else if (checks.has("off-screen")) { + add("off-screen", [item.owner], { element: describe(item), box: round(item.visible), saw: "it reaches past the " + ["left", "top", "right", "bottom"].filter((side, index) => [item.visible.l < -1, item.visible.t < -1, item.visible.r > screenWidth + 1, item.visible.b > screenHeight + 1][index]).join(" and ") + " edge of the screen" }) + } + continue + } + if (item.scroller !== undefined || !checks.has("safe-area")) continue + const inSafe = meet(onScreen, safe) + if (area(inSafe) < area(onScreen) - 1) { + placed.add(item.owner) + const sides = ["left", "top", "right", "bottom"].filter((side, index) => [onScreen.l < safe.l - 1, onScreen.t < safe.t - 1, onScreen.r > safe.r + 1, onScreen.b > safe.b + 1][index]) + add("safe-area", [item.owner], { element: describe(item), box: round(onScreen), saw: "it reaches into the space the device keeps on the " + sides.join(" and ") + " (" + sides.map((side) => insets[side] + " px").join(", ") + ": a notch or the home bar)" }) + } + } + + // small-target: the area that takes a tap, by hit tests through the control's centre. + if (checks.has("small-target")) { + for (const item of all) { + if (item.kind === "touch") { + if (item.radius * 2 < minTarget) add("small-target", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " is " + Math.round(item.radius * 2) + " px across (at least " + minTarget + ")" }) + continue + } + if (item.kind !== "control") continue + if (width(item.rect) >= minTarget && height(item.rect) >= minTarget) continue + if (area(item.visible) < area(item.rect) - 1 || area(meet(item.visible, viewport)) < area(item.visible) - 1) continue + const tap = tapExtent(item, minTarget * 2) + // Something else takes a tap at its centre: it is behind a layer now, not a target (overlap reports a partial cover). + if (tap.box === null) continue + if (tap.width >= minTarget && tap.height >= minTarget) continue + add("small-target", [item.element], { + element: describe(item), + box: round(tap.box), + saw: "a tap reaches it over " + tap.width + " x " + tap.height + " px (at least " + minTarget + " x " + minTarget + ")" + }) + } + } + return { findings, measured: all.length, allowed: skipped } + } + + const steady = () => { steadyFrom = performance.now() } + const shifts = (allow) => { + const out = [] + let skipped = 0 + const seen = new Set() + for (const entry of shiftLog) { + if (entry.time < steadyFrom) continue + for (const node of entry.nodes) { + const element = node.nodeType === 1 ? node : node.parentElement + if (element === null || seen.has(element) || element.id === marksId) continue + seen.add(element) + if (allowed(allow, "layout-shift", [element])) { skipped++; continue } + out.push({ check: "layout-shift", element: { kind: "text", selector: selector(element), text: textOf(element).slice(0, 60) }, box: round(box(element.getBoundingClientRect())), saw: "it moved while nothing was pressed (layout shift " + entry.value.toFixed(4) + ")" }) + } + } + return { findings: out, allowed: skipped } + } + + // Longer text: about 40% more of every text, kept as the game renders new text. + const written = new WeakMap() + const padNode = (node) => { + if (node.nodeValue.trim() === "" || written.get(node) === node.nodeValue) return + const parent = node.parentElement + if (parent === null || parent.closest("script, style, noscript, template, #" + marksId) !== null) return + // The text's own letters, so the added width is the font's own: about 40% more. + const longer = node.nodeValue.replace(/\S(?:[\s\S]*\S)?/, (text) => text + " " + text.repeat(2).slice(0, Math.ceil(text.length * 0.4))) + written.set(node, longer) + node.nodeValue = longer + } + const padAll = (root) => { + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT) + const nodes = [] + for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) nodes.push(node) + nodes.forEach(padNode) + } + const pad = () => { + padAll(document.body) + new MutationObserver((records) => { + for (const record of records) { + if (record.type === "characterData") padNode(record.target) + for (const added of record.addedNodes) { + if (added.nodeType === 3) padNode(added) + else if (added.nodeType === 1) padAll(added) + } + } + }).observe(document.body, { subtree: true, childList: true, characterData: true }) + } + + const unmark = () => document.getElementById(marksId)?.remove() + const mark = (boxes) => { + unmark() + const layer = document.createElement("div") + layer.id = marksId + layer.setAttribute("popover", "manual") + layer.style.cssText = "position:fixed;inset:0;margin:0;padding:0;border:0;width:100vw;height:100vh;max-width:none;max-height:none;background:transparent;overflow:visible;pointer-events:none" + for (const entry of boxes) { + const outline = document.createElement("div") + outline.style.cssText = "position:fixed;box-sizing:border-box;left:" + entry.l + "px;top:" + entry.t + "px;width:" + Math.max(2, entry.r - entry.l) + "px;height:" + Math.max(2, entry.b - entry.t) + "px;border:2px solid " + entry.color + ";background:" + entry.color + "22" + const tag = document.createElement("span") + tag.textContent = String(entry.number) + tag.style.cssText = "position:absolute;left:-2px;bottom:100%;font:700 11px/14px system-ui,sans-serif;color:#fff;background:" + entry.color + ";padding:0 4px" + outline.appendChild(tag) + layer.appendChild(outline) + } + document.body.appendChild(layer) + if (typeof layer.showPopover === "function") layer.showPopover() + } + + globalThis.__ritmoLayout = { snapshot, audit, steady, shifts, pad, mark, unmark } +})()` diff --git a/packages/devtools/src/internal/layoutWalk.ts b/packages/devtools/src/internal/layoutWalk.ts new file mode 100644 index 00000000..639bc23c --- /dev/null +++ b/packages/devtools/src/internal/layoutWalk.ts @@ -0,0 +1,348 @@ +/** + * `ritmo-devtools layout`: opens each screen of a route in Chromium at each + * of the route's viewports and reports what is broken in the layout the + * player sees, whatever the game. + * + * Errors (they fail the run) are what the browser measures exactly and a + * game never means: + * + * - `overlap`: two things drawn over each other, neither part of the other + * (text over text, text over a button, a button over a Touch button), with + * nothing drawn above both; + * - `cut-text`: text the edge of a box cuts, with nothing to say it is cut; + * - `off-screen`: part of a text or control past the edge of the screen; + * - `safe-area`: part of a text or control in the space the device keeps + * (a notch, the home bar), from the viewport's insets; + * - `small-target`: the area that takes a tap is under `minTarget` either + * side (hit tests through the control's centre, so a `::after` that + * extends it counts); + * - `layout-shift`: something moved while nothing was pressed (Chromium's + * own layout-shift entries); + * - `never-settled`: the layout kept moving, so nothing could be measured. + * + * Warnings (reported, never failing) are what a game may mean: + * + * - `truncated-text`: text shortened with an ellipsis or a line clamp; + * - `scroll-edge`: a control half hidden at the edge of a scrolling area, or + * a scrolling area that runs past the screen; + * - every finding of the longer-text pass, which loads each screen again + * with every text about 40% longer: a layout with no room to spare, not a + * layout broken today. + * + * A route's `allow` lists what a game means, by check and selector. The + * page side is `layoutPage.ts`; reaching a screen is the keyboard walk's. + * + * @internal + */ +import { mkdirSync } from "node:fs" +import { join } from "node:path" + +import type * as PlaywrightTypes from "playwright" + +import { pageScript } from "./layoutPage.ts" +import type { Opened } from "./pageOpen.ts" +import { open, reach } from "./pageOpen.ts" +import type { Allow, LayoutCheckId, Route, Screen, Viewport } from "./route.ts" + +/** Every check the audit decides on each screen at each viewport. */ +export type CheckId = LayoutCheckId | "never-settled" + +export const checks: ReadonlyArray = ["overlap", "cut-text", "off-screen", "safe-area", "small-target", "layout-shift", "never-settled", "truncated-text", "scroll-edge"] + +/** What a finding is about: the route, the page, or a check. */ +export type FindingCheck = CheckId | "route-step" | "page-error" | "nothing-measured" + +export type Severity = "error" | "warning" + +const warnings: ReadonlySet = new Set(["truncated-text", "scroll-edge"]) + +/** An element as the report names it. */ +export interface Element { + readonly kind: "text" | "control" | "touch" + readonly selector: string + readonly text: string +} + +export interface Box { + readonly l: number + readonly t: number + readonly r: number + readonly b: number +} + +/** + * One finding: the check, how much it matters, where (screen, viewport, and + * which pass), the element (and the other one of an overlap), what was + * measured, and what to change. + */ +export interface Finding { + readonly check: FindingCheck + readonly severity: Severity + readonly screen: string + readonly viewport: string + readonly pass: "normal" | "longer-text" + readonly element?: Element + readonly other?: Element + readonly box?: Box + readonly saw: string + readonly fix: string + /** The screenshot that marks it, and its number there, with `screenshots`. */ + readonly screenshot?: { readonly file: string; readonly number: number } +} + +/** One check on one screen at one viewport: `warned` has only warnings, `count` its findings. */ +export interface Outcome { + readonly check: CheckId + readonly screen: string + readonly viewport: string + readonly status: "passed" | "failed" | "warned" | "not-checked" + readonly count: number + readonly reason?: string +} + +export interface Report { + readonly ok: boolean + readonly url: string + readonly screens: ReadonlyArray + readonly viewports: ReadonlyArray + readonly findings: ReadonlyArray + readonly outcomes: ReadonlyArray + /** Findings the route's `allow` left out. */ + readonly allowed: number +} + +export const fixes: Record = { + overlap: "give each its own space at this size (anchor them to different edges, let one wrap or shrink); when one is meant to sit over the other, list it in the route's allow", + "cut-text": "let the text wrap or give its box room at this size; when a cut is meant, end it with text-overflow: ellipsis so the player sees that text is missing", + "off-screen": "keep it inside the screen at this size: anchor it to the edges, or let the layout wrap or shrink", + "safe-area": "keep it out of the device's own space: offset its edge by env(safe-area-inset-*)", + "small-target": "make the area that takes a tap bigger (padding, or a ::after that extends it), or set the route's minTarget when the game's own rule is smaller", + "layout-shift": "keep its size while its content changes: size it to its widest value (a min-width, font-variant-numeric: tabular-nums for numbers), and animate with transform", + "never-settled": "keep the layout still while nothing happens: animate with transform, not position or size, and honour prefers-reduced-motion; or wait longer in the route's steps", + "truncated-text": "check the player can still tell what it says, and show the whole text where there is room; list it in the route's allow when the cut is meant", + "scroll-edge": "when the player must find what is past the edge, show that the area scrolls; otherwise nothing to change", + "route-step": "fix the route so this step can run, or the page so it shows what the step names; this screen was not measured", + "page-error": "fix the error the page threw; the screen was still measured", + "nothing-measured": "give this screen steps that reach what the game shows, or take it out of the route; drawing on a canvas is not measured, only text, controls, and Touch buttons" +} + +/** What the audit needs from outside. */ +export interface Options { + readonly url: string + readonly route: Route + /** Milliseconds after a page load before the route's steps run (default 1500). */ + readonly settle?: number + /** Milliseconds of a still screen watched for layout shifts (default 1500). */ + readonly steadyFor?: number + /** Longest wait for the layout to hold still, in milliseconds (default 6000). */ + readonly settleLimit?: number + /** A directory for one screenshot per screen and viewport with findings, each finding boxed and numbered. */ + readonly screenshots?: string + /** The longer-text pass (default true). */ + readonly longerText?: boolean +} + +interface PageFinding { + readonly check: CheckId + readonly element?: Element + readonly other?: Element + readonly box?: Box + readonly saw: string +} + +interface Audited { + readonly findings: ReadonlyArray + readonly measured: number + readonly allowed: number +} + +const callPage = (page: PlaywrightTypes.Page, name: string, argument?: unknown): Promise => + page.evaluate( + ({ name, argument }) => (Reflect.get(globalThis, "__ritmoLayout") as Record unknown>)[name]!(argument), + { name, argument } + ) + +const fileName = (text: string): string => text.replace(/[^a-z0-9@._-]+/gi, "-") + +/** Waits for fonts, then until two layout readings 250 ms apart agree; the elements that still move when they never do. */ +const settleLayout = async (page: PlaywrightTypes.Page, limit: number): Promise | undefined> => { + await page.evaluate("document.fonts.ready.then(() => true)").catch(() => true) + let last = await callPage(page, "snapshot") as string + for (let waited = 0; waited < limit; waited += 250) { + await page.waitForTimeout(250) + const next = await callPage(page, "snapshot") as string + if (next === last) return undefined + last = next + } + const before = JSON.parse(last) as ReadonlyArray> + await page.waitForTimeout(250) + const after = JSON.parse(await callPage(page, "snapshot") as string) as ReadonlyArray> + const kept = new Map(before.map((entry) => [String(entry[0]), JSON.stringify(entry)])) + const moving = after.filter((entry) => kept.get(String(entry[0])) !== JSON.stringify(entry)).map((entry) => String(entry[0])) + return moving.length > 0 ? moving : ["elements that come and go"] +} + +const colors: Record = { error: "#e5195f", warning: "#f5a000" } + +const keyOf = (finding: PageFinding): string => `${finding.check}|${finding.element?.selector ?? ""}|${finding.other?.selector ?? ""}` + +const describeElement = (element: Element): string => + element.kind === "touch" + ? `touch button ${JSON.stringify(element.text)} on ${element.selector}` + : `${element.kind} ${element.text === "" ? "(no text)" : JSON.stringify(element.text)} at ${element.selector}` + +/** Runs the audit in an open Chromium; the caller launches and closes it. */ +export const walk = async (browser: PlaywrightTypes.Browser, options: Options): Promise => { + const findings: Array = [] + const outcomes: Array = [] + let allowed = 0 + if (options.screenshots !== undefined) mkdirSync(options.screenshots, { recursive: true }) + for (const screen of options.route.screens) { + for (const viewport of options.route.viewports) { + const result = await auditScreen(browser, options, screen, viewport) + findings.push(...result.findings) + outcomes.push(...result.outcomes) + allowed += result.allowed + } + } + return { + ok: findings.every((finding) => finding.severity === "warning"), + url: options.url, + screens: options.route.screens.map((screen) => screen.name), + viewports: options.route.viewports.map((viewport) => viewport.name), + findings, + outcomes, + allowed + } +} + +interface ScreenResult { + readonly findings: ReadonlyArray + readonly outcomes: ReadonlyArray + readonly allowed: number +} + +const auditScreen = async (browser: PlaywrightTypes.Browser, options: Options, screen: Screen, viewport: Viewport): Promise => { + const settle = options.settle ?? 1500 + const limit = options.settleLimit ?? 6000 + const allow: ReadonlyArray = [...options.route.allow, ...screen.allow] + const findings: Array = [] + const at = { screen: screen.name, viewport: viewport.name } + const walkFinding = (check: FindingCheck, saw: string): Finding => ({ check, severity: "error", ...at, pass: "normal", saw, fix: fixes[check] }) + const notChecked = (reason: string): ScreenResult => ({ + findings, + outcomes: checks.map((check) => ({ check, ...at, status: "not-checked", count: 0, reason })), + allowed: 0 + }) + const opening = { width: viewport.width, height: viewport.height, touch: viewport.touch, insets: viewport.insets, reducedMotion: true, scripts: [pageScript] } + + // The screen as the player sees it. + let opened: Opened = await open(browser, opening) + let base: Audited + let shifted: Audited + try { + const reached = await reach(opened, options.url, screen, settle) + if (reached !== undefined) { + findings.push(walkFinding("route-step", reached)) + return notChecked(`the route did not reach this screen (${reached})`) + } + const moving = await settleLayout(opened.page, limit) + if (opened.errors.length > 0) findings.push(walkFinding("page-error", `the page threw while the screen opened: ${opened.errors[0]}`)) + if (moving !== undefined) { + findings.push({ ...walkFinding("never-settled", `the layout kept moving for ${limit / 1000} s: ${moving.slice(0, 4).join(", ")}${moving.length > 4 ? ` and ${moving.length - 4} more` : ""}`) }) + return { ...notChecked("the layout never held still"), outcomes: checks.map((check) => check === "never-settled" ? { check, ...at, status: "failed" as const, count: 1 } : { check, ...at, status: "not-checked" as const, count: 0, reason: "the layout never held still" }) } + } + await callPage(opened.page, "steady") + base = await callPage(opened.page, "audit", { width: viewport.width, height: viewport.height, insets: viewport.insets, minTarget: options.route.minTarget, allow, checks: checks.filter((check) => check !== "layout-shift" && check !== "never-settled") }) as Audited + await opened.page.waitForTimeout(options.steadyFor ?? 1500) + shifted = await callPage(opened.page, "shifts", allow) as Audited + if (base.measured === 0) { + findings.push(walkFinding("nothing-measured", "the screen showed no text, control, or Touch button to measure")) + return notChecked("nothing to measure") + } + const normal = [...base.findings, ...shifted.findings].map((finding): Finding => ({ + ...finding, + severity: warnings.has(finding.check) ? "warning" : "error", + ...at, + pass: "normal", + fix: fixes[finding.check] + })) + findings.push(...await shoot(opened, options, normal, `${screen.name}@${viewport.name}`)) + } finally { + await opened.close() + } + + // The same screen with every text about 40% longer: what has no room to spare. + let longer: Audited | undefined + if (options.longerText !== false) { + opened = await open(browser, opening) + try { + const reached = await reach(opened, options.url, screen, settle) + if (reached === undefined) { + await callPage(opened.page, "pad") + const moving = await settleLayout(opened.page, limit) + if (moving === undefined) { + longer = await callPage(opened.page, "audit", { width: viewport.width, height: viewport.height, insets: viewport.insets, minTarget: options.route.minTarget, allow, checks: ["overlap", "cut-text", "off-screen", "safe-area"] }) as Audited + const known = new Set(base.findings.map(keyOf)) + const fresh = longer.findings.filter((finding) => !known.has(keyOf(finding))).map((finding): Finding => ({ + ...finding, + severity: "warning", + ...at, + pass: "longer-text", + saw: `with every text about 40% longer, ${finding.saw}`, + fix: `${fixes[finding.check]}; this is a layout with no room to spare (a longer name, a translation), not one broken today` + })) + findings.push(...await shoot(opened, options, fresh, `${screen.name}@${viewport.name}-longer-text`)) + } + } + } finally { + await opened.close() + } + } + + const outcomes = checks.map((check): Outcome => { + const mine = findings.filter((finding) => finding.check === check && finding.pass === "normal") + const status = mine.some((finding) => finding.severity === "error") ? "failed" : mine.length > 0 ? "warned" : "passed" + return { check, ...at, status, count: mine.length } + }) + return { findings, outcomes, allowed: base.allowed + shifted.allowed + (longer?.allowed ?? 0) } +} + +/** With `screenshots`, boxes and numbers each finding on the page and saves it; the findings then name their file and number. */ +const shoot = async (opened: Opened, options: Options, found: ReadonlyArray, name: string): Promise> => { + if (options.screenshots === undefined || found.length === 0) return found + const file = join(options.screenshots, `${fileName(name)}.png`) + const numbered = found.map((finding, index) => ({ ...finding, screenshot: { file, number: index + 1 } })) + await callPage(opened.page, "mark", numbered.filter((finding) => finding.box !== undefined).map((finding) => ({ ...finding.box, number: finding.screenshot.number, color: colors[finding.severity] }))) + await opened.page.screenshot({ path: file }) + await callPage(opened.page, "unmark") + return numbered +} + +/** The report as text: one line per finding, then each screen's checks per viewport, then the totals. */ +export const format = (report: Report): string => { + const lines: Array = [] + for (const finding of report.findings) { + const where = `[${finding.screen} @ ${finding.viewport}${finding.pass === "longer-text" ? ", longer text" : ""}]` + const what = finding.element === undefined ? "" : ` ${describeElement(finding.element)}${finding.other === undefined ? "" : `, and ${describeElement(finding.other)}`}` + const shot = finding.screenshot === undefined ? "" : ` (${finding.screenshot.file} #${finding.screenshot.number})` + lines.push(`${finding.severity === "error" ? "FAIL" : "warn"} ${finding.check} ${where}${what}: ${finding.saw}.${shot} Fix: ${finding.fix}.`) + } + if (lines.length > 0) lines.push("") + for (const screen of report.screens) { + lines.push(`[${screen}]`) + for (const viewport of report.viewports) { + const mine = report.outcomes.filter((outcome) => outcome.screen === screen && outcome.viewport === viewport) + const skipped = mine.find((outcome) => outcome.status === "not-checked" && outcome.check !== "never-settled") ?? mine.find((outcome) => outcome.status === "not-checked") + const said = mine.filter((outcome) => outcome.status === "failed" || outcome.status === "warned").map((outcome) => `${outcome.status === "failed" ? "FAIL" : "warn"} ${outcome.check} (${outcome.count})`) + const passed = mine.filter((outcome) => outcome.status === "passed").length + const parts = [...said, ...(passed > 0 ? [`pass ${passed} more`] : [])] + lines.push(` ${viewport}: ${parts.join(", ")}${skipped?.reason === undefined ? "" : `${parts.length > 0 ? "; " : ""}not checked: ${skipped.reason}`}`) + } + } + const errors = report.findings.filter((finding) => finding.severity === "error").length + const warned = report.findings.length - errors + lines.push("") + lines.push(`layout audit of ${report.url}: ${errors} error${errors === 1 ? "" : "s"}, ${warned} warning${warned === 1 ? "" : "s"} on ${report.screens.length} screen${report.screens.length === 1 ? "" : "s"} at ${report.viewports.length} viewport${report.viewports.length === 1 ? "" : "s"}${report.allowed > 0 ? ` (${report.allowed} left out by the route's allow)` : ""}`) + return lines.join("\n") +} diff --git a/packages/devtools/test/LayoutWalk.test.ts b/packages/devtools/test/LayoutWalk.test.ts new file mode 100644 index 00000000..2832de3b --- /dev/null +++ b/packages/devtools/test/LayoutWalk.test.ts @@ -0,0 +1,204 @@ +import { execFile } from "node:child_process" +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { createServer, type Server } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { afterAll, beforeAll, describe, expect, it } from "vitest" + +import type * as LayoutWalkModule from "../src/internal/layoutWalk.ts" +import type * as RouteModule from "../src/internal/route.ts" + +// The audit is internal to the CLI (the package exports no `internal/*`): load its source to call it with short waits. +const LayoutWalk = await import(new URL("../src/internal/layoutWalk.ts", import.meta.url).href) as typeof LayoutWalkModule +const Route = await import(new URL("../src/internal/route.ts", import.meta.url).href) as typeof RouteModule + +const cli = fileURLToPath(new URL("../src/cli.ts", import.meta.url)) +const fixtures = fileURLToPath(new URL("./fixtures/layout/", import.meta.url)) + +const run = (args: ReadonlyArray, cwd: string) => + new Promise<{ readonly status: number | null; readonly stdout: string; readonly stderr: string }>((done) => { + execFile(process.execPath, [cli, "layout", ...args], { cwd }, (error, stdout, stderr) => { + done({ status: error === null ? 0 : typeof error.code === "number" ? error.code : null, stdout, stderr }) + }) + }) + +describe("ritmo-devtools layout arguments", () => { + it("refuses a bad route, URL, or flag with exit code 2", async () => { + const dir = mkdtempSync(join(tmpdir(), "ritmo-layout-")) + const route = join(dir, "route.json") + writeFileSync(route, JSON.stringify({ screens: [{ name: "a" }], minTarget: -1 })) + expect(await run(["--route", route], fixtures)).toMatchObject({ status: 2, stdout: "", stderr: `${route} is not a layout route: route.minTarget must be a whole number of CSS pixels from 1 to 200 (the smallest tap area either side)\n` }) + expect(await run(["--url", "localhost:5173"], fixtures)).toMatchObject({ status: 2, stderr: "--url must be an http(s) URL of the game's page, such as http://localhost:5173/; got localhost:5173\n" }) + expect((await run(["--screenshots"], fixtures)).status).toBe(2) + }, 60_000) +}) + +// The browsers are downloaded only where browser runs are checked (the CI job "Determinism across engines"). +const skipped = await (async () => { + try { + const playwright = await import("playwright") + return existsSync(playwright.chromium.executablePath()) ? undefined : "chromium is not downloaded (npx playwright install chromium)" + } catch { + return "playwright is not installed" + } +})() +if (skipped !== undefined) console.warn(`layout audit tests skipped: ${skipped}`) + +const types: Record = { ".html": "text/html" } + +/** An iPhone held sideways: the camera's side and the home bar are the device's own. */ +const phone: RouteModule.Viewport = { name: "phone", width: 844, height: 390, touch: true, insets: { top: 0, right: 47, bottom: 21, left: 47 } } +const smallPhone: RouteModule.Viewport = { name: "small-phone", width: 667, height: 375, touch: true, insets: { top: 0, right: 0, bottom: 0, left: 0 } } + +describe.skipIf(skipped !== undefined)("ritmo-devtools layout", () => { + let server: Server + let base = "" + let browser: Awaited> + + beforeAll(async () => { + server = createServer((request, response) => { + const path = new URL(request.url ?? "/", "http://localhost").pathname + const file = join(fixtures, path === "/" ? "clean.html" : path) + if (!file.startsWith(fixtures) || !existsSync(file)) { + response.statusCode = 404 + response.end() + return + } + response.setHeader("content-type", types[file.slice(file.lastIndexOf("."))] ?? "text/plain") + response.end(readFileSync(file)) + }) + await new Promise((done) => server.listen(0, "127.0.0.1", done)) + const address = server.address() + base = `http://127.0.0.1:${typeof address === "object" && address !== null ? address.port : 0}/` + browser = await (await import("playwright")).chromium.launch() + }, 60_000) + + afterAll(async () => { + await browser?.close() + await new Promise((done) => server?.close(done)) + }) + + interface Audit { + readonly viewports?: ReadonlyArray + readonly allow?: ReadonlyArray + readonly minTarget?: number + readonly longerText?: boolean + readonly steadyFor?: number + readonly screenshots?: string + } + const audit = (fixture: string, options: Audit = {}) => + LayoutWalk.walk(browser, { + url: base, + route: { ...Route.startRoute, screens: [{ name: fixture, path: `${fixture}.html`, steps: [], noPress: [], allow: options.allow ?? [] }], viewports: options.viewports ?? [phone], minTarget: options.minTarget ?? 44 }, + settle: 100, + settleLimit: 2000, + steadyFor: options.steadyFor ?? 300, + longerText: options.longerText ?? false, + ...(options.screenshots === undefined ? {} : { screenshots: options.screenshots }) + }) + const found = (report: LayoutWalkModule.Report) => + report.findings.map((finding) => ({ check: finding.check, severity: finding.severity, pass: finding.pass, viewport: finding.viewport, text: finding.element?.text ?? "", other: finding.other?.text ?? "" })) + + it("finds nothing on a page laid out right at every default size, the longer text included", async () => { + // A badge on its card, an invisible touch area under the content, a button under a scrim, a 24 px + // button extended to 48 by ::after, a UI scaled to half, and a pulsing transform animation. + const report = await audit("clean", { viewports: Route.defaultViewports, longerText: true }) + expect(found(report)).toEqual([]) + expect(report.ok).toBe(true) + expect(report.outcomes.filter((outcome) => outcome.status !== "passed")).toEqual([]) + expect(LayoutWalk.format(report)).toMatch(/layout audit of http:\/\/127\.0\.0\.1:\d+\/: 0 errors, 0 warnings on 1 screen at 4 viewports$/) + }, 120_000) + + it("fails on two things drawn over each other, and names which is on top", async () => { + const report = await audit("overlap") + expect(found(report)).toEqual([ + { check: "overlap", severity: "error", pass: "normal", viewport: "phone", text: "Score 1200", other: "Lives 3" }, + { check: "overlap", severity: "error", pass: "normal", viewport: "phone", text: "Play", other: "Over it" } + ]) + expect(report.findings[1]!.saw).toMatch(/the text is drawn over the/) + expect(report.ok).toBe(false) + }, 60_000) + + it("leaves out what the route allows, and counts it", async () => { + const report = await audit("overlap", { allow: [{ check: "overlap", selector: ".b" }, { check: "overlap", selector: ".label" }] }) + expect(report.findings).toEqual([]) + expect(report.allowed).toBe(2) + expect(report.ok).toBe(true) + }, 60_000) + + it("fails on text a box cuts, and only warns when an ellipsis says it is cut", async () => { + const cut = await audit("cut-text") + expect(found(cut)).toEqual([{ check: "cut-text", severity: "error", pass: "normal", viewport: "phone", text: "The Gathering Storm", other: "" }]) + const truncated = await audit("truncated-text") + expect(found(truncated)).toEqual([{ check: "truncated-text", severity: "warning", pass: "normal", viewport: "phone", text: "Training Dummy", other: "" }]) + expect(truncated.ok).toBe(true) + }, 60_000) + + it("fails on a control past the screen's edge, and on one in the device's own space only where the device keeps it", async () => { + expect(found(await audit("off-screen"))).toEqual([{ check: "off-screen", severity: "error", pass: "normal", viewport: "phone", text: "Shop", other: "" }]) + // At the left edge: inside the notch on the phone, fine on a phone without one; the button offset by env() is fine on both. + expect(found(await audit("safe-area", { viewports: [phone, smallPhone] }))).toEqual([{ check: "safe-area", severity: "error", pass: "normal", viewport: "phone", text: "Menu", other: "" }]) + }, 60_000) + + it("fails on a small tap area, and on the route's own minimum", async () => { + const report = await audit("small-target") + expect(found(report)).toEqual([{ check: "small-target", severity: "error", pass: "normal", viewport: "phone", text: "Close", other: "" }]) + expect(report.findings[0]!.saw).toBe("a tap reaches it over 20 x 20 px (at least 44 x 44)") + expect((await audit("small-target", { minTarget: 20 })).ok).toBe(true) + }, 60_000) + + it("checks the buttons of a Touch layout drawn on the canvas", async () => { + const report = await audit("touch") + expect(found(report)).toEqual([ + { check: "overlap", severity: "error", pass: "normal", viewport: "phone", text: "dash", other: "Tip" }, + { check: "small-target", severity: "error", pass: "normal", viewport: "phone", text: "jump", other: "" } + ]) + }, 60_000) + + it("fails on layout that shifts while nothing is pressed, and on layout that never holds still", async () => { + expect(found(await audit("layout-shift", { steadyFor: 1500 }))).toEqual([{ check: "layout-shift", severity: "error", pass: "normal", viewport: "phone", text: "coins", other: "" }]) + const drifting = await audit("never-settled") + expect(found(drifting)).toEqual([{ check: "never-settled", severity: "error", pass: "normal", viewport: "phone", text: "", other: "" }]) + expect(drifting.findings[0]!.saw).toMatch(/#drift/) + expect(drifting.outcomes.filter((outcome) => outcome.status === "not-checked").map((outcome) => outcome.check)).toEqual(LayoutWalk.checks.filter((check) => check !== "never-settled")) + }, 60_000) + + it("warns on a control half hidden at a scroll edge", async () => { + const report = await audit("scroll-edge") + expect(found(report)).toEqual([{ check: "scroll-edge", severity: "warning", pass: "normal", viewport: "phone", text: "Rank 3", other: "" }]) + expect(report.ok).toBe(true) + }, 60_000) + + it("warns on what only breaks with every text 40% longer", async () => { + const report = await audit("longer-text", { longerText: true }) + expect(found(report)).toEqual([{ check: "cut-text", severity: "warning", pass: "longer-text", viewport: "phone", text: "Continue Cont", other: "" }]) + expect(report.ok).toBe(true) + }, 60_000) + + it("saves a screenshot with each finding numbered, and prints where", async () => { + const dir = mkdtempSync(join(tmpdir(), "ritmo-layout-shots-")) + const report = await audit("overlap", { screenshots: dir }) + expect(report.findings.map((finding) => finding.screenshot)).toEqual([ + { file: join(dir, "overlap@phone.png"), number: 1 }, + { file: join(dir, "overlap@phone.png"), number: 2 } + ]) + expect(existsSync(join(dir, "overlap@phone.png"))).toBe(true) + expect(LayoutWalk.format(report)).toContain(`(${join(dir, "overlap@phone.png")} #1)`) + }, 60_000) + + it("exits with 1 on an error and 0 on warnings alone", async () => { + const dir = mkdtempSync(join(tmpdir(), "ritmo-layout-cli-")) + const route = (fixture: string) => { + const file = join(dir, `${fixture}.json`) + writeFileSync(file, JSON.stringify({ screens: [{ name: fixture, path: `${fixture}.html` }], viewports: [phone] })) + return file + } + const failing = await run(["--url", base, "--route", route("small-target")], fixtures) + expect(failing.status).toBe(1) + expect(failing.stdout).toMatch(/^FAIL small-target \[small-target @ phone\] control "Close" at body > button: a tap reaches it over 20 x 20 px/) + const warned = await run(["--url", base, "--route", route("truncated-text"), "--json"], fixtures) + expect(warned.status).toBe(0) + expect(JSON.parse(warned.stdout)).toMatchObject({ ok: true, findings: [{ check: "truncated-text", severity: "warning" }] }) + }, 120_000) +}) diff --git a/packages/devtools/test/fixtures/layout/clean.html b/packages/devtools/test/fixtures/layout/clean.html new file mode 100644 index 00000000..f44855c0 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/clean.html @@ -0,0 +1,31 @@ + + + +A page with nothing broken + + +
+

Title

+
Card3
+ + +
+
+ +

Paused

diff --git a/packages/devtools/test/fixtures/layout/cut-text.html b/packages/devtools/test/fixtures/layout/cut-text.html new file mode 100644 index 00000000..2bc3ed67 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/cut-text.html @@ -0,0 +1,8 @@ + + + + +
The Gathering Storm
diff --git a/packages/devtools/test/fixtures/layout/layout-shift.html b/packages/devtools/test/fixtures/layout/layout-shift.html new file mode 100644 index 00000000..4552962a --- /dev/null +++ b/packages/devtools/test/fixtures/layout/layout-shift.html @@ -0,0 +1,9 @@ + + + + +

1 coins

+ diff --git a/packages/devtools/test/fixtures/layout/longer-text.html b/packages/devtools/test/fixtures/layout/longer-text.html new file mode 100644 index 00000000..764d18e6 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/longer-text.html @@ -0,0 +1,8 @@ + + + + + diff --git a/packages/devtools/test/fixtures/layout/never-settled.html b/packages/devtools/test/fixtures/layout/never-settled.html new file mode 100644 index 00000000..4d7dabf0 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/never-settled.html @@ -0,0 +1,10 @@ + + + + +

Drifting

+ diff --git a/packages/devtools/test/fixtures/layout/off-screen.html b/packages/devtools/test/fixtures/layout/off-screen.html new file mode 100644 index 00000000..f950483d --- /dev/null +++ b/packages/devtools/test/fixtures/layout/off-screen.html @@ -0,0 +1,8 @@ + + + + + diff --git a/packages/devtools/test/fixtures/layout/overlap.html b/packages/devtools/test/fixtures/layout/overlap.html new file mode 100644 index 00000000..e29dff72 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/overlap.html @@ -0,0 +1,14 @@ + + + + +

Score 1200

+

Lives 3

+ +Over it diff --git a/packages/devtools/test/fixtures/layout/safe-area.html b/packages/devtools/test/fixtures/layout/safe-area.html new file mode 100644 index 00000000..1c0af9a9 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/safe-area.html @@ -0,0 +1,10 @@ + + + + + + diff --git a/packages/devtools/test/fixtures/layout/scroll-edge.html b/packages/devtools/test/fixtures/layout/scroll-edge.html new file mode 100644 index 00000000..c2bfe442 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/scroll-edge.html @@ -0,0 +1,9 @@ + + + + +
diff --git a/packages/devtools/test/fixtures/layout/small-target.html b/packages/devtools/test/fixtures/layout/small-target.html new file mode 100644 index 00000000..21b123b7 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/small-target.html @@ -0,0 +1,8 @@ + + + + + diff --git a/packages/devtools/test/fixtures/layout/touch.html b/packages/devtools/test/fixtures/layout/touch.html new file mode 100644 index 00000000..77343b65 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/touch.html @@ -0,0 +1,18 @@ + + + + + +
Tip
+ diff --git a/packages/devtools/test/fixtures/layout/truncated-text.html b/packages/devtools/test/fixtures/layout/truncated-text.html new file mode 100644 index 00000000..28136122 --- /dev/null +++ b/packages/devtools/test/fixtures/layout/truncated-text.html @@ -0,0 +1,8 @@ + + + + +
Training Dummy
From bcd54d82f61fa55960c609ce34e6c9146ff2041d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 20:52:03 +0000 Subject: [PATCH 4/6] devtools: layout audit judges Touch buttons by their circle, skips covered layouts; docs and changesets A Touch layout under a menu takes no tap, so its buttons are not in play and are left out rather than reported as covered. Route clicks look in an open menu first, as a player would. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- .changeset/devtools-layout-audit.md | 7 ++ .changeset/touch-layout-reader.md | 5 + .changeset/ui-playbook-layout-check.md | 5 + .../skills/ritmo/playbooks/ui-and-render.md | 2 +- packages/devtools/README.md | 105 +++++++++++++++++- packages/devtools/src/internal/layoutPage.ts | 47 +++++--- packages/devtools/src/internal/pageOpen.ts | 8 +- 7 files changed, 157 insertions(+), 22 deletions(-) create mode 100644 .changeset/devtools-layout-audit.md create mode 100644 .changeset/touch-layout-reader.md create mode 100644 .changeset/ui-playbook-layout-check.md diff --git a/.changeset/devtools-layout-audit.md b/.changeset/devtools-layout-audit.md new file mode 100644 index 00000000..14c214e1 --- /dev/null +++ b/.changeset/devtools-layout-audit.md @@ -0,0 +1,7 @@ +--- +"@typeonce/ritmo-devtools": minor +--- + +`ritmo-devtools layout [--url ] [--route ] [--screenshots ] [--json]` opens each screen of a route in Chromium at several screen sizes (a small phone and an iPhone held sideways, an iPhone upright with its notch and home bar, and a desktop window, unless the route names its own) and reports what is broken in the layout the player sees. Errors fail the run: two things drawn over each other (`overlap`), text a box cuts with no ellipsis (`cut-text`), text or a control past the screen's edge (`off-screen`) or in the notch or home bar (`safe-area`), a tap area under 44 px (`small-target`, measured by hit tests, so a `::after` that extends it counts), something that moves while nothing is pressed (`layout-shift`), and a layout that never holds still (`never-settled`). Warnings never fail: text shortened with an ellipsis (`truncated-text`), a control half hidden at a scroll edge (`scroll-edge`), and what breaks only when every text is 40% longer. The buttons of a `Touch.actions` layout are checked too. A route's `allow` leaves out what the game means; `--screenshots` saves each screen with its findings boxed and numbered. + +The route file is shared with `ritmo-devtools keyboard`: it gains a `type` step (types text into the focused field), and `viewports`, `minTarget`, and `allow`, which only `layout` reads. A `click` step finds its control in the open menu first. diff --git a/.changeset/touch-layout-reader.md b/.changeset/touch-layout-reader.md new file mode 100644 index 00000000..59f00aa6 --- /dev/null +++ b/.changeset/touch-layout-reader.md @@ -0,0 +1,5 @@ +--- +"@typeonce/ritmo-browser": patch +--- + +While attached, `Touch.actions` leaves a function on its target that returns the layout at the target's current size, under `Symbol.for("@typeonce/ritmo-browser/Touch.layout")`, so `ritmo-devtools layout` can check the buttons the canvas draws. `dispose()` removes it. Nothing changes for a game. diff --git a/.changeset/ui-playbook-layout-check.md b/.changeset/ui-playbook-layout-check.md new file mode 100644 index 00000000..f1ad1ca6 --- /dev/null +++ b/.changeset/ui-playbook-layout-check.md @@ -0,0 +1,5 @@ +--- +"@typeonce/ritmo": patch +--- + +The `ritmo` skill's UI playbook (`ui-and-render.md`) checks the page with `ritmo-devtools layout` on the keyboard walk's route, at every screen size the game targets, and has the agent look at its screenshots. diff --git a/packages/core/skills/ritmo/playbooks/ui-and-render.md b/packages/core/skills/ritmo/playbooks/ui-and-render.md index 6d81e4f3..9248d8b7 100644 --- a/packages/core/skills/ritmo/playbooks/ui-and-render.md +++ b/packages/core/skills/ritmo/playbooks/ui-and-render.md @@ -13,4 +13,4 @@ 7. Build each DOM control (a button, a toggle, a slider) once and update its text and attributes in place; never rewrite a region that holds one (`innerHTML`, `replaceChildren`): the rewrite deletes the focused control and drops focus to ``. Give every control a name a test can find it by: its text, or `aria-label` on an icon. 8. A menu that stops play is a modal: `showModal()` on a ``, which moves focus in, keeps it there, and makes the page behind inert; an element with `aria-modal="true"` must do all three itself. The menu's own keys (arrows, letters) listen on that element. Handle a ``'s `cancel` and `close` events as intents and show it again from the game's state, because the browser closes a dialog on a repeated Escape even when `cancel` is prevented. A splash or wipe that must cover a menu is a `popover="manual"` shown after it, never a `z-index`; a layer the player presses (a crash panel) is its own modal dialog, since everything outside the newest modal is inert. -**Check:** the simulation's headless tests pass unchanged, and the game's check passes. When the page has DOM controls, `ritmo-devtools keyboard` passes with a route through every screen that shows them (the `@typeonce/ritmo-devtools` README). +**Check:** the simulation's headless tests pass unchanged, and the game's check passes. When the page has DOM controls, `ritmo-devtools keyboard` passes with a route through every screen that shows them, and `ritmo-devtools layout` passes on the same route at every screen size the game targets, with its screenshots looked at (the `@typeonce/ritmo-devtools` README). diff --git a/packages/devtools/README.md b/packages/devtools/README.md index c155029e..ae23ecd9 100644 --- a/packages/devtools/README.md +++ b/packages/devtools/README.md @@ -11,7 +11,8 @@ pnpm add -D @typeonce/ritmo-devtools The main import (`Session`, `Fuzz`, `Invariant`, `Replay`, `Lint`, ...) and `ritmo-devtools lint` need no peer dependency. Each optional peer is for one feature: `playwright` and `vite` for runs in browser engines -(`PlaywrightEngine`, `ritmo-devtools verify`, `ritmo-devtools keyboard`), `vite` for `ReplayVitePlugin`, +(`PlaywrightEngine`, `ritmo-devtools verify`, `ritmo-devtools keyboard`, +`ritmo-devtools layout`), `vite` for `ReplayVitePlugin`, and `rolldown` and `oxlint` for `ritmo-devtools scan`. The package reads the core `Debug` handle, which exists only on runtimes made @@ -588,8 +589,11 @@ Without `--route` the start page is one screen. A route is a JSON file: ``` - `path` is appended to the URL. `steps` run in order after the page - settles: `press` takes a key as Playwright names it, `click` a role and - the accessible name the report prints, `wait` milliseconds. + settles: `press` takes a key as Playwright names it, `type` types text + into the focused field, `click` a role and the accessible name the report + prints (in the open menu first), `wait` milliseconds. Each screen starts + from a fresh page with nothing saved, so a game that asks for a name + first starts its steps with `{ "click": { "role": "textbox", "name": "Name" } }, { "type": "Ada" }`. - `noPress` is `true` (press nothing on that screen) or a list of controls by name and optional role. - A step that cannot run, or an error the page throws, is a failure, and @@ -607,6 +611,101 @@ Without `--route` the start page is one screen. A route is a JSON file: tool, a dev server that did not start). `--json` prints `{ ok, url, screens, failures: [{ check, screen, control?, saw, fix }], outcomes: [{ check, screen, status, decided, undecided }] }`. +## Layout audit + +`ritmo-devtools layout` opens each screen of the same route in Chromium at +several screen sizes and fails on what is broken in the layout the player +sees, whatever the game. Run it after a change to anything the page shows +over the game (menus, HUD, touch controls): + +```sh +ritmo-devtools layout --route route.json --screenshots .layout +# FAIL overlap [home @ phone-landscape] text "Chapter 1" at .home__next > span.label, and control "Play" at .home__play > button.pill: they overlap by 120 x 14 px, and the control is drawn over the text. (.layout/home@phone-landscape.png #1) Fix: ... +# FAIL small-target [home @ phone-landscape] control "Settings" at .topbar__end > button.round: a tap reaches it over 40 x 50 px (at least 44 x 44). (.layout/home@phone-landscape.png #2) Fix: ... +# warn truncated-text [practice @ phone-landscape] text "Training Dummy" at button.card > span.card__foot: "Training Dummy" is shortened with an ellipsis: 52 of its 82 px show. (...) Fix: ... +# +# [home] +# small-phone-landscape: pass 9 more +# phone-landscape: FAIL overlap (1), FAIL small-target (1), pass 7 more +# ... +# layout audit of http://127.0.0.1:52011/: 2 errors, 1 warning on 4 screens at 4 viewports +``` + +It measures what the browser has already laid out, never a picture: each +line of visible text (the text itself, not the box around it), each +visible control (as the keyboard walk finds them), and each button of a +`Touch.actions` layout, which the canvas draws and the page cannot see +(`Touch.actions` leaves a reader on its target for this). Boxes are in +screen pixels after every transform, so a UI scaled as one piece is +measured as shown. Before measuring, it turns on reduced motion and waits +until the layout holds still. When an open `` or `aria-modal` +element owns the screen, only what is inside it is measured. + +Errors fail the run; each is a measurement a game never means: + +| check | fails when | +| --- | --- | +| `overlap` | two things are drawn over each other and neither is part of the other: text over text, text or a button over a button, the page over a Touch button, two Touch buttons. Hit tests at the shared area confirm both show there with nothing drawn above both, so a control that draws nothing (a touch area over the stage), something behind a scrim, and a badge inside its card are not overlaps | +| `cut-text` | the edge of a box cuts text, with no ellipsis to say text is missing | +| `off-screen` | part of a text or control is past the edge of the screen (a box that spans the screen cuts at the screen's edge, so that counts) | +| `safe-area` | part of a text or control is in the space the viewport's `insets` keep (a notch, the home bar), set through Chromium so `env(safe-area-inset-*)` reads them | +| `small-target` | the area that takes a tap is under `minTarget` either side, found by hit tests through the control's centre (a `::after` that extends it counts); a Touch button narrower than `minTarget` | +| `layout-shift` | something moved while nothing was pressed, from Chromium's own layout-shift entries over a still screen | +| `never-settled` | the layout kept moving, so nothing else was measured (a transform animation is motion, not layout, and settles) | + +Warnings are printed and never fail, because a game may mean them: + +| check | warns when | +| --- | --- | +| `truncated-text` | text is shortened with an ellipsis or a line clamp | +| `scroll-edge` | a control is half hidden at the edge of a scrolling area, or a scrolling area runs past the screen | + +Then each screen is loaded again with every text about 40% longer (each +text followed by the start of itself, so the width is the font's own), and +what breaks only then (`overlap`, `cut-text`, `off-screen`, `safe-area`) is +a warning marked "longer text": a layout with no room to spare for a longer +name or a translation. + +The route is the keyboard walk's, with three fields only the audit reads: + +```json +{ + "screens": [ + { "name": "home", "steps": [{ "click": { "role": "button", "name": "Play" } }], "allow": [{ "check": "overlap", "selector": ".card .badge" }] } + ], + "viewports": [ + { "name": "phone", "width": 844, "height": 390, "touch": true, "insets": { "left": 47, "right": 47, "bottom": 21 } }, + { "name": "desktop", "width": 1280, "height": 800 } + ], + "minTarget": 44, + "allow": [{ "check": "truncated-text", "selector": ".card__name" }] +} +``` + +- `viewports` default to a small phone held sideways (667 x 375), an + iPhone held sideways (844 x 390, notch and home bar) and upright + (390 x 844), and a desktop window (1280 x 800). A game played one way + only lists its own. +- `minTarget` is the smallest tap area in CSS pixels (44 by default). A UI + scaled to the screen is smaller on a small phone: set the game's own rule + as it comes out at the smallest size. +- `allow`, on the route or on a screen, leaves out a finding of `check` + whose element (or the other element of an overlap) matches `selector` or + is inside a match. The summary counts what it left out. +- `--screenshots ` saves each screen and viewport with findings as + `@.png` (and `-longer-text.png`), each finding boxed and + numbered as the report names it (red for errors, amber for warnings). +- It needs what `keyboard` needs, and serves the game the same way. Exit + codes: 0 no error (warnings allowed), 1 an error, a route step that + failed, a screen that never settled or showed nothing to measure, 2 usage + error. `--json` prints `{ ok, url, screens, viewports, findings: [{ check, + severity, screen, viewport, pass, element?, other?, box?, saw, fix, + screenshot? }], outcomes: [{ check, screen, viewport, status, count, + reason? }], allowed }`. + +What it cannot see: anything drawn on the canvas other than Touch buttons, +screens the route does not reach, and how Safari or Firefox lay out text. + ## Fuzzing `Fuzz.run` finds a bug before anyone plays into it: seeded random input, diff --git a/packages/devtools/src/internal/layoutPage.ts b/packages/devtools/src/internal/layoutPage.ts index b0639849..64435c3c 100644 --- a/packages/devtools/src/internal/layoutPage.ts +++ b/packages/devtools/src/internal/layoutPage.ts @@ -315,25 +315,38 @@ export const pageScript = String.raw`(() => { saw: "they overlap by " + Math.round(width(shared)) + " x " + Math.round(height(shared)) + " px, and the " + over.kind + " is drawn over the " + under.kind + (under.kind === "control" ? ", so a tap there reaches the wrong thing" : "") }) } - // A Touch button under something the page draws, or two Touch buttons that share space. - const touches = all.filter((item) => item.kind === "touch") - for (const touch of touches) { - const found = document.elementFromPoint(touch.centre.x, touch.centre.y) - if (found !== null && !related(touch.element, found) && found.id !== marksId && (painted(found) || textOf(found) !== "")) { - add("overlap", [touch.element, found], { element: describe(touch), other: { kind: "control", selector: selector(found), text: textOf(found).slice(0, 60) }, box: round(touch.rect), saw: "the page draws over the touch button " + JSON.stringify(touch.text) + " at its centre" }) - } - } - for (let i = 0; i < touches.length; i++) for (let j = i + 1; j < touches.length; j++) { - const a = touches[i], b = touches[j] - if (Math.hypot(a.centre.x - b.centre.x, a.centre.y - b.centre.y) < a.radius + b.radius - 2) { - add("overlap", [a.element], { element: describe(a), other: describe(b), box: round(meet(a.rect, b.rect)), saw: "touch buttons " + JSON.stringify(a.text) + " and " + JSON.stringify(b.text) + " share space, so a thumb between them presses either" }) - } - } } finally { force.remove() } } + // Touch buttons, with hit tests as a finger meets them: what takes a tap at each button's centre. + const touches = all.filter((item) => item.kind === "touch") + const taken = new Map(touches.map((touch) => { + const found = document.elementFromPoint(touch.centre.x, touch.centre.y) + return [touch, found === null || related(touch.element, found) ? null : found] + })) + for (const target of new Set(touches.map((touch) => touch.element))) { + const mine = touches.filter((touch) => touch.element === target) + // No button of the layout takes a tap: a screen covers the game (a menu, a pause screen), so none is in play. + if (mine.every((touch) => taken.get(touch) !== null)) { + for (const touch of mine) all.splice(all.indexOf(touch), 1) + continue + } + if (!checks.has("overlap")) continue + for (const touch of mine) { + const found = taken.get(touch) + if (found === null || found === undefined) continue + add("overlap", [touch.element, found], { element: describe(touch), other: { kind: "control", selector: selector(found), text: textOf(found).slice(0, 60) }, box: round(touch.rect), saw: "a tap at the centre of the touch button " + JSON.stringify(touch.text) + " reaches " + selector(found) + " instead" }) + } + for (let i = 0; i < mine.length; i++) for (let j = i + 1; j < mine.length; j++) { + const a = mine[i], b = mine[j] + if (Math.hypot(a.centre.x - b.centre.x, a.centre.y - b.centre.y) < a.radius + b.radius - 2) { + add("overlap", [a.element], { element: describe(a), other: describe(b), box: round(meet(a.rect, b.rect)), saw: "touch buttons " + JSON.stringify(a.text) + " and " + JSON.stringify(b.text) + " share space, so a thumb between them presses either" }) + } + } + } + // cut-text and truncated-text: text a clipping box hides part of. const cut = new Set() for (const item of all) { @@ -376,9 +389,11 @@ export const pageScript = String.raw`(() => { // A control's own box first: its text is judged only when the box is fine. for (const item of [...all].sort((a, b) => (a.kind === "text" ? 1 : 0) - (b.kind === "text" ? 1 : 0))) { if (item.kind === "touch") { - const out = item.rect.l < -1 || item.rect.t < -1 || item.rect.r > screenWidth + 1 || item.rect.b > screenHeight + 1 + // The button is a circle: how far it reaches past each edge, with the same 2 px allowance as a box. + const past = (edges) => Math.max(edges.l - (item.centre.x - item.radius), edges.t - (item.centre.y - item.radius), item.centre.x + item.radius - edges.r, item.centre.y + item.radius - edges.b) + const out = past(viewport) > 2 if (out && checks.has("off-screen")) add("off-screen", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " reaches past the edge of the screen" }) - else if (!out && checks.has("safe-area") && area(meet(item.rect, safe)) < area(item.rect) - 1) add("safe-area", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " reaches into the space the device keeps (a notch, a home bar)" }) + else if (!out && checks.has("safe-area") && past(safe) > 2) add("safe-area", [item.element], { element: describe(item), box: round(item.rect), saw: "the touch button " + JSON.stringify(item.text) + " reaches " + Math.round(past(safe)) + " px into the space the device keeps (a notch, a home bar)" }) continue } if (placed.has(item.owner) || area(item.visible) < 1) continue diff --git a/packages/devtools/src/internal/pageOpen.ts b/packages/devtools/src/internal/pageOpen.ts index 34481461..e87725fd 100644 --- a/packages/devtools/src/internal/pageOpen.ts +++ b/packages/devtools/src/internal/pageOpen.ts @@ -119,13 +119,17 @@ export const reach = async (opened: Opened, url: string, screen: Screen, settle: } /** - * The first visible element with this role and accessible name, by the names + * The first visible element with this role and accessible name (in the open + * menu first, then on the page), by the names * this report prints (Chromium's accessibility tree), so a name copied from a * report finds its control. Waits up to five seconds for it to show. */ export const find = async (opened: Opened, wanted: { readonly role: string; readonly name: string }): Promise => { for (let tries = 0; tries < 10; tries++) { - const candidates = await call(opened.page, "controls", null) as ReadonlyArray + // An open menu first: its button wins over one of the same name behind it (a confirm's "Skip" over the HUD's). + const menu = (await opened.state()).menu + const inMenu = menu === null ? [] : await call(opened.page, "controls", menu.id) as ReadonlyArray + const candidates = [...inMenu, ...await call(opened.page, "controls", null) as ReadonlyArray] for (const candidate of candidates) { const control = await opened.control(candidate) if (control.role === wanted.role && control.name === wanted.name) return candidate From d9b3b523d8882495f7539e0aa685a2883af1bc79 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 20:57:44 +0000 Subject: [PATCH 5/6] devtools: layout audit leaves out a control meant to fill the screen The website's town demo starts from a "Play" button that covers the whole page: it reached into the notch and lay under the other controls, which is what it is for. A control over the whole safe area is not judged by safe-area, and one under the whole of another is its backdrop. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- packages/devtools/src/internal/layoutPage.ts | 4 ++++ packages/devtools/test/LayoutWalk.test.ts | 2 +- packages/devtools/test/fixtures/layout/safe-area.html | 7 +++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/devtools/src/internal/layoutPage.ts b/packages/devtools/src/internal/layoutPage.ts index 64435c3c..1e296d16 100644 --- a/packages/devtools/src/internal/layoutPage.ts +++ b/packages/devtools/src/internal/layoutPage.ts @@ -308,6 +308,8 @@ export const pageScript = String.raw`(() => { reported.add(key) const over = seen.top === "a" ? a : b const under = seen.top === "a" ? b : a + // A control under the whole of the other is its backdrop ("tap anywhere", a clickable panel): what is on top takes its own taps. + if (under.kind === "control" && contains(under.visible, over.visible)) continue add("overlap", [a.owner, b.owner], { element: describe(under), other: describe(over), @@ -423,6 +425,8 @@ export const pageScript = String.raw`(() => { continue } if (item.scroller !== undefined || !checks.has("safe-area")) continue + // A control over the whole safe area ("tap anywhere to start", a scrim) is meant to fill the screen; its text is judged on its own. + if (item.kind === "control" && contains(onScreen, safe)) continue const inSafe = meet(onScreen, safe) if (area(inSafe) < area(onScreen) - 1) { placed.add(item.owner) diff --git a/packages/devtools/test/LayoutWalk.test.ts b/packages/devtools/test/LayoutWalk.test.ts index 2832de3b..f94955df 100644 --- a/packages/devtools/test/LayoutWalk.test.ts +++ b/packages/devtools/test/LayoutWalk.test.ts @@ -137,7 +137,7 @@ describe.skipIf(skipped !== undefined)("ritmo-devtools layout", () => { it("fails on a control past the screen's edge, and on one in the device's own space only where the device keeps it", async () => { expect(found(await audit("off-screen"))).toEqual([{ check: "off-screen", severity: "error", pass: "normal", viewport: "phone", text: "Shop", other: "" }]) - // At the left edge: inside the notch on the phone, fine on a phone without one; the button offset by env() is fine on both. + // At the left edge: inside the notch on the phone, fine on a phone without one; the button offset by env() and a full-screen "tap to start" are fine on both. expect(found(await audit("safe-area", { viewports: [phone, smallPhone] }))).toEqual([{ check: "safe-area", severity: "error", pass: "normal", viewport: "phone", text: "Menu", other: "" }]) }, 60_000) diff --git a/packages/devtools/test/fixtures/layout/safe-area.html b/packages/devtools/test/fixtures/layout/safe-area.html index 1c0af9a9..1e2408ef 100644 --- a/packages/devtools/test/fixtures/layout/safe-area.html +++ b/packages/devtools/test/fixtures/layout/safe-area.html @@ -5,6 +5,9 @@ body { margin: 0; font: 16px sans-serif; } .edge { position: absolute; left: 8px; top: 100px; width: 80px; height: 48px; } .padded { position: absolute; left: max(8px, env(safe-area-inset-left)); top: 200px; width: 80px; height: 48px; } + /* Tap anywhere to start: a control meant to fill the screen, its words inside the safe area. */ + .backdrop { position: fixed; inset: 0; border: 0; background: #234; color: #fff; font: inherit; } - - + + + From fdd06f1884144f1b1dc6d842c62a968b7bc8071a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 21:22:03 +0000 Subject: [PATCH 6/6] devtools: README shows a real layout audit's output and the backdrop rules Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HendNejwwEtfv3nfMtbBwn --- packages/devtools/README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/devtools/README.md b/packages/devtools/README.md index ae23ecd9..d7cb19d2 100644 --- a/packages/devtools/README.md +++ b/packages/devtools/README.md @@ -620,15 +620,16 @@ over the game (menus, HUD, touch controls): ```sh ritmo-devtools layout --route route.json --screenshots .layout -# FAIL overlap [home @ phone-landscape] text "Chapter 1" at .home__next > span.label, and control "Play" at .home__play > button.pill: they overlap by 120 x 14 px, and the control is drawn over the text. (.layout/home@phone-landscape.png #1) Fix: ... -# FAIL small-target [home @ phone-landscape] control "Settings" at .topbar__end > button.round: a tap reaches it over 40 x 50 px (at least 44 x 44). (.layout/home@phone-landscape.png #2) Fix: ... -# warn truncated-text [practice @ phone-landscape] text "Training Dummy" at button.card > span.card__foot: "Training Dummy" is shortened with an ellipsis: 52 of its 82 px show. (...) Fix: ... +# FAIL small-target [settings @ phone-landscape] control "Close" at div.screen.settings > header.topbar > span.topbar__end > button.round: a tap reaches it over 40 x 50 px (at least 42 x 42). (.layout/settings@phone-landscape.png #1) Fix: ... +# FAIL safe-area [arena @ phone-landscape] touch button "dash" on #game > canvas:nth-of-type(1): the touch button "dash" reaches 4 px into the space the device keeps (a notch, a home bar). (.layout/arena@phone-landscape.png #1) Fix: ... +# warn truncated-text [practice @ phone-landscape] text "Training Dummy" at div.cards > button.card:nth-of-type(1) > span.card__foot: "Training Dummy" is shortened with an ellipsis: 55 of its 82 px show. (.layout/practice@phone-landscape.png #1) Fix: ... +# warn safe-area [home @ phone-landscape, longer text] control "Play Pl" at div.home__play > button.pill:nth-of-type(2): with every text about 40% longer, it reaches into the space the device keeps on the right (47 px: a notch or the home bar). (...) Fix: ... # -# [home] -# small-phone-landscape: pass 9 more -# phone-landscape: FAIL overlap (1), FAIL small-target (1), pass 7 more +# [settings] +# small-phone-landscape: FAIL small-target (1), pass 8 more +# phone-landscape: FAIL small-target (1), pass 8 more # ... -# layout audit of http://127.0.0.1:52011/: 2 errors, 1 warning on 4 screens at 4 viewports +# layout audit of http://127.0.0.1:5199/: 55 errors, 31 warnings on 12 screens at 3 viewports ``` It measures what the browser has already laid out, never a picture: each @@ -645,10 +646,10 @@ Errors fail the run; each is a measurement a game never means: | check | fails when | | --- | --- | -| `overlap` | two things are drawn over each other and neither is part of the other: text over text, text or a button over a button, the page over a Touch button, two Touch buttons. Hit tests at the shared area confirm both show there with nothing drawn above both, so a control that draws nothing (a touch area over the stage), something behind a scrim, and a badge inside its card are not overlaps | +| `overlap` | two things are drawn over each other and neither is part of the other: text over text, text or a button over a button, the page over a Touch button, two Touch buttons. Hit tests at the shared area confirm both show there with nothing drawn above both, so a control that draws nothing (a touch area over the stage), something behind a scrim, a badge inside its card, and a button on a control under the whole of it ("tap anywhere", a clickable panel) are not overlaps | | `cut-text` | the edge of a box cuts text, with no ellipsis to say text is missing | | `off-screen` | part of a text or control is past the edge of the screen (a box that spans the screen cuts at the screen's edge, so that counts) | -| `safe-area` | part of a text or control is in the space the viewport's `insets` keep (a notch, the home bar), set through Chromium so `env(safe-area-inset-*)` reads them | +| `safe-area` | part of a text or control is in the space the viewport's `insets` keep (a notch, the home bar), set through Chromium so `env(safe-area-inset-*)` reads them. A control over the whole safe area ("tap anywhere to start") is meant to fill the screen, and only its text is judged | | `small-target` | the area that takes a tap is under `minTarget` either side, found by hit tests through the control's centre (a `::after` that extends it counts); a Touch button narrower than `minTarget` | | `layout-shift` | something moved while nothing was pressed, from Chromium's own layout-shift entries over a still screen | | `never-settled` | the layout kept moving, so nothing else was measured (a transform animation is motion, not layout, and settles) |