From 9ba0a801c015ccadbe6690e491e9fb65667fa51e Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Tue, 3 Nov 2026 15:26:33 +0530 Subject: [PATCH 1/4] fix(accessibility): hand back scans that outlast the wait, scan through the user's own Local tunnel, rename on a taken name, say why plan rules are off - startAccessibilityScan returns a scan that outlasts the 5-minute wait as still running, with its scan id, run id and dashboard link, instead of "Failed to start accessibility scan" with no ids. A cancelled run ends the wait, and the wait's timer is cleared. - startAccessibilityScan takes an optional localIdentifier: the scan goes through a BrowserStack Local tunnel the user started, and no tunnel is started or killed. The hosted refusal of local URLs says how to start one. - A scan name already in use is retried once with a UTC timestamp appended, and the result opens with the name used. - When advanced or experimental rules were not asked off but the run reports them off, the settings line says the plan most likely doesn't include them. - apiClient gives each of a host's addresses 2 s to connect instead of Node's 250 ms, and tries a request once more when its connection never opened. Co-Authored-By: Claude Opus 5.5 --- src/lib/apiClient.ts | 47 ++++- src/tools/accessibility.ts | 189 ++++++++++++++---- src/tools/accessiblity-utils/scanner.ts | 180 ++++++++++++----- tests/lib/apiClient-connect-retry.test.ts | 87 ++++++++ tests/tools/accessibilityScanOutput.test.ts | 29 +++ tests/tools/accessibilityScanSettings.test.ts | 142 ++++++++++++- .../accessibilityScanTunnelAndWait.test.ts | 141 +++++++++++++ 7 files changed, 720 insertions(+), 95 deletions(-) create mode 100644 tests/lib/apiClient-connect-retry.test.ts create mode 100644 tests/tools/accessibilityScanTunnelAndWait.test.ts diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 8698364d..92ccfb70 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -63,6 +63,40 @@ class ApiResponse { } } +/** + * How long a connection gives each of a host's addresses before trying the next. Node's own + * default is 250 ms: on a slow route the IPv4 attempt is dropped that early and the IPv6 one + * fails where there is no IPv6 route, so the call fails with "connect ETIMEDOUT ; + * connect ENETUNREACH " within a second. The hosted server saw exactly that. + */ +export const CONNECT_ATTEMPT_TIMEOUT_MS = 2000; + +// Shared, so connections are reused as they are on Node's own global agent. +const directAgent = new https.Agent({ + keepAlive: true, + autoSelectFamilyAttemptTimeout: CONNECT_ATTEMPT_TIMEOUT_MS, +}); + +/** Failures to open the connection at all: nothing was sent, so a request can be repeated. */ +const CONNECT_FAILURE = + /\bconnect (ETIMEDOUT|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED)\b/; +const CONNECT_FAILURE_CODES = new Set([ + "ETIMEDOUT", + "ENETUNREACH", + "EHOSTUNREACH", + "ECONNREFUSED", +]); + +export function isConnectFailure(error: any): boolean { + if (!error || error.response) return false; + const causes = [error, error.cause, ...(error.errors ?? [])].filter(Boolean); + return causes.some( + (cause: any) => + (cause.syscall === "connect" && CONNECT_FAILURE_CODES.has(cause.code)) || + CONNECT_FAILURE.test(String(cause.message ?? "")), + ); +} + // Utility to create HTTPS agent if needed (proxy/CA) function getAxiosAgent(): AxiosRequestConfig["httpsAgent"] | undefined { const proxyHost = config.browserstackLocalOptions.proxyHost; @@ -88,10 +122,11 @@ function getAxiosAgent(): AxiosRequestConfig["httpsAgent"] | undefined { // CA only return new https.Agent({ ca: fs.readFileSync(caCertPath), + autoSelectFamilyAttemptTimeout: CONNECT_ATTEMPT_TIMEOUT_MS, }); } // Default agent (no proxy, no CA) - return undefined; + return directAgent; } class ApiClient { @@ -148,7 +183,15 @@ class ApiClient { try { this.validateUrl(url, config); - const res = await fn(this.axiosAgent); + let res: AxiosResponse; + try { + res = await fn(this.axiosAgent); + } catch (error: any) { + // A connection that never opened sent nothing, so one more try is safe even for a + // POST; a second failure is reported as before. + if (!isConnectFailure(error)) throw error; + res = await fn(this.axiosAgent); + } return new ApiResponse(res); } catch (error: any) { if (error.response && !raise_error) { diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 2936bc04..a1dd75bc 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { AccessibilityScanner, + ScanTimeoutError, type ScanSettings, } from "./accessiblity-utils/scanner.js"; import { @@ -1249,6 +1250,47 @@ const WCAG_VERSIONS = { rgaa41: "RGAA 4.1", } as const; +/** + * Why advanced or experimental rules came back off when they were not asked off. Both are + * on unless turned off, and on an API scan a plan without them turns them off instead of + * refusing the scan, so the settings line alone reads as if the scan ran as asked. Asked + * off, or not read back, there is nothing to explain. + */ +function planDowngradeNote(asked: ScanSettings, applied: ScanSettings): string { + const off = [ + { + name: "advanced rules", + asked: asked.advancedRules, + ran: applied.advancedRules, + }, + { + name: "experimental rules", + asked: asked.experimentalRules, + ran: applied.experimentalRules, + }, + ].filter((rules) => rules.asked !== false && rules.ran === false); + if (!off.length) return ""; + const names = (wanted: boolean | undefined) => + off + .filter((rules) => rules.asked === wanted) + .map((rules) => rules.name) + .join(" and "); + const askedFor = names(true); + const byDefault = names(undefined); + const expected = [ + askedFor ? `${askedFor} were asked for` : "", + byDefault ? `${byDefault} are on by default` : "", + ] + .filter(Boolean) + .join(" and "); + return ( + `${expected[0].toUpperCase()}${expected.slice(1)}, but this scan ran without them, ` + + "most likely because this account's Accessibility plan does not include them: " + + "BrowserStack then runs the scan at the plan's level instead of refusing it. " + + "The issues found do not include those rules." + ); +} + /** * What the scan ran with, read back from the run, since an unset setting takes the API's * default and the plan can turn advanced rules off. When it cannot be read, only what was @@ -1280,7 +1322,12 @@ function scanSettingsNote( .filter(Boolean) .join(", "); if (applied && describe(applied)) { - return `Scan settings, as the scan ran: ${describe(applied)}.`; + return [ + `Scan settings, as the scan ran: ${describe(applied)}.`, + planDowngradeNote(asked, applied), + ] + .filter(Boolean) + .join(" "); } return describe(asked) ? `Scan settings asked for: ${describe(asked)}; anything not asked for takes the scan's default.` @@ -1292,6 +1339,7 @@ async function executeAccessibilityScan( name: string; pageURL: string; authConfigId?: number; + localIdentifier?: string; } & ScanSettings, context: ScanProgressContext, server: McpServer, @@ -1317,6 +1365,7 @@ async function executeAccessibilityScan( advancedRules: args.advancedRules, experimentalRules: args.experimentalRules, }, + args.localIdentifier, ); } catch (error) { return handleMCPError("startAccessibilityScan", server, config, error); @@ -1465,6 +1514,25 @@ function createScanFailureResponse( ); } +/** + * The wait ran out, not the scan: it carries on server-side and its issues can be read once + * it finishes. Not an error, so an agent comes back for the results instead of reporting a + * failure, as it did when this read "Failed to start accessibility scan" with no ids. + */ +function createScanStillRunningResponse( + name: string, + scanId: string, + scanRunId: string, + dashboardLink: string | undefined, + waitedMs: number, +): CallToolResult { + return createSuccessResponse([ + `Accessibility scan "${name}" is still running after ${Math.round(waitedMs / 60000)} minutes, so this call stopped waiting for it. The scan was not cancelled: it carries on, and its issues can be read once it finishes. ${scanDashboardNote(dashboardLink, scanId)}`, + `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, + `To read its issues, call fetchAccessibilityIssues with scanId "${scanId}" and scanRunId "${scanRunId}" once it has finished. Do not start the scan again: that starts a second scan.`, + ]); +} + function createScanSuccessResponse( name: string, report: Awaited>, @@ -1508,6 +1576,7 @@ async function runAccessibilityScan( config: BrowserStackConfig, authConfigId?: number, settings: ScanSettings = {}, + localIdentifier?: string, ): Promise { const scanner = await initializeScanner(config); @@ -1516,20 +1585,56 @@ async function runAccessibilityScan( [pageURL], authConfigId, settings, + localIdentifier, ); const scanId = startResp.data!.id; const scanRunId = startResp.data!.scanRunId; + const scanName = startResp.scanName ?? name; + // Said first, so the name asked for is not taken for the scan's. + const withRenameNote = (result: CallToolResult): CallToolResult => + scanName === name + ? result + : { + ...result, + content: [ + { + type: "text" as const, + text: `A scan named "${name}" already exists on this account, so this one was started as "${scanName}".\n`, + }, + ...result.content, + ], + }; - await notifyScanProgress(context, `Accessibility scan "${name}" started`, 0); + await notifyScanProgress( + context, + `Accessibility scan "${scanName}" started`, + 0, + ); - const status = await scanner.waitUntilComplete(scanId, scanRunId, context); + let status: string; + try { + status = await scanner.waitUntilComplete(scanId, scanRunId, context); + } catch (error) { + if (!(error instanceof ScanTimeoutError)) throw error; + return withRenameNote( + createScanStillRunningResponse( + scanName, + scanId, + scanRunId, + await scanner.dashboardLink(scanId), + error.waitedMs, + ), + ); + } if (status !== "completed") { - return createScanFailureResponse( - name, - status, - scanId, - scanRunId, - await scanner.dashboardLink(scanId), + return withRenameNote( + createScanFailureResponse( + scanName, + status, + scanId, + scanRunId, + await scanner.dashboardLink(scanId), + ), ); } @@ -1546,33 +1651,37 @@ async function runAccessibilityScan( ); } catch (error) { if (error instanceof ExportPendingError) { - return createSuccessResponse([ - `Accessibility scan "${name}" completed. ${scanDashboardNote(dashboardLink, scanId)}`, - `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, - scanSettingsNote(settings, applied), - ...exportPendingMessages( - { - kind: "scan", - scanId: String(scanId), - scanRunId: String(scanRunId), - }, - error, - "", - ), - ]); + return withRenameNote( + createSuccessResponse([ + `Accessibility scan "${scanName}" completed. ${scanDashboardNote(dashboardLink, scanId)}`, + `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, + scanSettingsNote(settings, applied), + ...exportPendingMessages( + { + kind: "scan", + scanId: String(scanId), + scanRunId: String(scanRunId), + }, + error, + "", + ), + ]), + ); } throw error; } - return createScanSuccessResponse( - name, - await parseAccessibilityReportFromCSV(reportLink), - scanId, - scanRunId, - reportLink, - dashboardLink, - settings, - applied, + return withRenameNote( + createScanSuccessResponse( + scanName, + await parseAccessibilityReportFromCSV(reportLink), + scanId, + scanRunId, + reportLink, + dashboardLink, + settings, + applied, + ), ); } @@ -1616,10 +1725,20 @@ export default function addAccessibilityTools( .number() .optional() .describe("Optional auth config ID for authenticated scans"), + localIdentifier: z + .string() + .trim() + .min(1) + .optional() + .describe( + "Identifier of a BrowserStack Local tunnel you started, for localhost or internal URLs.", + ), advancedRules: z .boolean() .optional() - .describe("Run the advanced accessibility rules (default true)."), + .describe( + "Run advanced rules (default true; off if the plan lacks them).", + ), wcagVersion: z .enum(Object.keys(WCAG_VERSIONS) as [keyof typeof WCAG_VERSIONS]) .optional() @@ -1639,7 +1758,9 @@ export default function addAccessibilityTools( experimentalRules: z .boolean() .optional() - .describe("Run the experimental accessibility rules (default true)."), + .describe( + "Run experimental rules (default true; off if the plan lacks them).", + ), }, { title: "Start Accessibility Scan", diff --git a/src/tools/accessiblity-utils/scanner.ts b/src/tools/accessiblity-utils/scanner.ts index 821b082f..54ccea6d 100644 --- a/src/tools/accessiblity-utils/scanner.ts +++ b/src/tools/accessiblity-utils/scanner.ts @@ -12,6 +12,75 @@ export interface AccessibilityScanResponse { success: boolean; data?: { id: string; scanRunId: string }; errors?: string[]; + /** The name the scan was started under, when the name asked for was already taken. */ + scanName?: string; +} + +/** How long startAccessibilityScan waits for a scan to finish before handing back its ids. */ +export const SCAN_WAIT_MS = 5 * 60 * 1000; + +/** Run statuses after which a scan does no more work. */ +const FINAL_SCAN_STATUSES = new Set(["completed", "failed", "cancelled"]); + +/** + * The wait ran out while the scan was still running. Only the wait ends: the scan carries + * on server-side, so the caller still has a scan to come back to. + */ +export class ScanTimeoutError extends Error { + constructor(public readonly waitedMs: number) { + super(`Scan still running after ${Math.round(waitedMs / 60000)} minutes`); + this.name = "ScanTimeoutError"; + } +} + +/** How to scan through a tunnel the user starts, which the scan API takes by identifier. */ +const OWN_TUNNEL_HINT = + "Start BrowserStack Local yourself (the BrowserStack Local app, or BrowserStackLocal --key --local-identifier ) and pass that name as localIdentifier."; + +/** The longest scan name the API takes. */ +const MAX_SCAN_NAME_LENGTH = 255; + +/** + * The name with the time appended, as " 2026-11-03 07:45:12 UTC". Only digits, + * "-", ":" and spaces are added, all of which the API allows in a name, and the name is + * shortened to keep the whole within the API's length limit. + */ +export function timestampedName(name: string, now: Date = new Date()): string { + const stamp = `${now.toISOString().slice(0, 19).replace("T", " ")} UTC`; + return `${name.slice(0, MAX_SCAN_NAME_LENGTH - stamp.length - 1).trimEnd()} ${stamp}`; +} + +/** The API's own reason for refusing a request, when its body gives one. */ +function refusalReason(err: any): string | undefined { + const data = err?.response?.data; + return ( + data?.error || + data?.message || + (Array.isArray(data?.errors) ? data.errors.join(", ") : undefined) + ); +} + +/** + * A 422 for a name already in use. A 422 also refuses a setting (a WCAG version the API + * does not take), so only one that says so, or gives no reason at all, is read this way. + */ +function nameIsTaken(err: any): boolean { + if (err?.response?.status !== 422) return false; + const reason = refusalReason(err); + return !reason || /already exists/i.test(String(reason)); +} + +function scanStartError(err: any): Error { + // apiClient throws generic errors, try to extract message. + const reason = refusalReason(err); + if (err?.response?.status === 422 && !reason) { + return new Error( + "A scan with this name already exists. please update the name and run again.", + ); + } + return new Error( + `Failed to start scan: ${reason || err?.message || String(err)}`, + ); } export interface AccessibilityScanStatus { @@ -49,6 +118,7 @@ export class AccessibilityScanner { urlList: string[], authConfigId?: number, settings: ScanSettings = {}, + ownTunnel?: string, ): Promise { if (!this.auth?.username || !this.auth?.password) { throw new Error( @@ -57,30 +127,36 @@ export class AccessibilityScanner { } // Check if any URL is local const hasLocal = urlList.some(isLocalURL); - const localIdentifier = randomUUID(); + const localIdentifier = ownTunnel ?? randomUUID(); const localHosts = new Set(["127.0.0.1", "localhost", "0.0.0.0"]); const BS_LOCAL_DOMAIN = "bs-local.com"; - if (config.USE_OWN_LOCAL_BINARY_PROCESS && hasLocal) { - throw new Error( - "Cannot start scan with local URLs when using own BrowserStack Local binary process. Please set USE_OWN_LOCAL_BINARY_PROCESS to false.", - ); - } + // A tunnel the user started is theirs to run: none is started here, and none is killed, + // as killing every BrowserStackLocal process would take theirs down with it. + if (!ownTunnel) { + if (config.USE_OWN_LOCAL_BINARY_PROCESS && hasLocal) { + throw new Error( + `Cannot start scan with local URLs when using own BrowserStack Local binary process. ${OWN_TUNNEL_HINT}`, + ); + } - if (config.REMOTE_MCP && hasLocal) { - throw new Error( - "Local URLs are not supported in this remote mcp. Please use a public URL.", - ); - } + // The hosted server cannot start a tunnel on the user's machine, but the scan API + // takes the identifier of one the user started. + if (config.REMOTE_MCP && hasLocal) { + throw new Error( + `This hosted BrowserStack MCP runs on BrowserStack's servers, so it cannot open a tunnel to a local URL on your machine. ${OWN_TUNNEL_HINT} Or scan a public URL.`, + ); + } - if (hasLocal) { - await ensureLocalBinarySetup( - this.auth.username, - this.auth.password, - localIdentifier, - ); - } else { - await killExistingBrowserStackLocalProcesses(); + if (hasLocal) { + await ensureLocalBinarySetup( + this.auth.username, + this.auth.password, + localIdentifier, + ); + } else { + await killExistingBrowserStackLocalProcesses(); + } } const transformedUrlList = urlList.map((url) => { @@ -109,7 +185,7 @@ export class AccessibilityScanner { }; let requestBody = baseRequestBody; - if (hasLocal) { + if (hasLocal || ownTunnel) { const localConfig = { localTestingInfo: { localIdentifier, @@ -119,18 +195,17 @@ export class AccessibilityScanner { requestBody = { ...baseRequestBody, ...localConfig }; } - try { + const { username, password } = this.auth; + const create = async (body: typeof requestBody) => { const response = await apiClient.post({ url: "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans", headers: { Authorization: "Basic " + - Buffer.from(`${this.auth.username}:${this.auth.password}`).toString( - "base64", - ), + Buffer.from(`${username}:${password}`).toString("base64"), "Content-Type": "application/json", }, - body: requestBody, + body, }); const data = response.data; if (!data.success) @@ -138,22 +213,24 @@ export class AccessibilityScanner { `Unable to start scan: ${data.errors?.join(", ") || JSON.stringify(data)}`, ); return data; + }; + + try { + return await create(requestBody); } catch (err: any) { - // apiClient throws generic errors, try to extract message. A 422 also refuses a - // setting (a WCAG version the API does not take), so its own reason comes first. - const data = err?.response?.data; - const reason = - data?.error || - data?.message || - (Array.isArray(data?.errors) ? data.errors.join(", ") : undefined); - if (err?.response?.status === 422 && !reason) { - throw new Error( - "A scan with this name already exists. please update the name and run again.", - ); - } - throw new Error( - `Failed to start scan: ${reason || err?.message || String(err)}`, - ); + if (!nameIsTaken(err)) throw scanStartError(err); + } + // Scan names are unique per account, so an agent re-scanning a page under the name it + // used before is refused. Starting it once more under a timestamped name is what the + // user would do by hand. + const scanName = timestampedName(name); + try { + return { + ...(await create({ ...requestBody, name: scanName })), + scanName, + }; + } catch (err: any) { + throw scanStartError(err); } } @@ -277,10 +354,10 @@ export class AccessibilityScanner { const progress = status === "completed" ? 100 : timepercent; const dots = ".".repeat(dotCount); dotCount = (dotCount % 4) + 1; - const message = - status === "completed" || status === "failed" - ? `Scan completed with status: ${status}` - : `Scan in progress${dots}`; + const finished = FINAL_SCAN_STATUSES.has(status); + const message = finished + ? `Scan completed with status: ${status}` + : `Scan in progress${dots}`; await context.sendNotification({ method: "notifications/progress", params: { @@ -290,23 +367,22 @@ export class AccessibilityScanner { total: 100, }, }); - if (status === "completed" || status === "failed") { + if (finished) { clearInterval(interval); + clearTimeout(timeout); resolve(status); } } catch (e) { clearInterval(interval); + clearTimeout(timeout); reject(e); } }, 5000); - setTimeout( - () => { - clearInterval(interval); - reject(new Error("Scan timed out after 5 minutes")); - }, - 5 * 60 * 1000, - ); + const timeout = setTimeout(() => { + clearInterval(interval); + reject(new ScanTimeoutError(SCAN_WAIT_MS)); + }, SCAN_WAIT_MS); }); } } diff --git a/tests/lib/apiClient-connect-retry.test.ts b/tests/lib/apiClient-connect-retry.test.ts new file mode 100644 index 00000000..456fac2a --- /dev/null +++ b/tests/lib/apiClient-connect-retry.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// The hosted server failed calls with "connect ETIMEDOUT ; connect ENETUNREACH " +// within a second: Node gives each address 250 ms by default. A connection that never +// opened sent nothing, so it is tried once more; anything that got a response is not. + +const m = vi.hoisted(() => ({ get: vi.fn(), post: vi.fn() })); + +vi.mock("axios", () => ({ + default: { create: () => ({ get: m.get, post: m.post }) }, +})); +vi.mock("../../src/config.js", () => ({ + default: { browserstackLocalOptions: {} }, +})); +vi.mock("../../src/lib/utils.js", () => ({ isDataUrlPayloadTooLarge: () => false })); + +function connectFailure() { + // The shape Node's dual-stack connect gives once every address has failed. + const ipv4 = Object.assign(new Error("connect ETIMEDOUT 146.75.37.242:443"), { + code: "ETIMEDOUT", + syscall: "connect", + }); + const ipv6 = Object.assign(new Error("connect ENETUNREACH 2a04:4e42:79::498:443"), { + code: "ENETUNREACH", + syscall: "connect", + }); + return Object.assign(new AggregateError([ipv4, ipv6], ""), { code: "ETIMEDOUT" }); +} + +async function client() { + return (await import("../../src/lib/apiClient.js")).apiClient; +} + +describe("apiClient connect failures", () => { + beforeEach(() => { + vi.resetModules(); + m.get.mockReset(); + m.post.mockReset(); + }); + + it("tries a POST once more when its connection never opened", async () => { + m.post + .mockRejectedValueOnce(connectFailure()) + .mockResolvedValueOnce({ status: 200, data: { ok: true } }); + + const res = await (await client()).post({ url: "https://api.example/scans", body: { a: 1 } }); + + expect(res.data).toEqual({ ok: true }); + expect(m.post).toHaveBeenCalledTimes(2); + }); + + it("reports the failure when the second try fails too", async () => { + m.get.mockRejectedValue(connectFailure()); + + await expect((await client()).get({ url: "https://api.example/status" })).rejects.toThrow(); + expect(m.get).toHaveBeenCalledTimes(2); + }); + + it("does not repeat a request that got a response, or failed after connecting", async () => { + m.post.mockRejectedValueOnce( + Object.assign(new Error("Request failed with status code 500"), { + response: { status: 500, data: {} }, + }), + ); + await expect((await client()).post({ url: "https://api.example/scans" })).rejects.toThrow(); + expect(m.post).toHaveBeenCalledTimes(1); + + m.post.mockReset(); + m.post.mockRejectedValueOnce( + Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }), + ); + await expect((await client()).post({ url: "https://api.example/scans" })).rejects.toThrow(); + expect(m.post).toHaveBeenCalledTimes(1); + }); + + it("reads a connect failure from its code, a wrapped cause, or the message", async () => { + const { isConnectFailure } = await import("../../src/lib/apiClient.js"); + + expect(isConnectFailure(connectFailure())).toBe(true); + expect(isConnectFailure({ cause: { code: "ECONNREFUSED", syscall: "connect" } })).toBe(true); + expect( + isConnectFailure({ message: "connect ETIMEDOUT 140.248.149.242:443; connect ENETUNREACH 2a04:4e42:77::498:443 - Local (:::0)" }), + ).toBe(true); + expect(isConnectFailure({ code: "ETIMEDOUT", syscall: "read" })).toBe(false); + expect(isConnectFailure({ code: "ECONNABORTED", message: "timeout of 15000ms exceeded" })).toBe(false); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 0a8084c1..a8483a10 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -25,6 +25,11 @@ vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ runSettings = vi.fn().mockResolvedValue(undefined); dashboardLink = m.dashboardLink; }, + ScanTimeoutError: class extends Error { + constructor(readonly waitedMs: number) { + super("still running"); + } + }, })); vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ AccessibilityReportFetcher: class { @@ -991,4 +996,28 @@ describe("what a web report's answer says about its issues", () => { expect(body).toContain("run-7"); expect(body).not.toContain("scan-details"); }); + + // The wait ran out, not the scan. This used to read "Failed to start accessibility scan" + // with no ids, and an agent told the user no scan had been created. + it("hands back a scan that outlasts the wait as still running, with its ids", async () => { + const LINK = "https://accessibility.browserstack.com/site-scanner/projects/Home%20page/builds?thBuildId=abc"; + const { ScanTimeoutError } = await import("../../src/tools/accessiblity-utils/scanner.js"); + m.waitUntilComplete.mockRejectedValue(new ScanTimeoutError(5 * 60 * 1000)); + m.dashboardLink.mockResolvedValue(LINK); + + const result = await handlers.startAccessibilityScan( + { name: "Home page", pageURL: "https://site.example" }, + CONTEXT, + ); + + expect(result.isError).toBeFalsy(); + const body = text(result); + expect(body).toContain('Accessibility scan "Home page" is still running after 5 minutes'); + expect(body).toContain("The scan was not cancelled"); + expect(body).toContain("Scan ID: scan-42 and Scan Run ID: run-7"); + expect(body).toContain('call fetchAccessibilityIssues with scanId "scan-42" and scanRunId "run-7"'); + expect(body).toContain(LINK); + expect(body).not.toContain("Failed to start"); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); }); diff --git a/tests/tools/accessibilityScanSettings.test.ts b/tests/tools/accessibilityScanSettings.test.ts index d72c4273..a510a4fe 100644 --- a/tests/tools/accessibilityScanSettings.test.ts +++ b/tests/tools/accessibilityScanSettings.test.ts @@ -87,6 +87,49 @@ describe("AccessibilityScanner.startScan", () => { ).rejects.toThrow(/A scan with this name already exists/); }); + // Scan names are unique per account, and agents re-scanning a page reuse the name they + // used before: the API refused those scans outright. + it("starts the scan under a timestamped name when the name is taken", async () => { + m.post + .mockRejectedValueOnce({ + response: { + status: 422, + data: { message: "A scan with the provided name already exists in this group/project." }, + }, + }) + .mockResolvedValueOnce({ data: { success: true, data: { id: "1", scanRunId: "2" } } }); + + const started = await (await scanner()).startScan("checkout", ["https://shop.example/"]); + + expect(m.post).toHaveBeenCalledTimes(2); + const renamed = m.post.mock.calls[1][0].body.name; + expect(renamed).toMatch(/^checkout \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} UTC$/); + expect(started).toMatchObject({ data: { id: "1", scanRunId: "2" }, scanName: renamed }); + }); + + it("does not rename a scan the API refused for another reason", async () => { + m.post.mockRejectedValue({ + response: { status: 422, data: { message: "wcagVersion field must be one of: wcag21aa" } }, + }); + + await expect( + (await scanner()).startScan("checkout", ["https://shop.example/"]), + ).rejects.toThrow(/wcagVersion field must be one of/); + expect(m.post).toHaveBeenCalledTimes(1); + }); + + it("keeps a timestamped name within the API's 255 characters and its allowed characters", async () => { + const { timestampedName } = await vi.importActual< + typeof import("../../src/tools/accessiblity-utils/scanner.js") + >("../../src/tools/accessiblity-utils/scanner.js"); + const now = new Date("2026-11-03T07:45:12.345Z"); + + expect(timestampedName("checkout", now)).toBe("checkout 2026-11-03 07:45:12 UTC"); + const long = timestampedName("x".repeat(300), now); + expect(long).toHaveLength(255); + expect(long.endsWith(" 2026-11-03 07:45:12 UTC")).toBe(true); + }); + it("sends experimental rules off when asked, as false is a setting rather than a missing one", async () => { await (await scanner()).startScan("checkout", ["https://shop.example/"], undefined, { experimentalRules: false, @@ -226,10 +269,13 @@ describe("startAccessibilityScan scan settings", () => { bestPractices: true, }); - expect(m.startScan).toHaveBeenCalledWith("checkout", ["https://shop.example/"], undefined, { - wcagVersion: "wcag22aaa", - bestPractices: true, - }); + expect(m.startScan).toHaveBeenCalledWith( + "checkout", + ["https://shop.example/"], + undefined, + { wcagVersion: "wcag22aaa", bestPractices: true }, + undefined, + ); expect(m.runSettings).toHaveBeenCalledWith("7310042", "5520318"); // Advanced rules read back off, as a plan without them turns them off. expect(text(result)).toContain( @@ -237,6 +283,84 @@ describe("startAccessibilityScan scan settings", () => { ); }); + // On an API scan a plan without advanced or experimental rules turns them off instead of + // refusing the scan, and the run's settings were the only sign of it: an agent that asked + // for advanced rules read "advanced rules off" with no reason given. + it("says why advanced rules asked for came back off", async () => { + m.runSettings.mockResolvedValue({ + wcagVersion: "wcag22aa", + advancedRules: false, + }); + const { run } = await tool(); + + const result = await run({ + name: "checkout", + pageURL: "https://shop.example/", + wcagVersion: "wcag22aa", + advancedRules: true, + }); + + expect(text(result)).toContain( + "Scan settings, as the scan ran: WCAG 2.2 AA, advanced rules off. " + + "Advanced rules were asked for, but this scan ran without them, most likely because " + + "this account's Accessibility plan does not include them: BrowserStack then runs the " + + "scan at the plan's level instead of refusing it. The issues found do not include those rules.", + ); + }); + + it("says why rules on by default came back off, apart from the ones asked for", async () => { + m.runSettings.mockResolvedValue({ advancedRules: false, experimentalRules: false }); + const { run } = await tool(); + + const asked = text(await run({ name: "checkout", pageURL: "https://shop.example/", advancedRules: true })); + const unset = text(await run({ name: "checkout", pageURL: "https://shop.example/" })); + + expect(asked).toContain( + "Advanced rules were asked for and experimental rules are on by default, but this scan ran without them", + ); + expect(unset).toContain( + "Advanced rules and experimental rules are on by default, but this scan ran without them", + ); + }); + + it("leads with the name the scan was started under when the one asked for was taken", async () => { + m.startScan.mockResolvedValue({ + data: { id: "7310042", scanRunId: "5520318" }, + scanName: "checkout 2026-11-03 07:45:12 UTC", + }); + m.runSettings.mockResolvedValue(undefined); + const { run } = await tool(); + + const result = await run({ name: "checkout", pageURL: "https://shop.example/" }); + + expect(result.content[0].text).toBe( + 'A scan named "checkout" already exists on this account, so this one was started as "checkout 2026-11-03 07:45:12 UTC".\n', + ); + expect(text(result)).toContain('Accessibility scan "checkout 2026-11-03 07:45:12 UTC" completed.'); + }); + + it("gives no reason when the rules were asked off, ran on, or could not be read", async () => { + const { run } = await tool(); + + m.runSettings.mockResolvedValue({ advancedRules: false, experimentalRules: false }); + const askedOff = text( + await run({ + name: "checkout", + pageURL: "https://shop.example/", + advancedRules: false, + experimentalRules: false, + }), + ); + m.runSettings.mockResolvedValue({ advancedRules: true, experimentalRules: true }); + const ranOn = text(await run({ name: "checkout", pageURL: "https://shop.example/", advancedRules: true })); + m.runSettings.mockResolvedValue(undefined); + const unread = text(await run({ name: "checkout", pageURL: "https://shop.example/", advancedRules: true })); + + for (const result of [askedOff, ranOn, unread]) { + expect(result).not.toContain("Accessibility plan"); + } + }); + it("passes experimental rules to the scan and reports them as the scan ran", async () => { m.runSettings.mockResolvedValue({ wcagVersion: "wcag21aa", @@ -253,9 +377,13 @@ describe("startAccessibilityScan scan settings", () => { experimentalRules: false, }); - expect(m.startScan).toHaveBeenCalledWith("checkout", ["https://shop.example/"], undefined, { - experimentalRules: false, - }); + expect(m.startScan).toHaveBeenCalledWith( + "checkout", + ["https://shop.example/"], + undefined, + { experimentalRules: false }, + undefined, + ); expect(text(result)).toContain( "Scan settings, as the scan ran: WCAG 2.1 AA, best practices off, needs review on, advanced rules on, experimental rules off.", ); diff --git a/tests/tools/accessibilityScanTunnelAndWait.test.ts b/tests/tools/accessibilityScanTunnelAndWait.test.ts new file mode 100644 index 00000000..8d9fa4b3 --- /dev/null +++ b/tests/tools/accessibilityScanTunnelAndWait.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// A scan of a local URL goes through a BrowserStack Local tunnel. The hosted server cannot +// start one on the user's machine, but the scan API takes the identifier of a tunnel the +// user started. And a scan outlasting the 5-minute wait keeps running: the tool used to say +// "Failed to start accessibility scan" with no ids, so agents reported no scan existed. + +const m = vi.hoisted(() => ({ + post: vi.fn(), + get: vi.fn(), + ensureLocalBinarySetup: vi.fn(), + killExistingBrowserStackLocalProcesses: vi.fn(), + config: { REMOTE_MCP: false, USE_OWN_LOCAL_BINARY_PROCESS: false }, +})); + +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post, get: m.get } })); +vi.mock("../../src/lib/local.js", () => ({ + isLocalURL: (url: string) => new URL(url).hostname === "localhost", + ensureLocalBinarySetup: m.ensureLocalBinarySetup, + killExistingBrowserStackLocalProcesses: m.killExistingBrowserStackLocalProcesses, +})); +vi.mock("../../src/config.js", () => ({ default: m.config })); +vi.mock("../../src/logger.js", () => ({ + default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); + +async function scannerModule() { + return vi.importActual( + "../../src/tools/accessiblity-utils/scanner.js", + ); +} + +async function scanner() { + const { AccessibilityScanner } = await scannerModule(); + const s = new AccessibilityScanner(); + s.setAuth({ username: "u", password: "p" }); + return s; +} + +describe("AccessibilityScanner.startScan through a tunnel the user started", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + m.config.REMOTE_MCP = false; + m.config.USE_OWN_LOCAL_BINARY_PROCESS = false; + m.post.mockResolvedValue({ data: { success: true, data: { id: "1", scanRunId: "2" } } }); + }); + + it("sends the user's tunnel and starts or kills no tunnel of its own", async () => { + m.config.REMOTE_MCP = true; + + await (await scanner()).startScan("dev", ["http://localhost:3000/cart"], undefined, {}, "my-tunnel"); + + const body = m.post.mock.calls[0][0].body; + expect(body.localTestingInfo).toEqual({ localIdentifier: "my-tunnel", localEnabled: true }); + expect(body.urlList).toEqual(["http://bs-local.com:3000/cart"]); + expect(m.ensureLocalBinarySetup).not.toHaveBeenCalled(); + expect(m.killExistingBrowserStackLocalProcesses).not.toHaveBeenCalled(); + }); + + it("sends an internal host through the user's tunnel unchanged", async () => { + await (await scanner()).startScan("staging", ["https://staging.internal.example/"], undefined, {}, "my-tunnel"); + + const body = m.post.mock.calls[0][0].body; + expect(body.urlList).toEqual(["https://staging.internal.example/"]); + expect(body.localTestingInfo).toEqual({ localIdentifier: "my-tunnel", localEnabled: true }); + expect(m.killExistingBrowserStackLocalProcesses).not.toHaveBeenCalled(); + }); + + it("on the hosted server, refuses a local URL without a tunnel and says how to start one", async () => { + m.config.REMOTE_MCP = true; + + const start = (await scanner()).startScan("dev", ["http://localhost:3000/"]); + + await expect(start).rejects.toThrow(/cannot open a tunnel to a local URL on your machine/); + await expect(start).rejects.toThrow(/--local-identifier \) and pass that name as localIdentifier/); + expect(m.post).not.toHaveBeenCalled(); + }); + + it("with USE_OWN_LOCAL_BINARY_PROCESS, takes a local URL once the tunnel's identifier is given", async () => { + m.config.USE_OWN_LOCAL_BINARY_PROCESS = true; + + await expect( + (await scanner()).startScan("dev", ["http://localhost:3000/"]), + ).rejects.toThrow(/pass that name as localIdentifier/); + await (await scanner()).startScan("dev", ["http://localhost:3000/"], undefined, {}, "mine"); + + expect(m.post.mock.calls[0][0].body.localTestingInfo.localIdentifier).toBe("mine"); + }); + + it("without a tunnel, still starts its own for a local URL on a local server", async () => { + await (await scanner()).startScan("dev", ["http://localhost:3000/"]); + + const identifier = m.post.mock.calls[0][0].body.localTestingInfo.localIdentifier; + expect(m.ensureLocalBinarySetup).toHaveBeenCalledWith("u", "p", identifier); + }); +}); + +describe("AccessibilityScanner.waitUntilComplete", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + vi.useFakeTimers(); + }); + afterEach(() => vi.useRealTimers()); + + const context = { sendNotification: vi.fn(), _meta: {} }; + const status = (value: string) => ({ data: { success: true, data: { status: value } } }); + + it("hands back a ScanTimeoutError when the scan outlasts the wait", async () => { + m.get.mockResolvedValue(status("processing")); + const { ScanTimeoutError, SCAN_WAIT_MS } = await scannerModule(); + + const wait = (await scanner()).waitUntilComplete("1", "2", context); + const settled = wait.catch((e: unknown) => e); + await vi.advanceTimersByTimeAsync(SCAN_WAIT_MS); + + const error = await settled; + expect(error).toBeInstanceOf(ScanTimeoutError); + expect((error as InstanceType).waitedMs).toBe(SCAN_WAIT_MS); + }); + + it("stops at a cancelled scan instead of waiting out the 5 minutes", async () => { + m.get.mockResolvedValue(status("cancelled")); + + const wait = (await scanner()).waitUntilComplete("1", "2", context); + await vi.advanceTimersByTimeAsync(5000); + + await expect(wait).resolves.toBe("cancelled"); + }); + + it("clears the 5-minute timer once the scan finishes", async () => { + m.get.mockResolvedValue(status("completed")); + + const wait = (await scanner()).waitUntilComplete("1", "2", context); + await vi.advanceTimersByTimeAsync(5000); + + await expect(wait).resolves.toBe("completed"); + expect(vi.getTimerCount()).toBe(0); + }); +}); From c24017df10befdc47a13a478acffc9beeec3277a Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Tue, 3 Nov 2026 15:43:13 +0530 Subject: [PATCH 2/4] fix(accessibility): clean scan names the API would refuse, and say why a scan failed - startAccessibilityScan replaces characters central-scanner refuses in a scan name ("/", "|", "&", ",", ")" and more, and "--") instead of failing the scan. Accents are dropped, parentheses become square brackets, and a name with nothing usable left falls back to the page's host. The result says the name was changed and what it was started as. - A failed scan now says why: the run's own reason from the scans list, and each page that did not scan with its reason from the run's scan logs, up to 10 pages. Without either, it still points to the dashboard. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 52 +++++- src/tools/accessiblity-utils/scanner.ts | 129 +++++++++++++- .../accessibilityScanNamesAndFailures.test.ts | 162 ++++++++++++++++++ tests/tools/accessibilityScanOutput.test.ts | 55 ++++++ tests/tools/accessibilityScanSettings.test.ts | 18 +- 5 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 tests/tools/accessibilityScanNamesAndFailures.test.ts diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index a1dd75bc..d32aacf4 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -4,6 +4,7 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { AccessibilityScanner, ScanTimeoutError, + type ScanFailureDetails, type ScanSettings, } from "./accessiblity-utils/scanner.js"; import { @@ -1502,15 +1503,55 @@ function scanDashboardNote( : `See it under Website Scanner in the Accessibility dashboard (scan ID ${scanId}).`; } +/** Why the scan's name differs from the one asked for, as the start of a sentence. */ +function renameReason( + name: string, + changes: ("characters" | "taken")[], +): string { + const characters = changes.includes("characters"); + const taken = changes.includes("taken"); + if (characters && taken) + return `The name "${name}" has characters scan names can't contain, and the cleaned name was already taken`; + if (characters) + return `The name "${name}" has characters scan names can't contain`; + return `A scan named "${name}" already exists on this account`; +} + +/** How many failed pages a failure result lists before summing up the rest. */ +const FAILED_PAGES_LISTED = 10; + +/** + * A failed run, with the reason the API gives for it: the run's own and each page's. It used + * to say only "The dashboard says why", which left an agent nothing to tell the user. + */ function createScanFailureResponse( name: string, status: string, scanId: string, scanRunId: string, dashboardLink: string | undefined, + details: ScanFailureDetails = { pages: [] }, ): CallToolResult { + const { reason, pages } = details; + const listed = pages.slice(0, FAILED_PAGES_LISTED); + const why = [ + reason ? `Why: ${reason}${/[.!?]$/.test(reason) ? "" : "."}` : "", + pages.length + ? [ + `${pages.length} ${pages.length === 1 ? "page" : "pages"} could not be scanned:`, + ...listed.map((page) => `- ${page.pageUrl}: ${page.reason}`), + ...(pages.length > listed.length + ? [`- and ${pages.length - listed.length} more.`] + : []), + ].join("\n") + : "", + ].filter(Boolean); return createErrorResponse( - `❌ Accessibility scan "${name}" failed with status: ${status}. Scan ID: ${scanId} and Scan Run ID: ${scanRunId}. The dashboard says why. ${scanDashboardNote(dashboardLink, scanId)}`, + [ + `❌ Accessibility scan "${name}" failed with status: ${status}. Scan ID: ${scanId} and Scan Run ID: ${scanRunId}.`, + ...(why.length ? why : ["The dashboard says why."]), + scanDashboardNote(dashboardLink, scanId), + ].join("\n"), ); } @@ -1599,7 +1640,7 @@ async function runAccessibilityScan( content: [ { type: "text" as const, - text: `A scan named "${name}" already exists on this account, so this one was started as "${scanName}".\n`, + text: `${renameReason(name, startResp.nameChanges ?? ["taken"])}, so this scan was started as "${scanName}".\n`, }, ...result.content, ], @@ -1627,13 +1668,18 @@ async function runAccessibilityScan( ); } if (status !== "completed") { + const [dashboardLink, details] = await Promise.all([ + scanner.dashboardLink(scanId), + scanner.failureDetails(scanId, scanRunId), + ]); return withRenameNote( createScanFailureResponse( scanName, status, scanId, scanRunId, - await scanner.dashboardLink(scanId), + dashboardLink, + details, ), ); } diff --git a/src/tools/accessiblity-utils/scanner.ts b/src/tools/accessiblity-utils/scanner.ts index 54ccea6d..b5b24855 100644 --- a/src/tools/accessiblity-utils/scanner.ts +++ b/src/tools/accessiblity-utils/scanner.ts @@ -12,8 +12,16 @@ export interface AccessibilityScanResponse { success: boolean; data?: { id: string; scanRunId: string }; errors?: string[]; - /** The name the scan was started under, when the name asked for was already taken. */ + /** The name the scan was started under, when it differs from the name asked for. */ scanName?: string; + /** Why it differs: characters a scan name cannot hold, and/or a name already taken. */ + nameChanges?: ("characters" | "taken")[]; +} + +/** Why a run failed, as far as the API says: the run's own reason and each page's log. */ +export interface ScanFailureDetails { + reason?: string; + pages: { pageUrl: string; status: string; reason: string }[]; } /** How long startAccessibilityScan waits for a scan to finish before handing back its ids. */ @@ -40,6 +48,46 @@ const OWN_TUNNEL_HINT = /** The longest scan name the API takes. */ const MAX_SCAN_NAME_LENGTH = 255; +/** + * One character central-scanner takes in a scan name (lib/util.rb validate_project_name), + * which also refuses "--". It takes "(" but not ")", so both become square brackets. + */ +const NAME_CHARACTER = /[A-Za-z0-9_:*.@!$ '{}[\]-]/; + +/** + * The name with every character central-scanner refuses replaced, rather than the scan + * refused: agents name scans after page titles and paths, with "/", "|", "&" or ",". Accents + * are dropped first ("Café" becomes "Cafe"), and a run of replaced characters becomes one + * "-", or " - " where it stood between words. A name left with no letters or digits, as a + * name in another script is, falls back to the given one. + */ +export function cleanScanName(name: string, fallback: string): string { + const cleaned = Array.from( + name + .normalize("NFD") + .replace(/[̀-ͯ]/g, "") + .replace(/\(/g, "[") + .replace(/\)/g, "]") + .replace(/\s+/g, " "), + ) + .map((ch) => (NAME_CHARACTER.test(ch) ? ch : "-")) + .join("") + .replace(/[\s-]*-[\s-]*/g, (run) => (/\s/.test(run) ? " - " : "-")) + .replace(/^[\s-]+|[\s-]+$/g, "") + .slice(0, MAX_SCAN_NAME_LENGTH) + .replace(/[\s-]+$/, ""); + return /[A-Za-z0-9]/.test(cleaned) ? cleaned : fallback; +} + +/** A scan name to fall back on: the first page's host, which is always a valid name. */ +function hostName(urlList: string[]): string { + try { + return new URL(urlList[0]).hostname || "Accessibility scan"; + } catch { + return "Accessibility scan"; + } +} + /** * The name with the time appended, as " 2026-11-03 07:45:12 UTC". Only digits, * "-", ":" and spaces are added, all of which the API allows in a name, and the name is @@ -176,8 +224,9 @@ export class AccessibilityScanner { const scanSettings = Object.fromEntries( Object.entries(settings).filter(([, value]) => value !== undefined), ); + const cleanName = cleanScanName(name, hostName(urlList)); const baseRequestBody = { - name, + name: cleanName, urlList: transformedUrlList, recurring: false, ...(authConfigId && { authConfigId }), @@ -215,19 +264,24 @@ export class AccessibilityScanner { return data; }; + const cleaned = cleanName !== name; try { - return await create(requestBody); + const data = await create(requestBody); + return cleaned + ? { ...data, scanName: cleanName, nameChanges: ["characters"] } + : data; } catch (err: any) { if (!nameIsTaken(err)) throw scanStartError(err); } // Scan names are unique per account, so an agent re-scanning a page under the name it // used before is refused. Starting it once more under a timestamped name is what the // user would do by hand. - const scanName = timestampedName(name); + const scanName = timestampedName(cleanName); try { return { ...(await create({ ...requestBody, name: scanName })), scanName, + nameChanges: cleaned ? ["characters", "taken"] : ["taken"], }; } catch (err: any) { throw scanStartError(err); @@ -313,6 +367,73 @@ export class AccessibilityScanner { return typeof link === "string" && link ? link : undefined; } + /** + * Why a run failed. The run's own reason is only on the scans list (lastScanDetails.error, + * such as "The requested page is not reachable"), so it is taken only when the list's last + * run is this one; each page's reason is in the run's scan logs ("[HTTP 403] Forbidden", + * "Failed to start the test"). Either can be missing, and a read that fails gives nothing + * rather than an error on top of the failed scan. + */ + async failureDetails( + scanId: string, + scanRunId: string, + ): Promise { + const read = (url: string) => + apiClient + .get({ + url, + headers: { + Authorization: + "Basic " + + Buffer.from( + `${this.auth?.username}:${this.auth?.password}`, + ).toString("base64"), + }, + raise_error: false, + }) + .catch(() => undefined); + const base = + "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans"; + const [list, logs] = await Promise.all([ + read(base), + read(`${base}/${scanId}/scan_runs/${scanRunId}/scan_logs`), + ]); + const last = ( + list?.ok + ? ((list.data?.data?.scans ?? []) as any[]).find( + (scan) => String(scan?.id) === String(scanId), + ) + : undefined + )?.lastScanDetails; + const reason = + last && + String(last.scanRunId) === String(scanRunId) && + typeof last.error === "string" && + last.error.trim() + ? last.error.trim() + : undefined; + const entries: any[] = logs?.ok ? (logs.data?.data?.scanLogs ?? []) : []; + const pages = entries + .filter((entry) => entry && entry.status !== "success") + .map((entry) => { + const said = [entry.description, entry.error] + .filter((part) => typeof part === "string" && part.trim()) + .map((part: string) => part.trim()); + const redirect = + typeof entry.finalRedirectUrl === "string" && entry.finalRedirectUrl + ? `redirected to ${entry.finalRedirectUrl}` + : ""; + return { + pageUrl: String(entry.pageUrl ?? ""), + status: String(entry.status ?? ""), + reason: + [...new Set(said), redirect].filter(Boolean).join("; ") || + String(entry.status ?? "unknown"), + }; + }); + return { ...(reason ? { reason } : {}), pages }; + } + async pollStatus( scanId: string, scanRunId: string, diff --git a/tests/tools/accessibilityScanNamesAndFailures.test.ts b/tests/tools/accessibilityScanNamesAndFailures.test.ts new file mode 100644 index 00000000..8a130db9 --- /dev/null +++ b/tests/tools/accessibilityScanNamesAndFailures.test.ts @@ -0,0 +1,162 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Scan names: central-scanner refuses any character outside a short list (and "--"), and +// agents name scans after page titles and paths. 21 scans in a week failed on that. +// Failed runs: the result said only "The dashboard says why". + +const m = vi.hoisted(() => ({ post: vi.fn(), get: vi.fn() })); + +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post, get: m.get } })); +vi.mock("../../src/lib/local.js", () => ({ + isLocalURL: () => false, + ensureLocalBinarySetup: vi.fn(), + killExistingBrowserStackLocalProcesses: vi.fn(), +})); +vi.mock("../../src/config.js", () => ({ + default: { REMOTE_MCP: false, USE_OWN_LOCAL_BINARY_PROCESS: false }, +})); +vi.mock("../../src/logger.js", () => ({ + default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); + +async function scannerModule() { + return vi.importActual( + "../../src/tools/accessiblity-utils/scanner.js", + ); +} + +async function scanner() { + const { AccessibilityScanner } = await scannerModule(); + const s = new AccessibilityScanner(); + s.setAuth({ username: "u", password: "p" }); + return s; +} + +// The rule central-scanner applies (lib/util.rb validate_project_name). +const ALLOWED = /^[A-Za-z0-9_:*.@!$\s\-'{}[\](]*$/; + +describe("cleanScanName", () => { + it.each([ + ["Checkout / Cart | Prod", "Checkout - Cart - Prod"], + ["Home (v2)", "Home [v2]"], + ["Café, menu & more", "Cafe - menu - more"], + ["docs/getting-started", "docs-getting-started"], + ["release--candidate", "release-candidate"], + ["50% off; today?", "50 - off - today"], + ['Say "hi"', "Say - hi"], + [" Checkout page\tone ", "Checkout page one"], + ["Already-valid name_1.2:*@!$ {x} [y]", "Already-valid name_1.2:*@!$ {x} [y]"], + ])("cleans %j to %j", async (name, cleaned) => { + const { cleanScanName } = await scannerModule(); + expect(cleanScanName(name, "site.example")).toBe(cleaned); + }); + + it("falls back to the given name when nothing usable is left", async () => { + const { cleanScanName } = await scannerModule(); + expect(cleanScanName("日本語のページ", "www.example.jp")).toBe("www.example.jp"); + expect(cleanScanName("/// | ///", "site.example")).toBe("site.example"); + }); + + it("only ever gives a name the API takes, within 255 characters", async () => { + const { cleanScanName } = await scannerModule(); + for (const name of [ + "a".repeat(400), + "x/".repeat(200), + "Ünïcödé — test (draft) #3 bold `code` \\path\\ \"q\" 'a' & | ; > < ? % , \n end", + ]) { + const cleaned = cleanScanName(name, "site.example"); + expect(cleaned).toMatch(ALLOWED); + expect(cleaned).not.toContain("--"); + expect(cleaned.length).toBeLessThanOrEqual(255); + expect(cleaned).toMatch(/[A-Za-z0-9]/); + } + }); +}); + +describe("AccessibilityScanner.startScan names", () => { + beforeEach(() => { + vi.resetModules(); + m.post.mockReset(); + m.post.mockResolvedValue({ data: { success: true, data: { id: "1", scanRunId: "2" } } }); + }); + + it("sends a cleaned name and says the name changed", async () => { + const started = await (await scanner()).startScan("Checkout / Cart", ["https://shop.example/"]); + + expect(m.post.mock.calls[0][0].body.name).toBe("Checkout - Cart"); + expect(started).toMatchObject({ scanName: "Checkout - Cart", nameChanges: ["characters"] }); + }); + + it("sends a valid name unchanged and reports no change", async () => { + const started = await (await scanner()).startScan("checkout", ["https://shop.example/"]); + + expect(m.post.mock.calls[0][0].body.name).toBe("checkout"); + expect(started).not.toHaveProperty("scanName"); + expect(started).not.toHaveProperty("nameChanges"); + }); + + it("timestamps the cleaned name when it is taken too", async () => { + m.post + .mockRejectedValueOnce({ response: { status: 422, data: { message: "A scan with the provided name already exists" } } }) + .mockResolvedValueOnce({ data: { success: true, data: { id: "1", scanRunId: "2" } } }); + + const started = await (await scanner()).startScan("Checkout / Cart", ["https://shop.example/"]); + + const renamed = m.post.mock.calls[1][0].body.name; + expect(renamed).toMatch(/^Checkout - Cart \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} UTC$/); + expect(started).toMatchObject({ scanName: renamed, nameChanges: ["characters", "taken"] }); + }); +}); + +describe("AccessibilityScanner.failureDetails", () => { + beforeEach(() => { + vi.resetModules(); + m.get.mockReset(); + }); + + const ok = (data: unknown) => ({ ok: true, data: { success: true, data } }); + const list = (lastScanDetails: unknown) => + ok({ scans: [{ id: 7, lastScanDetails }, { id: 8, lastScanDetails: { scanRunId: 2, error: "other scan" } }] }); + const logs = ok({ + urlCount: 3, + scanLogs: [ + { pageUrl: "https://site.example/", description: "Scan complete", status: "success", error: null }, + { pageUrl: "https://site.example/a", description: "[HTTP 403] Forbidden", status: "failure", error: null }, + { pageUrl: "https://site.example/b", description: "Redirected", status: "redirect", error: "too many hops", finalRedirectUrl: "https://site.example/c" }, + ], + }); + function route(listResp: unknown, logsResp: unknown) { + m.get.mockImplementation(async ({ url }: { url: string }) => + url.endsWith("/scan_logs") ? logsResp : listResp, + ); + } + + it("gives the run's reason and each page that did not scan", async () => { + route(list({ scanRunId: 2, error: "The requested page is not reachable" }), logs); + + const details = await (await scanner()).failureDetails("7", "2"); + + expect(details.reason).toBe("The requested page is not reachable"); + expect(details.pages).toEqual([ + { pageUrl: "https://site.example/a", status: "failure", reason: "[HTTP 403] Forbidden" }, + { pageUrl: "https://site.example/b", status: "redirect", reason: "Redirected; too many hops; redirected to https://site.example/c" }, + ]); + expect(m.get.mock.calls.map(([o]) => o.url)).toContain( + "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/7/scan_runs/2/scan_logs", + ); + }); + + it("takes no reason from another run of the scan", async () => { + route(list({ scanRunId: 1, error: "an older run's reason" }), logs); + + expect((await (await scanner()).failureDetails("7", "2")).reason).toBeUndefined(); + }); + + it("gives nothing, rather than an error, when the reads fail", async () => { + m.get.mockRejectedValue(new Error("network down")); + await expect((await scanner()).failureDetails("7", "2")).resolves.toEqual({ pages: [] }); + + route({ ok: false, data: {} }, { ok: false, data: {} }); + await expect((await scanner()).failureDetails("7", "2")).resolves.toEqual({ pages: [] }); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index a8483a10..bc1b0f83 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -15,6 +15,7 @@ const m = vi.hoisted(() => ({ appReportIssues: vi.fn(), parse: vi.fn(), dashboardLink: vi.fn(), + failureDetails: vi.fn(async () => ({ pages: [] })), })); vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ @@ -24,6 +25,7 @@ vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ waitUntilComplete = m.waitUntilComplete; runSettings = vi.fn().mockResolvedValue(undefined); dashboardLink = m.dashboardLink; + failureDetails = m.failureDetails; }, ScanTimeoutError: class extends Error { constructor(readonly waitedMs: number) { @@ -995,6 +997,59 @@ describe("what a web report's answer says about its issues", () => { expect(body).toContain("scan-42"); expect(body).toContain("run-7"); expect(body).not.toContain("scan-details"); + // With no reason from the API, the dashboard is still where to look. + expect(body).toContain("The dashboard says why."); + }); + + // A failed run used to say only "The dashboard says why", which left an agent nothing to + // tell the user. The run's reason and each page's now come with it. + it("says why a scan failed, for the run and for each page", async () => { + m.waitUntilComplete.mockResolvedValue("failed"); + m.dashboardLink.mockResolvedValue(undefined); + m.failureDetails.mockResolvedValueOnce({ + reason: "The requested page is not reachable. Please ensure the domain is accessible", + pages: [ + { pageUrl: "https://site.example/", status: "failure", reason: "[HTTP 403] Forbidden" }, + { pageUrl: "https://site.example/old", status: "redirect", reason: "Redirected; redirected to https://site.example/new" }, + ], + }); + + const result = await handlers.startAccessibilityScan( + { name: "Home page", pageURL: "https://site.example" }, + CONTEXT, + ); + + expect(result.isError).toBe(true); + expect(m.failureDetails).toHaveBeenCalledWith("scan-42", "run-7"); + const body = text(result); + expect(body).toContain( + "Why: The requested page is not reachable. Please ensure the domain is accessible.", + ); + expect(body).toContain("2 pages could not be scanned:\n- https://site.example/: [HTTP 403] Forbidden\n- https://site.example/old: Redirected; redirected to https://site.example/new"); + expect(body).not.toContain("The dashboard says why"); + }); + + it("lists at most 10 failed pages and counts the rest", async () => { + m.waitUntilComplete.mockResolvedValue("failed"); + m.failureDetails.mockResolvedValueOnce({ + pages: Array.from({ length: 13 }, (_, i) => ({ + pageUrl: `https://site.example/${i}`, + status: "failure", + reason: "Failed to start the test", + })), + }); + + const body = text( + await handlers.startAccessibilityScan( + { name: "Home page", pageURL: "https://site.example" }, + CONTEXT, + ), + ); + + expect(body).toContain("13 pages could not be scanned:"); + expect(body).toContain("- https://site.example/9: Failed to start the test"); + expect(body).not.toContain("https://site.example/10:"); + expect(body).toContain("- and 3 more."); }); // The wait ran out, not the scan. This used to read "Failed to start accessibility scan" diff --git a/tests/tools/accessibilityScanSettings.test.ts b/tests/tools/accessibilityScanSettings.test.ts index a510a4fe..e6cb5773 100644 --- a/tests/tools/accessibilityScanSettings.test.ts +++ b/tests/tools/accessibilityScanSettings.test.ts @@ -334,11 +334,27 @@ describe("startAccessibilityScan scan settings", () => { const result = await run({ name: "checkout", pageURL: "https://shop.example/" }); expect(result.content[0].text).toBe( - 'A scan named "checkout" already exists on this account, so this one was started as "checkout 2026-11-03 07:45:12 UTC".\n', + 'A scan named "checkout" already exists on this account, so this scan was started as "checkout 2026-11-03 07:45:12 UTC".\n', ); expect(text(result)).toContain('Accessibility scan "checkout 2026-11-03 07:45:12 UTC" completed.'); }); + it("says the name was cleaned when it had characters scan names can't contain", async () => { + m.startScan.mockResolvedValue({ + data: { id: "7310042", scanRunId: "5520318" }, + scanName: "Checkout - Cart", + nameChanges: ["characters"], + }); + m.runSettings.mockResolvedValue(undefined); + const { run } = await tool(); + + const result = await run({ name: "Checkout / Cart", pageURL: "https://shop.example/" }); + + expect(result.content[0].text).toBe( + 'The name "Checkout / Cart" has characters scan names can\'t contain, so this scan was started as "Checkout - Cart".\n', + ); + }); + it("gives no reason when the rules were asked off, ran on, or could not be read", async () => { const { run } = await tool(); From dd1edcdb7ed9bbe63d915d6fb9912627c76ad169 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Fri, 6 Nov 2026 12:21:34 +0530 Subject: [PATCH 3/4] fix(api-client): retry a connect failure only when it failed fast A failure after every address has timed out means the host is down. Retrying it doubled the wait, which loops over many calls (scan-list walks, polling) would pay on every call. Only a failure within 5 s is tried again now. Co-Authored-By: Claude Opus 5.5 --- src/lib/apiClient.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/lib/apiClient.ts b/src/lib/apiClient.ts index 92ccfb70..f247bce9 100644 --- a/src/lib/apiClient.ts +++ b/src/lib/apiClient.ts @@ -77,6 +77,13 @@ const directAgent = new https.Agent({ autoSelectFamilyAttemptTimeout: CONNECT_ATTEMPT_TIMEOUT_MS, }); +/** + * A connect failure is retried only when it came within this long. A slower one means every + * address already timed out, so the host is down: failing now keeps loops (scan-list walks, + * polling) from stalling twice as long on it. + */ +export const CONNECT_RETRY_WINDOW_MS = 5000; + /** Failures to open the connection at all: nothing was sent, so a request can be repeated. */ const CONNECT_FAILURE = /\bconnect (ETIMEDOUT|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED)\b/; @@ -184,12 +191,17 @@ class ApiClient { this.validateUrl(url, config); let res: AxiosResponse; + const started = Date.now(); try { res = await fn(this.axiosAgent); } catch (error: any) { // A connection that never opened sent nothing, so one more try is safe even for a - // POST; a second failure is reported as before. - if (!isConnectFailure(error)) throw error; + // POST; a second failure is reported as before. Only a fast failure is retried. + if ( + !isConnectFailure(error) || + Date.now() - started >= CONNECT_RETRY_WINDOW_MS + ) + throw error; res = await fn(this.axiosAgent); } return new ApiResponse(res); From 9a7c595bf2988cd8c10844db197b2ec5ddc04f1d Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary <76112295+Aaryan430@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:17:39 +0530 Subject: [PATCH 4/4] =?UTF-8?q?fix(accessibility):=20dogfooding=20fixes=20?= =?UTF-8?q?from=203=20Nov=20=E2=80=94=20failed=20runs,=20build=20compare,?= =?UTF-8?q?=20re-scan,=20Local=20tunnels,=20report=20links=20(#553)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(accessibility): say when a run did not check its pages, instead of reading it as clean fetchAccessibilityIssues read runs that checked nothing as clean or complete reports: - A failed Website Scanner run exported an empty report and read as "0 issues ✅ All issues retrieved"; a failed Automated Tests build (the API's status for one that completed with no test cases) threw "Failed to fetch report: null". A failed scan run, build or Workflow Analyzer report is now an error that says so, with the scan's reasons from its logs, and no export is started. An app build's "failed" is its tests' result, so it is read as before. Comparing scan runs refuses a failed run, whose empty report read every issue of the other run as resolved or new. - A page that did not load had an error page (Chrome's, or BrowserStack Local's privoxy page) scanned in its place, and its issues came back as the page's. The report parser now sets such pages apart, as the comparison already did, keeping their rows out of the issues, totals, summary and duplicates; the read (and startAccessibilityScan) leads with a note. - A completed scan's pages that failed to scan were simply absent. Its scan logs are read with the export and the first page lists them (redirects aside, as those pages were scanned where they redirected to). With pages skipped, the last page no longer claims every issue. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): quote a scan's failure reasons and page lines as untrusted data The reasons a run or a page failed, and the pages a completed run could not scan, come from the scan logs and the site: the page's address, its HTTP status, where it redirected. They are now quoted in an untrusted-data block, as the report's issues and page addresses are, with the sentences around them ("N pages were not scanned …:", "Fix the cause and scan again.") left outside it. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): steer compareWith on reports with no compare to reading both compareWith on a Workflow Analyzer report, an Automated Tests build or a manual app report answered with an error ("... is neither"), and agents spent a call working out that they should read both reports and compare them by hand. The answer now says so directly, as guidance rather than an error: read both reports with fetchAccessibilityIssues and match issues by rule and CSS selector on each page (by rule and component for app reports); an issue in the later report only is new, in the earlier only is fixed, in both is still open. compareWith naming a report that is not an app build, for an app build, gets the same guidance. The compareWith description now says which reports it compares (Scanner runs of one scan, app builds) and to read both for any other. Co-Authored-By: Claude Opus 5.5 * feat(accessibility): compare Automated Tests builds with compareWith compareWith on an Automated Tests (web) build answered "can't be compared here yet" and then, since 6a777ac, told the agent to read both builds and compare them by hand. In MCP dogfooding (3 Nov) 3 of 3 agents passed correct testhub ids and got no compare. compareWith on such a build (buildId, or its report link as reportUrl) now takes another build's id, its report link, or "previous", and compares them: - The older build, by start time, is "before" and the newer "after", whichever order they were passed in. When compareWith is the newer one the answer says so in one line. Both builds are named (name, project, #number, id, start), the names quoted as data. - Two builds of one build name and project are compared by the Automated Tests compare API (GET /api/automated-tests/v1/builds/compare/issues, newReport = after, baseReport = before). It takes the builds in the order given and does not check it. "previous" calls it without baseReport, and the answer names the build the API picked. Its issues are keyed by the API's own signatureKey, and the answer says it matches by that, not by rule, element and page alone. The API told apart a css-orientation-lock issue on before and after a fix added lang to it, and listed it as both fixed and new. With a rule|element|page key on both, agents recounted the dashboard's 6/13/1 to 4/11/3. - Any other pair, or a pair the API refuses (issue-count threshold, "TestRun Not Found") or does not answer (504 after its 30 s, or no answer within 40 s), is compared by matching the two builds' issue exports by rule, element and page, as Scanner runs are. The answer says which method was used, in one line. - Either way the answer gives three lists with counts: new (only in after), fixed (only in before), still open (in both). Each list shows at most 50 issues and says how many more. For the API, a list cut short on its first page (100 rows) is read on its own. - Issues found on a BrowserStack Local or Chrome network error page are set apart as "page did not load". The diff also catches a page whose error-page issues are all duplicates of another page's, as a grouped build's are; before, that page's earlier issues read as fixed. - A build the build list calls "failed" (completed with no test cases) has no accessibility results: the answer says so as guidance and does not compare. "previous" with no earlier completed run of the build's name is said plainly, with no request to report it. The compareWith description now names web builds (21,495 of the 21,500 budget). Co-Authored-By: Claude Opus 5.5 * feat(accessibility): re-run an earlier scan of the same page instead of starting a new one startAccessibilityScan always created a new scan, so re-scanning a page after a fix made a second scan whose runs could not be compared with the first (fetchAccessibilityIssues compareWith failed with "Scan logs not found"). - New optional scanId. When that scan is for this same page (compared as startScan saves URLs: local hosts as bs-local.com, host case and a trailing slash ignored), start a new run of it (POST /scans/:id/new_scan) with its saved settings, wait for it like a new scan, and end with how to compare with compareWith "previous". Without scanId a new scan is created as before. - scanId for another page (or several pages): a new scan, saying why. - Settings passed that differ from the scan's saved ones: no run; each difference and the two ways forward are listed. Only settings the call passed are compared; a login or tunnel can only be checked for being saved at all, which is said. - A re-run keeps the scan's name (a different name passed is mentioned), and a Local scan's re-run says the tunnel must run with the saved identifier. - Re-run refusals as plain results: a run still going (with its run id), a deleted login config, an unknown scan, the plan's message. - Every new scan says how to scan the same page again with its scanId. - The 50-scan plan limit on create is said plainly instead of "open an issue on GitHub". - Shorter wording for a few accessibility argument descriptions, to stay within the description budget with the new argument. Co-Authored-By: Claude Opus 5.5 * fix(local): never kill BrowserStack Local tunnels this MCP did not start; local-URL limits as plain answers Every public-URL accessibility scan and Live session ran `pkill -f BrowserStackLocal` (`taskkill /IM BrowserStackLocal.exe` on Windows), as did starting a tunnel for a local URL, so a tunnel the user had up (their own, or their test suite's) was killed by an unrelated call. - lib/local.ts no longer kills any process by name. A public URL takes no tunnel action at all. - A local-URL scan without localIdentifier runs through this MCP's own tunnel, started through browserstack-local under a unique mcp- identifier, kept in module state and reused while it runs. It is replaced (the old one stopped through its own handle) only once it has died or the credentials change, and stopped by its own pid when the process exits or is signalled. The binary runs tunnels with different --local-identifier values side by side, so the user's keeps running; `force` is dropped from the configured options. Should ours fail to start while another Local is running, the error says to pass that tunnel's name as localIdentifier. - Live sessions (the other caller) start their unidentified tunnel only when no other BrowserStack Local is running, and otherwise use the one that is, leaving it alone. - A local URL the MCP cannot tunnel to (the hosted MCP, or USE_OWN_LOCAL_BINARY_PROCESS) is answered as a plain result with #541's guidance (start BrowserStack Local and pass its name as localIdentifier, or scan a public URL), with no GitHub or submitFeedback line. - The accessibility tools' error wrapper adds a full stop only when the message lacks one, so it no longer writes "..". Co-Authored-By: Claude Opus 5.5 * fix(accessibility): search 25 pages of scans for a report link, not 15 A Website Scanner dashboard link is resolved by walking the scan list newest first. At the list's default 20 per page, 15 pages reached ~300 scans, so a link to an older scan came back "not found". 25 pages reach ~500, at about 1 s a page (~26 s worst case). Stopgap until the issues API reads by thBuildId (accessibility #5689). Co-Authored-By: Claude Opus 5.5 * fix(accessibility): no 5-minute wait on a finalizing report; account refusals as plain answers; expert questions within 500 chars fetchAccessibilityIssues waited up to five minutes inside the call for a report still finalizing (or an app build still running its rules). Clients cut off a call that long: the claude.ai connector aborts a tool call silent for 305 s, progress notifications or not, so the agent got nothing, and one reported a fix as verified that had added 26 issues. - A first read of a report that is "finalizing" or "rule_processing" now answers at once. It is not an error: it says no issues were read yet, so the page is not to be called clean nor a fix verified, and gives the exact call to make again in a minute or two (pageUrl and compareWith kept), or to call the report stuck after about 10 minutes. The wait loop and REPORT_STATUS_POLL_MS / REPORT_STATUS_WAIT_MS are gone. A later page, or a report whose tests are still running, is read as before with its "still processing" note. - A cancelled Website Scanner run is answered like a failed one (failedRunResponse): an error saying it has no results and that 0 issues does not mean the page is clean, instead of reading its empty export as "0 issues, all retrieved". - Refusals the user resolves on their own account are explained instead of reported as faults: the plan's saved-scan limit ("make over N scans"), a plan without the External APIs add-on, and Website Scanner's "user info missing" (open Website Scanner once in the dashboard first). Each says what to tell the user and quotes the API's words, with no "open an issue on GitHub" line. They carry NO_FEEDBACK_HINT, a new symbol-keyed flag in lib/feedback-hint.ts that is never serialised, so withFeedbackHint does not send the agent to submitFeedback. (startAccessibilityScan's 402 scan-limit refusal is still answered first by the scanner's ScanRefusedError; this covers the message wherever else it reaches the error handler.) - accessibilityExpert keeps a question within the knowledge base's 500-character limit, which refused longer ones ("Query is too long"). A longer question keeps its last sentence, where agents put the question itself, and as much of its opening as fits, cut at a word break; the answer starts with a note that it was shortened. Co-Authored-By: Claude Opus 5.5 * test(accessibility): drop the new test files from this PR Removes the six new test files and the test cases this PR added to existing files. Keeps only the edits that existing tests need because behaviour changed here: no 5-minute wait on a finalizing report, compareWith guidance on reports with no compare, failed-page wording, and no tunnel killing. Co-Authored-By: Claude Opus 5.5 * docs(accessibility): fetchAccessibilityIssues no longer waits on a processing report Co-Authored-By: Claude Opus 5.5 * fix(accessibility): resolve a Scanner link or build id passed as compareWith, and compare runs oldest first compareScanRuns took compareWith verbatim as a run id, so a Website Scanner report link or a run's thBuildId failed with "Scan run id must be an Integer", a run id of another scan with "Scan logs not found", and a newer run id swapped new and resolved without a word. compareWith is now resolved as a source is: a link or 40-character id through sourceFromReportUrl / resolveBuild and findScanRun. A run of the same scan is compared with the older run as "before", saying so when the order was swapped. Another scan's run, a build or a report, and a run id the scan's full run list does not hold, get a plain answer naming both reads' arguments. compareWith's description drops "older": the runs are ordered here now. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): do not compare a run or build that is still processing Comparing against a run still processing gave a clean-looking diff ("0 new, 0 resolved, 2 still open") while its issues were still being added, which could read a fix as verified. The Website Scanner, Automated Tests and app build compares now check both sides' statuses first: while either is processing or finalizing, the answer is a plain "not finished yet" naming the run or build and its status, telling the agent to have the user ask again in a few minutes and not to call again in this turn. The Automated Tests compare's "only the issues found so far are compared" warning goes, as such a build is no longer compared. An app build compare is dispatched before the finalizing read, so a build still running its rules gets this answer rather than "call again in a minute or two". Co-Authored-By: Claude Opus 5.5 * fix(accessibility): find a Website Scanner run given only its build id in the scan list A Website Scanner run's thBuildId passed as buildId, with no link, was refused whenever the API could not yet read a run by its build id: "pass the scan's report link as reportUrl instead", plus the GitHub and submitFeedback lines. An agent then searched for the run itself, and on an old scan gave up. Such an id now walks the scan list with findScanRun, as a link does, under the same 25-page cap and with the same "found in the scan list instead" note, naming the scan and run it found. Only when the walk finds nothing (or the API that reads runs by build id has none) is the answer a plain not-found, saying what to pass instead, without the GitHub or submitFeedback lines. findScanRun's not-found is now a ScanNotInListError, with the same message. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): read an Automate build's hashed_id passed as buildId as its accessibility build An SDK run prints the Automate build's hashed_id ("stopBinSession hashed_id=…"), and an agent passing it as buildId was refused, with only a Website Scanner build id offered as an alternative. It then spent recovery calls on fetchBuildInsights and listSessions before finding a session link. A 40-hex id that is neither an Automated Tests nor an App Accessibility build is now tried as an Automate build, then an App Automate build, through the same session mapping their dashboard links use, before the scan-list walk. The answer says which build the id was read as. An id none of them knows says so in the plain not-found answer. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): a report still running with no export yet is "no report yet", not "Failed to fetch report: null" A build stuck in "running" since 2025-09-16 was read as processing, so its export ran anyway and failed with error null: "Failed to fetch report: null. Please open an issue on GitHub", and the agent went searching for another product before handing the question back to the user. When a report is still processing and its export fails, the answer is now a plain "still running (status, started ), no report yet". If it started more than 24 hours ago it is called probably stuck, to be run again, rather than something to call again for. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): name the Workflow Analyzer report before this one for compareWith "previous" compareWith "previous" on a Workflow Analyzer report only said there is no built-in compare and to read both, without saying which report came before. Agents went looking for a way to list reports, hit routing refusals, and the hosted run gave no answer. The earlier report is now found here: the account's report list (newest first) locates this report, and the summaries of the completed reports after it give their start and pages. The newest one before it that scanned one of its pages is named, or else the newest on its site, with its id, its dashboard link and the two reads to make. When none is among the newest reports, the answer says so and to ask the user which report to compare with. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): keep a manual app report's page under 40k characters, describe cursor, and find its previous report A 70-issue manual report came back as one page of 51,132 characters, over Claude Code's 50,000-character limit, so the client saved it to a file. The page budget counted only the bare issue rows, while the answer prints them as indented JSON with the rules and screens they name. An app page (build or manual report) now counts each issue as printed, and each rule's text and screen's screenshot the first time the page names it, against a 35,000 character budget: with the summary, notes and wrappers the whole answer stays under about 40,000. The cursor still counts the bare issues. Agents then passed cursor as an issue index (40, 35) and got the same page again. cursor is now described as the next_page value from the previous answer, not an issue index. compareWith "previous" on a manual report now names the newest finished manual report of the same app recorded before it (found through the report list, by the report's name and then its app's package), with its id, its dashboard link and the two reads to make, as for a Workflow Analyzer report. Descriptions (total unchanged at 21,500): - cursor: "Pagination offset (default 0); with compareWith, issues already listed." (71) -> "The next_page value from the previous answer (default 0), not an issue index." (77) Co-Authored-By: Claude Opus 5.5 * fix(accessibility): match a Workflow Analyzer report's site only on a page address that parses Follow-up to the "previous" Workflow Analyzer lookup: a page address that does not parse has no site, and two such addresses no longer count as one site. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): say compareWith "previous" works for every report compareWith's description ended "Others: read both", so for a Workflow Analyzer or manual app report agents never passed "previous" and went searching for the earlier report themselves: in re-runs, 4 of 5 sessions skipped it, and one hosted run again gave no answer. "previous" now finds the earlier report for those too, and the tool itself answers any pair it has no compare for with how to read both, so the description says "previous" works for any report. compareWith: 'Runs of one scan, web/app builds: "previous", scanRunId, build id/link. Others: read both.' (90) -> '"previous" (any report), or a run of the same scan or a web/app build: its id or link.' (86). Total 21,500 -> 21,496. Co-Authored-By: Claude Opus 5.5 * revert(local): leave BrowserStack Local tunnel handling and the feedback hint as they are on #541 Restores src/lib/local.ts, src/lib/feedback-hint.ts and src/tools/live-utils/start-session.ts to their #541 versions, which puts back killExistingBrowserStackLocalProcesses and drops ensureOwnTunnel, stopOwnTunnels, killOwnTunnelsNow and NO_FEEDBACK_HINT. startScan again starts a tunnel with ensureLocalBinarySetup for a local URL and calls killExistingBrowserStackLocalProcesses for a public one. The refusals for the hosted MCP and for USE_OWN_LOCAL_BINARY_PROCESS with a local URL stay ScanRefusedError plain answers, and account refusals keep their wording without the feedback-hint opt-out. Co-Authored-By: Claude Opus 5.5 * refactor(accessibility): move this PR's report, compare and re-scan helpers into accessiblity-utils accessibility.ts had grown to 3,337 lines. The helpers this PR added move, unchanged, into accessiblity-utils: - error-messages.ts: asSentence and the account-refusal wording - expert-query.ts: fitting an accessibilityExpert question to 500 characters - report-answers.ts: failed, cancelled, finalizing and no-report-yet answers, unknown build ids, and pages not scanned or that showed an error page - compare-with.ts: what compareWith names (a scan run, the "previous" report), the read-both guidance and the not-finished-yet answer - compare-lists.ts: the export diff, compared lists and not-loaded note that scan-run and web-build comparisons share - web-build-compare.ts: comparing Automated Tests (web) builds - scan-rerun.ts: the saved-settings checks, notes and hints of a re-run shared.ts takes the small helpers of accessibility.ts these need too (the response builders, source names, progress, the scanner client, WCAG names), so neither side imports the other. accessibility.ts re-exports EXPERT_QUERY_MAX_CHARS. No behaviour or tool description changes. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): no stale exports, faster link lookups, Automate probe falls through - An export started while a report is still processing no longer has its task remembered: continued after the report finished, it handed back the mid-processing snapshot as final. - A run found by its build id is cached (per account, bounded), since a build id names one fixed run: a link read again, or both sides of a compare, skip the scan-list walk. The EXPORT_WAIT_MS comment no longer claims a whole call stays under 60 s. - A 40-hex id that Automate or App Automate can't answer about (an HTTP failure other than 404, or an Automate build with no sessions) moves on to the next API instead of ending the call, and the not-found answer says why. Once a session is found, what it says is still the answer. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): don't offer re-runs that can't reach a local page; say what a re-run ignores - A local page this MCP tunnelled itself is saved with an identifier nobody was shown, and the tunnel is gone after the next public scan or a restart. Its scan no longer gets the "re-run with scanId" hint, and a re-run asked for a scan saved with Local but no localIdentifier starts a new scan instead, saying why and how to re-run it with its own tunnel. - A re-run can't take a login or a Local identifier: each passed one is now named as not used, with how to scan with it (a new scan), instead of only "could not be checked". - The saved scan's pages and name come from the account, which another user may have set: they are now quoted as untrusted data in re-run answers. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): refuse to compare with a cancelled scan run, as with a failed one A cancelled run has no results and exports an empty report, so compared with it every issue of the other run read as resolved or new. Reads already treat a cancelled run as having no results; compares now do too. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): don't silently take compareWith as the older build when start times are missing A build with no start time made the order check always false, so a newer build passed as compareWith was used as "before" and new and fixed issues swapped without a word. The order now falls back to the build number for runs of one build name and project; with neither, the passed order is kept and the answer says so. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): shorten CJK expert questions by sentence, and say what was kept The sentence split knew only . ? !, so a Chinese or Japanese question was one "sentence" cut at 500, losing the question at its end. 。？！ now end sentences too. A cut no longer leaves half a surrogate pair, and the note says whether the opening alone or the opening and last sentence were sent, as it happened. Co-Authored-By: Claude Opus 5.5 * fix(accessibility): name the plan's scan limit the API gave, not a hard-coded 50 Co-Authored-By: Claude Opus 5.5 * style: prettier Co-Authored-By: Claude Opus 5.5 * fix(accessibility): put the saved scan's name after the settings list in a re-run refusal The quoted name sat between "these differ:" and the list, so it read as one of the differences. Co-Authored-By: Claude Opus 5.5 * test(accessibility): cover web-build compare, re-runs, failed runs and the Automate id probe Brings back the web-build compare, re-run and failed-run tests dropped in 7ee4c3d, updated to this PR's current behaviour, and adds cases for the review fixes: - web-build compare: refusing a build still processing, ordering by build number when a start time is missing, and keeping the passed order with a note otherwise; - re-runs: a Local-saved scan without localIdentifier starts a new scan, a local page the MCP tunnelled gets no re-run hint, a login or tunnel passed is named as not used, the saved pages and name are quoted as untrusted, and the scan limit is the API's number; - a 40-hex Automate or App Automate build id: found, falling through on 403/5xx or an empty build, and naming why Automate could not be asked; - compare refused for a cancelled run; - exports: no task kept while a report is processing, a run read by build id reused for 5 minutes at most, and a link's run found once not looked up again; - expert questions: the note matches what was cut, CJK sentence ends, no split character. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- docs/TOOLS.md | 4 +- src/tools/accessibility.ts | 1122 +++++++++-------- .../accessiblity-utils/app-report-fetcher.ts | 175 ++- src/tools/accessiblity-utils/compare-lists.ts | 131 ++ src/tools/accessiblity-utils/compare-with.ts | 141 +++ .../accessiblity-utils/error-messages.ts | 34 + src/tools/accessiblity-utils/expert-query.ts | 57 + .../accessiblity-utils/report-answers.ts | 252 ++++ .../accessiblity-utils/report-fetcher.ts | 566 ++++++++- src/tools/accessiblity-utils/report-parser.ts | 88 +- src/tools/accessiblity-utils/scan-rerun.ts | 201 +++ src/tools/accessiblity-utils/scanner.ts | 333 ++++- src/tools/accessiblity-utils/shared.ts | 184 +++ .../accessiblity-utils/web-build-compare.ts | 390 ++++++ tests/tools/accessibility.test.ts | 1 + .../accessibilityAppCompareOutput.test.ts | 28 +- .../accessibilityAutomateBuildId.test.ts | 178 +++ tests/tools/accessibilityExpertQuery.test.ts | 115 ++ ...accessibilityFailedAndSkippedPages.test.ts | 311 +++++ .../accessibilityFetchPagesAndCompare.test.ts | 35 +- .../tools/accessibilityReportFetcher.test.ts | 58 + tests/tools/accessibilityReportLinks.test.ts | 4 + tests/tools/accessibilityRescan.test.ts | 670 ++++++++++ tests/tools/accessibilityScanOutput.test.ts | 122 +- .../accessibilityScanTunnelAndWait.test.ts | 2 + .../accessibilityWebBuildCompare.test.ts | 499 ++++++++ 26 files changed, 5027 insertions(+), 674 deletions(-) create mode 100644 src/tools/accessiblity-utils/compare-lists.ts create mode 100644 src/tools/accessiblity-utils/compare-with.ts create mode 100644 src/tools/accessiblity-utils/error-messages.ts create mode 100644 src/tools/accessiblity-utils/expert-query.ts create mode 100644 src/tools/accessiblity-utils/report-answers.ts create mode 100644 src/tools/accessiblity-utils/scan-rerun.ts create mode 100644 src/tools/accessiblity-utils/shared.ts create mode 100644 src/tools/accessiblity-utils/web-build-compare.ts create mode 100644 tests/tools/accessibilityAutomateBuildId.test.ts create mode 100644 tests/tools/accessibilityExpertQuery.test.ts create mode 100644 tests/tools/accessibilityFailedAndSkippedPages.test.ts create mode 100644 tests/tools/accessibilityRescan.test.ts create mode 100644 tests/tools/accessibilityWebBuildCompare.test.ts diff --git a/docs/TOOLS.md b/docs/TOOLS.md index ac6e0e89..52d9c2f8 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -340,11 +340,11 @@ searchCapability → describeCapability → invokeCapability | Tool | What it does | |---|---| -| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Optional settings: `wcagVersion` (WCAG 2.0–2.2 at A/AA/AAA, or RGAA 4.1), `bestPractices`, `needsReview`, `advancedRules` and `experimentalRules`; the answer states the settings the scan ran with. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | +| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Optional settings: `wcagVersion` (WCAG 2.0–2.2 at A/AA/AAA, or RGAA 4.1), `bestPractices`, `needsReview`, `advancedRules` and `experimentalRules`; the answer states the settings the scan ran with. To scan the same page again (for example after a fix), pass the earlier scan's `scanId`: that starts a new run of that scan with its saved settings, which `fetchAccessibilityIssues` can compare with `compareWith: "previous"`. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | | `accessibilityExpert` | Ask the A11y Expert (WCAG 2.0/2.1/2.2, mobile/web usability, best practices). | | `createAccessibilityAuthConfig` | Create an authentication configuration (form-based or basic) for accessibility scans behind a login. | | `getAccessibilityAuthConfig` | Retrieve an existing accessibility authentication configuration by ID. | -| `fetchAccessibilityIssues` | Fetch accessibility issues, with pagination, from a Website Scanner run, an Automated Tests build (web) or App Accessibility build (mobile app, by the build id the SDK prints), a Workflow Analyzer report, an App Accessibility manual report (`appReportId`), or a report's dashboard link (a Website Scanner link is read by the run's build id, however old the scan; a Workflow Analyzer link by the report id it carries). Once the tests or scan have finished, waits while the report is still processing; a run still in progress is read as it is, with a note. For an app build with no scans, lists the likely causes. | +| `fetchAccessibilityIssues` | Fetch accessibility issues, with pagination, from a Website Scanner run, an Automated Tests build (web) or App Accessibility build (mobile app, by the build id the SDK prints), a Workflow Analyzer report, an App Accessibility manual report (`appReportId`), or a report's dashboard link (a Website Scanner link is read by the run's build id, however old the scan; a Workflow Analyzer link by the report id it carries). Once the tests or scan have finished, a report that is still processing is not waited on: the answer says to call again in a minute or two; a run still in progress is read as it is, with a note. For an app build with no scans, lists the likely causes. | | `list_app_accessibility_builds` | List an App Accessibility project's builds newest first, with each build's id, status, app and issue counts - use this to find the latest run, a build by name, or one person's builds. | | `list_app_accessibility_projects` | List the account's App Accessibility projects newest first, or find one by name - use this to get the project id that lists its builds. | | `list_manual_app_accessibility_reports` | List manual App Accessibility reports (App Live sessions scanned with Workflow Analyzer) newest first, with each report's id, app, status and issue counts - use this to find a report QA shared or the latest manual scan of an app. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 6c334445..e69af3de 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -3,16 +3,18 @@ import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { AccessibilityScanner, + ScanRefusedError, ScanTimeoutError, + cleanScanName, + samePage as sameScanPage, type ScanFailureDetails, type ScanSettings, } from "./accessiblity-utils/scanner.js"; import { AccessibilityReportFetcher, - EXPORT_WAIT_MS, ExportPendingError, + ScanNotInListError, ScanRunNotFoundError, - scanRunLink, type IssueSource, } from "./accessiblity-utils/report-fetcher.js"; import { @@ -28,30 +30,81 @@ import { } from "./accessiblity-utils/auth-config.js"; import { trackMCP } from "../lib/instrumentation.js"; import { - compareIssues, - loadMainIssues, parseAccessibilityReportFromCSV, + samePage, type IssueSummary as WebIssueSummary, + type RunIssue, } from "./accessiblity-utils/report-parser.js"; import { wrapUntrusted } from "../lib/untrusted-content.js"; +import { isLocalURL } from "../lib/local.js"; import { queryAccessibilityRAG } from "./accessiblity-utils/accessibility-rag.js"; -import { getBrowserStackAuth } from "../lib/get-auth.js"; import { BrowserStackConfig } from "../lib/types.js"; import { elicitCredentialsIfSupported } from "../lib/elicit-credentials.js"; import logger from "../logger.js"; import { NEEDS_A11Y_CONFIG_ID, NEEDS_A11Y_SCAN_ID } from "./tool-handoff.js"; - -interface AuthCredentials { - username: string; - password: string; -} - -interface ScanProgressContext { - sendNotification: (notification: any) => Promise; - _meta?: { - progressToken?: string | number; - }; -} +import { + createErrorResponse, + createSuccessResponse, + describeSource, + exportProgress, + formatDuration, + initializeScanner, + notifyScanProgress, + onOff, + PROCESSING_STATUSES, + scanDashboardNote, + setupAuth, + sourceArgs, + wcagName, + WCAG_VERSIONS, + type ScanProgressContext, +} from "./accessiblity-utils/shared.js"; +import { + accountRefusal, + asSentence, +} from "./accessiblity-utils/error-messages.js"; +import { + EXPERT_QUERY_MAX_CHARS, + fitExpertQuery, +} from "./accessiblity-utils/expert-query.js"; +import { + buildIdNotFound, + errorPagesNote, + failedRunResponse, + failedScanRunResponse, + failureReasons, + FINALIZING_STATUSES, + noReportYetResponse, + pageReasons, + stillFinalizingResponse, +} from "./accessiblity-utils/report-answers.js"; +import { + NO_BUILT_IN_COMPARE, + notReadyToCompare, + previousReportToCompare, + readBothToCompare, + scanRunToCompare, +} from "./accessiblity-utils/compare-with.js"; +import { + comparedList, + diffExports, + pagesNotLoadedNote, +} from "./accessiblity-utils/compare-lists.js"; +import { compareWebBuilds } from "./accessiblity-utils/web-build-compare.js"; +import { + compareHint, + localRerunNote, + localScanNotRerunNote, + otherPageNote, + rescanHint, + savedSettingDifferences, + uncheckedSettings, + unusedLoginAndTunnel, + withNotes, + type ScanArgs, +} from "./accessiblity-utils/scan-rerun.js"; + +export { EXPERT_QUERY_MAX_CHARS }; interface FormAuthArgs { name: string; @@ -74,37 +127,6 @@ interface BasicAuthArgs { type AuthConfigArgs = FormAuthArgs | BasicAuthArgs; -function setupAuth(config: BrowserStackConfig): AuthCredentials { - const authString = getBrowserStackAuth(config); - const [username, password] = authString.split(":"); - return { username, password }; -} - -function createErrorResponse(message: string): CallToolResult { - return { - content: [ - { - type: "text", - text: message, - }, - ], - isError: true, - }; -} - -/** - * Each message ends with a newline: clients that join text blocks with no separator - * would otherwise run a link at the end of one block into the next block's first word. - */ -function createSuccessResponse(messages: string[]): CallToolResult { - return { - content: messages.map((text) => ({ - type: "text" as const, - text: `${text}\n`, - })), - }; -} - function handleMCPError( toolName: string, server: McpServer, @@ -112,37 +134,19 @@ function handleMCPError( error: unknown, ): CallToolResult { trackMCP(toolName, server.server.getClientVersion()!, error, config); - const errorMessage = error instanceof Error ? error.message : "Unknown error"; + const errorMessage = + (error instanceof Error && error.message.trim()) || "Unknown error"; + const refusal = accountRefusal(errorMessage); + if (refusal) { + return createErrorResponse( + `${refusal}\nBrowserStack said: ${errorMessage}`, + ); + } return createErrorResponse( - `Failed to ${toolName.replace(/([A-Z])/g, " $1").toLowerCase()}: ${errorMessage}. Please open an issue on GitHub if the problem persists`, + `Failed to ${toolName.replace(/([A-Z])/g, " $1").toLowerCase()}: ${asSentence(errorMessage)} Please open an issue on GitHub if the problem persists`, ); } -async function notifyScanProgress( - context: ScanProgressContext, - message: string, - progress = 0, -): Promise { - await context.sendNotification({ - method: "notifications/progress", - params: { - progressToken: context._meta?.progressToken?.toString() ?? "NOT_FOUND", - message, - progress, - total: 100, - }, - }); -} - -async function initializeScanner( - config: BrowserStackConfig, -): Promise { - const scanner = new AccessibilityScanner(); - const auth = setupAuth(config); - scanner.setAuth(auth); - return scanner; -} - async function initializeReportFetcher( config: BrowserStackConfig, ): Promise { @@ -164,7 +168,19 @@ async function executeAccessibilityRAG( undefined, config, ); - return await queryAccessibilityRAG(args.query, config); + const { query, shortened, kept } = fitExpertQuery(args.query); + const answer = await queryAccessibilityRAG(query, config); + if (!shortened) return answer; + return { + ...answer, + content: [ + { + type: "text", + text: `The question was ${args.query.length} characters; the knowledge base takes at most ${EXPERT_QUERY_MAX_CHARS}, so ${query.length} were sent: its ${kept ?? "opening"}. If the answer misses the point, ask again more briefly.`, + }, + ...(answer.content ?? []), + ], + }; } catch (error) { return handleMCPError("accessibilityExpert", server, config, error); } @@ -212,44 +228,6 @@ function issueSourceFrom(args: FetchIssuesArgs): RequestedSource | string { return { kind: "scan", scanId: args.scanId, scanRunId: args.scanRunId }; } -/** The arguments that name the same source again, for the next-page hint. */ -function sourceArgs(source: IssueSource): string { - switch (source.kind) { - case "scan": - return `scanId: "${source.scanId}", scanRunId: "${source.scanRunId}"`; - // A run read by its build id is named by its link again, or, given as a build id, by that. - case "scanBuild": - return source.scanName !== undefined - ? `reportUrl: "${scanRunLink(source)}"` - : `buildId: "${source.thBuildId}"`; - case "build": - case "appBuild": - return `buildId: "${source.buildId}"`; - case "workflow": - return `workflowReportId: "${source.reportId}"`; - case "appReport": - return `appReportId: "${source.reportId}"`; - } -} - -/** What a dashboard link resolved to, in words. */ -function describeSource(source: IssueSource): string { - switch (source.kind) { - case "scan": - return `Website Scanner scan ${source.scanId}, run ${source.scanRunId}`; - case "scanBuild": - return `Website Scanner run with build id ${source.thBuildId}`; - case "build": - return `Automated Tests build ${source.buildId}`; - case "workflow": - return `Workflow Analyzer report ${source.reportId}`; - case "appBuild": - return `App Accessibility build ${source.buildId}`; - case "appReport": - return `App Accessibility manual report ${source.reportId}`; - } -} - async function executeFetchAccessibilityIssues( args: FetchIssuesArgs, context: ScanProgressContext | undefined, @@ -311,108 +289,34 @@ function excludedIssuesNote( return notes; } -/** How often, and for how long, a first read waits on a report that is still processing. */ -export const REPORT_STATUS_POLL_MS = 10_000; -export const REPORT_STATUS_WAIT_MS = 5 * 60 * 1000; - -/** Statuses during which a report's issues are still being added (`rule_processing`: an app build's). */ -const PROCESSING_STATUSES = new Set([ - "queued", - "running", - "processing", - "finalizing", - "rule_processing", - "in_progress", -]); - -/** - * Of those, the ones worth waiting on: the processing after the run (a web scan's advanced - * issues, an app build's rules), which ends within minutes. A report whose scan or tests are still running is read as it is, - * with a note, since that can last as long as the suite does. - */ -const WAIT_STATUSES = new Set(["finalizing", "rule_processing"]); - interface ProcessingState { - /** The status when the issues were read; undefined when it could not be read. */ + /** The report's status when its issues were read; undefined when it could not be read. */ status?: string; - /** The status first seen, before any waiting. */ - firstStatus?: string; - waitedMs: number; } -/** - * Where a report is in its processing before its issues are read. Issues keep being added - * until it completes; an advanced scan adds its advanced-rule issues while "finalizing", for - * minutes after the tests end, so a first read waits for that, reporting progress, for up to - * REPORT_STATUS_WAIT_MS. A run still in progress is not waited on (it can take as long as - * the suite), nor is a later page, nor a status that cannot be read. - */ -async function waitForProcessing( - reportFetcher: AccessibilityReportFetcher, - source: IssueSource, - wait: boolean, - context: ScanProgressContext | undefined, -): Promise { - const started = Date.now(); - const firstStatus = await reportFetcher.reportStatus(source); - let status = firstStatus; - while ( - wait && - status && - WAIT_STATUSES.has(status) && - Date.now() - started < REPORT_STATUS_WAIT_MS - ) { - const waited = Date.now() - started; - if (context) { - await notifyScanProgress( - context, - `The report is still ${status}; waiting for it to complete (${formatDuration(waited)} of up to ${formatDuration(REPORT_STATUS_WAIT_MS)}).`, - Math.min(99, Math.round((waited / REPORT_STATUS_WAIT_MS) * 100)), - ); - } - await new Promise((resolve) => setTimeout(resolve, REPORT_STATUS_POLL_MS)); - status = await reportFetcher.reportStatus(source); - } - return { status, firstStatus, waitedMs: Date.now() - started }; -} - -function formatDuration(ms: number): string { - const seconds = Math.round(ms / 1000); - return seconds < 60 - ? `${seconds}s` - : `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`; -} - -/** What the answer says about the report's processing, if anything. */ -function processingNote({ - status, - firstStatus, - waitedMs, -}: ProcessingState): string[] { - if (status && PROCESSING_STATUSES.has(status)) { - const meaning = - status === "finalizing" - ? ", which on an advanced scan means advanced issue processing" - : ""; - return [ - `⚠️ This report is still processing (status: ${status}${meaning}). The issues below are only those found so far: more will be added and the total will grow. Call fetchAccessibilityIssues again in a few minutes for the complete list.`, - ]; - } - if (firstStatus && PROCESSING_STATUSES.has(firstStatus)) { - return [ - `The report was still processing (status: ${firstStatus}); waited ${formatDuration(waitedMs)} for it to complete before reading its issues.`, - ]; - } - return []; +/** What the answer says about a report read while it is still processing, if anything. */ +function processingNote({ status }: ProcessingState): string[] { + if (!status || !PROCESSING_STATUSES.has(status)) return []; + const meaning = + status === "finalizing" + ? ", which on an advanced scan means advanced issue processing" + : ""; + return [ + `⚠️ This report is still processing (status: ${status}${meaning}). The issues below are only those found so far: more will be added and the total will grow. Call fetchAccessibilityIssues again in a few minutes for the complete list.`, + ]; } -/** The last line: how to get the next page, or that there is none. */ +/** + * The last line: how to get the next page, or that there is none. With pages that were not + * scanned, the issues are all there are of the pages that were, not of the run's pages. + */ function closingMessage( next_page: number | null, remaining: number, source: IssueSource, stillProcessing: boolean, extraArgs = "", + pagesSkipped = false, ): string { if (next_page !== null) { return `${remaining} more issues available. Use fetchAccessibilityIssues with ${sourceArgs(source)}${extraArgs} and cursor: ${next_page} to get the next batch.`; @@ -420,6 +324,9 @@ function closingMessage( if (stillProcessing) { return `All issues found so far retrieved; the report is still processing.`; } + if (pagesSkipped) { + return `All issues retrieved for the pages that were scanned.`; + } return `✅ All issues retrieved.`; } @@ -653,19 +560,22 @@ function issueList( return `${label} (${rows.length})${range}: ${wrapUntrusted("accessibility scan results", JSON.stringify(shown, null, 2))}`; } -/** The build to compare with, or why there is none. */ +/** The build to compare with, or the answer to give when there is none. */ async function baselineAppBuild( reportFetcher: AccessibilityReportFetcher, buildId: string, compareWith: string, ): Promise< - { buildId: string; sameName?: boolean; searchedAll?: boolean } | string + | { buildId: string; sameName?: boolean; searchedAll?: boolean } + | CallToolResult > { let baseline: { buildId: string; sameName?: boolean; searchedAll?: boolean }; if (compareWith === "previous") { const previous = await reportFetcher.previousAppBuild(buildId); if (!previous) { - return "This is its project's first build, so there is nothing earlier to compare it with. Pass an earlier build's id or link as compareWith instead."; + return createErrorResponse( + "This is its project's first build, so there is nothing earlier to compare it with. Pass an earlier build's id or link as compareWith instead.", + ); } baseline = { buildId: previous.buildId, @@ -678,12 +588,17 @@ async function baselineAppBuild( : { kind: "build", buildId: compareWith }; const source = await reportFetcher.resolveBuild(named); if (source.kind !== "appBuild") { - return `compareWith names ${describeSource(source)}, which is not an App Accessibility build. Pass an earlier app build's id or link, or "previous".`; + return readBothToCompare( + `compareWith names ${describeSource(source)}, which is not an App Accessibility build; an app build is compared here only with another ("previous", or an earlier build's id or link).`, + true, + ); } baseline = { buildId: source.buildId }; } if (baseline.buildId === buildId) { - return `compareWith names this build itself. Pass an earlier build's id or link, or "previous".`; + return createErrorResponse( + `compareWith names this build itself. Pass an earlier build's id or link, or "previous".`, + ); } return baseline; } @@ -692,7 +607,8 @@ async function baselineAppBuild( * What changed between an earlier app build and this one: issues new in this build, * issues the earlier one had that this one does not, and how many it kept. An issue on a * screen this build did not reach also reads as resolved, so a build that scanned other - * screens or ran other tests is called out. + * screens or ran other tests is called out. A build still running its tests or rules is not + * compared. */ async function compareAppBuilds( reportFetcher: AccessibilityReportFetcher, @@ -700,18 +616,27 @@ async function compareAppBuilds( compareWith: string, cursor: number, preface: string[], - processing: ProcessingState, ): Promise { const baseline = await baselineAppBuild( reportFetcher, source.buildId, compareWith, ); - if (typeof baseline === "string") return createErrorResponse(baseline); + if ("content" in baseline) return baseline; const [named, current] = await Promise.all([ reportFetcher.appBuildIssues(baseline.buildId), reportFetcher.appBuildIssues(source.buildId), ]); + const unfinished = [current, named].find((b) => + PROCESSING_STATUSES.has(b.status), + ); + if (unfinished) { + return notReadyToCompare( + `App Accessibility build #${unfinished.build_number} (${unfinished.buildId})`, + unfinished.status, + preface, + ); + } // Issues on screens past the read limit would read as resolved or new. const partial = [current, named].find((b) => b.screens_read !== undefined); if (partial) { @@ -782,7 +707,6 @@ async function compareAppBuilds( ); return createSuccessResponse([ ...preface, - ...processingNote(processing), ...(reversed ? [ "compareWith names a build newer than this one, so the comparison runs from this build (the earlier) to that one (the later).", @@ -927,20 +851,6 @@ async function appReportMessages( ]; } -/** Progress while a report's CSV export is generated, so clients see the call is alive. */ -function exportProgress( - context: ScanProgressContext | undefined, -): (waitedMs: number) => Promise { - return async (waitedMs) => { - if (!context) return; - await notifyScanProgress( - context, - `Generating the report's CSV export (large reports take longer): ${formatDuration(waitedMs)} so far.`, - Math.min(99, Math.round((waitedMs / EXPORT_WAIT_MS) * 100)), - ); - }; -} - /** What to say of an export still being generated: how to continue it, not an error. */ function exportPendingMessages( source: IssueSource, @@ -967,18 +877,53 @@ async function fetchAccessibilityIssues( ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; // A build id names a web build, a mobile app build, or a Website Scanner run (a scanner - // link's thBuildId); the APIs tell which. + // link's thBuildId); the APIs tell which. An Automate or App Automate build's own id + // (hashed_id) resolves to the accessibility build its sessions ran in. let source = await reportFetcher.resolveBuild(named, { orScanRun: true }); - // Comparing runs needs the scan's ids, which only the scan list maps a build id to. - if (compareWith !== undefined && source.kind === "scanBuild") { - source = await reportFetcher.findScanRun(source); - } - // Two scan runs are compared through their CSV exports; two app builds, here below. - if (compareWith !== undefined && source.kind === "scan") { - return compareScanRuns(source, compareWith, config, context); + const mapped = + named.kind === "build" && + (source.kind === "build" || source.kind === "appBuild") && + source.buildId !== named.buildId + ? [ + `Build id ${named.buildId} is ${source.kind === "build" ? "an Automate" : "an App Automate"} build's own id (hashed_id); its accessibility results are in ${describeSource(source)}.`, + ] + : []; + // Two scan runs are compared through their CSV exports; two web builds through the compare + // API or their exports; two app builds, here below. + if (compareWith !== undefined) { + // Comparing runs needs the scan's ids, which only the scan list maps a build id to. + if (source.kind === "scanBuild") { + source = await reportFetcher.findScanRun(source); + } + if (source.kind === "scan") { + return compareScanRuns(source, compareWith, config, context); + } + if (source.kind === "build") { + return compareWebBuilds( + reportFetcher, + source, + compareWith, + requested.kind === "link" + ? [`Resolved the report link to ${describeSource(source)}.`] + : mapped, + context, + ); + } + if ( + (source.kind === "workflow" || source.kind === "appReport") && + compareWith === "previous" + ) { + return previousReportToCompare(reportFetcher, source); + } + if (source.kind !== "appBuild") { + return readBothToCompare( + NO_BUILT_IN_COMPARE[source.kind], + source.kind === "appReport", + ); + } } const pageArg = pageUrl ? `, pageUrl: "${pageUrl}"` : ""; - const notes: string[] = []; + const notes: string[] = [...mapped]; // A run named by its build id is read by it, which works however old the scan is. If the // API does not know it by that id, the scan list is searched as before. let reportLink: string | undefined; @@ -1005,19 +950,20 @@ async function fetchAccessibilityIssues( ); } if (!(error instanceof ScanRunNotFoundError)) throw error; - if (source.scanName === undefined) { - throw new Error( - error.routeMissing - ? `No Automated Tests (web) or App Accessibility build with id "${source.thBuildId}". If it is a Website Scanner run's build id, this API cannot read a run by it yet: pass the scan's report link as reportUrl instead` - : `No Automated Tests (web), App Accessibility or Website Scanner build with id "${source.thBuildId}". Pass the id the BrowserStack SDK printed as "testhub id" at the end of the run (40 lowercase letters and digits), or the report's dashboard link as reportUrl`, - ); - } + // A bare build id (no link) that no build API knows: where the API reads runs by build + // id, it knows no such run; where it cannot yet, the scan list is searched, as for a link. + const bare = source.scanName === undefined; + const thBuildId = source.thBuildId; + if (bare && !error.routeMissing) return buildIdNotFound(thBuildId); const why = error.routeMissing ? "This API does not read a run by its build id yet" : "The API could not read this run by the link's build id (not found, or not visible to this account)"; try { source = await reportFetcher.findScanRun(source); } catch (searchError) { + if (bare && searchError instanceof ScanNotInListError) { + return buildIdNotFound(thBuildId, searchError); + } throw new Error( `${why}, and searching the scan list found nothing either: ${ searchError instanceof Error @@ -1026,30 +972,21 @@ async function fetchAccessibilityIssues( }`, ); } + if (bare) { + notes.push( + `Build id ${thBuildId} is the build id of ${describeSource(source)}.`, + ); + } notes.push(`${why}, so it was found in the scan list instead.`); } } - if (compareWith !== undefined && source.kind !== "appBuild") { - return createErrorResponse( - `compareWith compares Website Scanner runs and App Accessibility builds; ${describeSource(source)} is neither.`, - ); - } - const processing = await waitForProcessing( - reportFetcher, - source, - cursor === 0, - context, - ); - const stillProcessing = - processing.status !== undefined && - PROCESSING_STATUSES.has(processing.status); const preface = [ ...(requested.kind === "link" ? [`Resolved the report link to ${describeSource(source)}.`] : []), ...notes, ]; - + // Two app builds are compared once both have finished; the comparison reads their statuses. if (source.kind === "appBuild" && compareWith !== undefined) { return compareAppBuilds( reportFetcher, @@ -1057,9 +994,38 @@ async function fetchAccessibilityIssues( compareWith, cursor, preface, - processing, ); } + const processing: ProcessingState = { + status: await reportFetcher.reportStatus(source), + }; + const stillProcessing = + processing.status !== undefined && + PROCESSING_STATUSES.has(processing.status); + + // A failed run has nothing to read, and its export said so badly: an empty report for a + // scan ("0 issues ✅"), "Failed to fetch report: null" for a build. Neither is exported. + // A cancelled scan run leaves the same empty report. + if ( + processing.status === "failed" || + (source.kind === "scan" && processing.status === "cancelled") + ) { + const failed = await failedRunResponse( + source, + processing.status, + config, + preface, + ); + if (failed) return failed; + } + if ( + cursor === 0 && + processing.status && + FINALIZING_STATUSES.has(processing.status) + ) { + return stillFinalizingResponse(source, processing.status, preface, pageArg); + } + if (source.kind === "appBuild") { return createSuccessResponse( await appBuildMessages( @@ -1085,6 +1051,15 @@ async function fetchAccessibilityIssues( ); } + // The pages a completed run could not scan are simply absent from its report; its scan + // logs say which, read alongside the export. A run still going has not logged them all. + const unscanned = + source.kind === "scan" && processing.status === "completed" + ? (await initializeScanner(config)).unscannedPages( + source.scanId, + source.scanRunId, + ) + : Promise.resolve([]); try { reportLink ??= await reportFetcher.getReportLink(source, { exportTaskId, @@ -1097,6 +1072,16 @@ async function fetchAccessibilityIssues( exportPendingMessages(source, error, pageArg), ); } + // A run or build still going may have no report to export yet: the export fails, which + // its words ("Failed to fetch report: null") make read like a fault. + if (stillProcessing) { + return noReportYetResponse( + reportFetcher, + source, + processing.status ?? "processing", + preface, + ); + } throw error; } @@ -1112,15 +1097,37 @@ async function fetchAccessibilityIssues( unlinked_duplicates, hidden_issues, page, + error_pages = [], } = await parseAccessibilityReportFromCSV(reportLink, { nextPage: cursor, pageUrl, }); + // A redirected page was scanned, where it redirected to; a page that showed an error page + // is said once, with the error page's issues. + const errorPageUrls = new Set(error_pages.map((p) => samePage(p.url))); + const notScanned = (await unscanned).filter( + (p) => p.status !== "redirect" && !errorPageUrls.has(samePage(p.pageUrl)), + ); return createSuccessResponse([ ...preface, ...processingNote(processing), - ...(page ? pageNote(page) : []), + // What the run did not check, said on the first page with the whole report's counts. + ...(cursor + ? [] + : [ + ...errorPagesNote( + error_pages, + source.kind === "build" ? "run the tests again" : "scan again", + ), + ...(notScanned.length + ? [ + `⚠️ ${pageReasons(notScanned, ["was not scanned, so its issues are not in this report", "were not scanned, so their issues are not in this report"])}`, + ] + : []), + ]), + // A page that did not load has no issues of its own to count. + ...(page && error_pages.length === 0 ? pageNote(page) : []), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, // The whole report's counts, said on its first page. ...(cursor ? [] : webSummaryNote(summary, total_issues)), @@ -1128,7 +1135,14 @@ async function fetchAccessibilityIssues( fullReportLine(reportLink), WEB_ISSUES_NOTE, `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: records, rules }))}`, - closingMessage(next_page, remaining, source, stillProcessing, pageArg), + closingMessage( + next_page, + remaining, + source, + stillProcessing, + pageArg, + error_pages.length > 0 || notScanned.length > 0, + ), ]); } @@ -1157,65 +1171,11 @@ function pageNote(page: { } /** - * The pages of the newer run that did not load (an error page was scanned in their place), - * or whose earlier issues all gave way to new ones. Neither the scan nor the dashboard says - * so, and their real issues would otherwise read as fixed. - */ -function pagesNotLoadedNote( - diff: { - not_checked?: unknown[]; - failed_pages?: string[]; - not_checked_earlier?: unknown[]; - failed_pages_earlier?: string[]; - changed_pages?: string[]; - }, - runId: string, - baseRunId: string, -): string[] { - const pages = (urls: string[]) => - wrapUntrusted("report page addresses", JSON.stringify(urls)); - // Worded for one page or several, one issue or several. - const one = (count: number, singular: string, plural: string) => - count === 1 ? singular : plural; - const they = (n: number) => one(n, "it", "they"); - const didNotLoad = (urls: string[], run: string) => { - const n = urls.length; - return `⚠️ ${one(n, "1 page", `${n} pages`)} did not load in run ${run}: ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place. The ${one(n, "error page's", "error pages'")} issues are not counted as`; - }; - // A page that failed in both runs has no issues of its own in either. - const foundThere = (count: number, run: string, notCountedAs: string) => - count > 0 - ? `, and the ${one(count, "1 issue", `${count} issues`)} found there in run ${run} ${one(count, "is", "are")} not counted as ${notCountedAs}` - : ""; - const failed = diff.failed_pages ?? []; - const failedEarlier = diff.failed_pages_earlier ?? []; - const changed = diff.changed_pages ?? []; - return [ - ...(failed.length > 0 - ? [ - `${didNotLoad(failed, runId)} new${foundThere(diff.not_checked?.length ?? 0, baseRunId, "resolved (listed as not checked)")}. Scan again once ${they(failed.length)} ${one(failed.length, "loads", "load")}. ${pages(failed)}`, - ] - : []), - ...(failedEarlier.length > 0 - ? [ - `${didNotLoad(failedEarlier, baseRunId)} resolved${foundThere(diff.not_checked_earlier?.length ?? 0, runId, "new (listed as not checked against the earlier run)")}. Compare with a run in which ${they(failedEarlier.length)} loaded. ${pages(failedEarlier)}`, - ] - : []), - ...(changed.length > 0 - ? [ - `⚠️ None of the earlier issues on ${one(changed.length, "1 page remain, and it has", `${changed.length} pages remain, and they have`)} new ones: ${they(changed.length)} may not have loaded, or loaded something else, in run ${runId}. Check ${one(changed.length, "it", "them")} before calling those issues fixed. ${pages(changed)}`, - ] - : []), - ]; -} - -/** At most this many new and resolved issues are listed in a comparison. */ -const COMPARE_LIST_LIMIT = 50; - -/** - * A Website Scanner run against an earlier run of the same scan: the run before it - * ("previous") or a given scanRunId. Issues are matched by rule, element and page; against - * the previous run, the dashboard's own counts (the run's changesSinceLastRun) are given too. + * A Website Scanner run against another run of the same scan: the run before it + * ("previous"), or one named by its scanRunId, report link or build id. The older run is + * "before", whichever was passed as compareWith. Issues are matched by rule, element and + * page; against the run just before, the dashboard's own counts (the run's + * changesSinceLastRun) are given too. */ async function compareScanRuns( requested: RequestedSource, @@ -1236,62 +1196,130 @@ async function compareScanRuns( 'compareWith compares Website Scanner runs: pass the newer run (scanId and scanRunId, or its report link) and compareWith: "previous" or the older run\'s scanRunId.', ); } - const previous = await reportFetcher.previousScanRun( - source.scanId, - source.scanRunId, - ); - const baseRunId = - compareWith === "previous" ? previous && String(previous.id) : compareWith; - if (!baseRunId) { - return createErrorResponse( - `Run ${source.scanRunId} is the first run of scan ${source.scanId}; there is no earlier run to compare it with.`, + const preface = + requested.kind === "link" + ? [`Resolved the report link to ${describeSource(source)}.`] + : []; + + let after = source.scanRunId; + let before: string; + let beforeDate: string | undefined; + let againstPrevious: boolean; + let reversed = false; + if (compareWith === "previous") { + const previous = await reportFetcher.previousScanRun( + source.scanId, + source.scanRunId, ); + if (!previous) { + return createErrorResponse( + `Run ${source.scanRunId} is the first run of scan ${source.scanId}; there is no earlier run to compare it with.`, + ); + } + before = String(previous.id); + beforeDate = previous.scanDate; + againstPrevious = true; + } else { + const other = await scanRunToCompare(reportFetcher, source, compareWith); + if ("content" in other) return other; + if (other.scanRunId === source.scanRunId) { + return createErrorResponse( + `compareWith names this run itself. Pass another run's scanRunId or link, or "previous".`, + ); + } + const { runs, complete } = await reportFetcher.scanRuns(source.scanId); + const at = (id: string) => runs.findIndex((run) => String(run.id) === id); + // A run id the scan's whole run list does not hold is another scan's run: the API would + // refuse it with "Scan logs not found", which reads like a fault. + if (complete && at(other.scanRunId) < 0) { + return readBothToCompare( + `Run ${other.scanRunId} is not a run of scan ${source.scanId}, so it is another scan's run: only runs of one scan are compared here.`, + false, + [ + sourceArgs(source), + `scanRunId: "${other.scanRunId}" and its own scan's scanId`, + ], + ); + } + // The older run is "before": by the run list's order, or by id when either is not in it. + const [selfAt, otherAt] = [at(source.scanRunId), at(other.scanRunId)]; + reversed = + selfAt >= 0 && otherAt >= 0 + ? otherAt < selfAt + : Number(other.scanRunId) > Number(source.scanRunId); + [after, before] = reversed + ? [other.scanRunId, source.scanRunId] + : [source.scanRunId, other.scanRunId]; + againstPrevious = at(after) >= 0 && at(before) === at(after) + 1; + beforeDate = runs[at(before)]?.scanDate; } - const base: IssueSource = { ...source, scanRunId: baseRunId }; - const againstPrevious = - previous !== undefined && String(previous.id) === baseRunId; + const run = (scanRunId: string): IssueSource => ({ ...source, scanRunId }); - let links: [string, string]; - try { - links = await Promise.all([ - reportFetcher.getReportLink(source, { onWait: exportProgress(context) }), - reportFetcher.getReportLink(base, {}), - ]); - } catch (error) { - if (error instanceof ExportPendingError) { - return createSuccessResponse([ - `The CSV exports of these runs are still being generated (waited ${formatDuration(error.waitedMs)}). Call fetchAccessibilityIssues again with the same arguments in about a minute; the exports continue where they are.`, - ]); + // A failed or cancelled run has no results and exports an empty report: compared with it, + // every issue of the other run would read as resolved (or new). + const [status, beforeStatus] = await Promise.all([ + reportFetcher.reportStatus(run(after)), + reportFetcher.reportStatus(run(before)), + ]); + const noResults = (runStatus: string | undefined) => + runStatus === "failed" || runStatus === "cancelled"; + if (noResults(status) || noResults(beforeStatus)) { + const [failedRun, otherRun, failedStatus] = noResults(status) + ? [after, before, status] + : [before, after, beforeStatus]; + return failedScanRunResponse( + config, + source.scanId, + failedRun, + preface, + () => + `compared with it, every issue of run ${otherRun} would read as ${failedRun === after ? "resolved" : "new"}, so the runs were not compared.`, + "Compare two runs that completed, or fix the cause and scan again.", + failedStatus === "cancelled" ? "cancelled" : "failed", + ); + } + for (const [id, runStatus] of [ + [after, status], + [before, beforeStatus], + ] as const) { + if (runStatus && PROCESSING_STATUSES.has(runStatus)) { + return notReadyToCompare( + `Run ${id} of Website Scanner scan ${source.scanId}`, + runStatus, + preface, + ); } - throw error; } - const [after, before] = await Promise.all(links.map(loadMainIssues)); - const diff = compareIssues(before, after); + + const read = await diffExports( + reportFetcher, + run(after), + run(before), + "runs", + context, + ); + if ("content" in read) return read; + const { links, diff } = read; const changes = againstPrevious - ? await reportFetcher.changesSinceLastRun(source.scanId, source.scanRunId) + ? await reportFetcher.changesSinceLastRun(source.scanId, after) : undefined; const baseLabel = againstPrevious - ? `the run before it, run ${baseRunId}${previous?.scanDate ? ` (${previous.scanDate})` : ""}` - : `run ${baseRunId}`; + ? `the run before it, run ${before}${beforeDate ? ` (${beforeDate})` : ""}` + : `run ${before}`; const matched = `${diff.added.length} new, ${diff.resolved.length} resolved, ${diff.retained} still open`; - const listed = (issues: typeof diff.added) => - wrapUntrusted( - "accessibility comparison", - JSON.stringify(issues.slice(0, COMPARE_LIST_LIMIT), null, 2), - ) + - (issues.length > COMPARE_LIST_LIMIT - ? `\n(${issues.length - COMPARE_LIST_LIMIT} more not listed; see the CSV links.)` - : ""); + const listed = (issues: RunIssue[]) => + comparedList(issues, issues.length, "; see the CSV links"); return createSuccessResponse([ - ...(requested.kind === "link" - ? [`Resolved the report link to ${describeSource(source)}.`] + ...preface, + `Compared Website Scanner scan ${source.scanId}: run ${after} with ${baseLabel}.`, + ...(reversed + ? ["compareWith was the newer run, so it is used as 'after'."] : []), - `Compared Website Scanner scan ${source.scanId}: run ${source.scanRunId} with ${baseLabel}.`, changes ? `The dashboard's comparison: ${changes.newIssues} new, ${changes.resolvedIssues} resolved, ${changes.retainedIssues} still open. Matching the issues by rule, element and page gives ${matched}.` : `Matching the issues by rule, element and page: ${matched}.`, - ...pagesNotLoadedNote(diff, source.scanRunId, baseRunId), + ...pagesNotLoadedNote(diff, `run ${after}`, `run ${before}`, "run"), `New issues: ${listed(diff.added)}`, `Resolved issues: ${listed(diff.resolved)}`, ...(diff.not_checked?.length @@ -1299,27 +1327,13 @@ async function compareScanRuns( : []), ...(diff.not_checked_earlier?.length ? [ - `Not checked against the earlier run (page did not load in run ${baseRunId}): ${listed(diff.not_checked_earlier)}`, + `Not checked against the earlier run (page did not load in run ${before}): ${listed(diff.not_checked_earlier)}`, ] : []), `CSV of each run, valid for about 12 hours: newer ${links[0]} , older ${links[1]}`, ]); } -/** The WCAG versions and levels Website Scanner checks against, and RGAA 4.1, by name. */ -const WCAG_VERSIONS = { - wcag2a: "WCAG 2.0 A", - wcag2aa: "WCAG 2.0 AA", - wcag2aaa: "WCAG 2.0 AAA", - wcag21a: "WCAG 2.1 A", - wcag21aa: "WCAG 2.1 AA", - wcag21aaa: "WCAG 2.1 AAA", - wcag22a: "WCAG 2.2 A", - wcag22aa: "WCAG 2.2 AA", - wcag22aaa: "WCAG 2.2 AAA", - rgaa41: "RGAA 4.1", -} as const; - /** * Why advanced or experimental rules came back off when they were not asked off. Both are * on unless turned off, and on an API scan a plan without them turns them off instead of @@ -1370,9 +1384,6 @@ function scanSettingsNote( asked: ScanSettings, applied: ScanSettings | undefined, ): string { - const wcagName = (value: string) => - WCAG_VERSIONS[value as keyof typeof WCAG_VERSIONS] ?? value; - const onOff = (value: boolean) => (value ? "on" : "off"); const describe = (settings: ScanSettings) => [ settings.wcagVersion !== undefined ? wcagName(settings.wcagVersion) : "", @@ -1405,12 +1416,7 @@ function scanSettingsNote( } async function executeAccessibilityScan( - args: { - name: string; - pageURL: string; - authConfigId?: number; - localIdentifier?: string; - } & ScanSettings, + args: ScanArgs, context: ScanProgressContext, server: McpServer, config: BrowserStackConfig, @@ -1422,26 +1428,53 @@ async function executeAccessibilityScan( undefined, config, ); - return await runAccessibilityScan( - args.name, - args.pageURL, - context, - config, - args.authConfigId, - { - wcagVersion: args.wcagVersion, - bestPractices: args.bestPractices, - needsReview: args.needsReview, - advancedRules: args.advancedRules, - experimentalRules: args.experimentalRules, - }, - args.localIdentifier, - ); + if (args.scanId !== undefined) { + return await rerunAccessibilityScan( + { ...args, scanId: args.scanId }, + context, + config, + ); + } + return await startNewScan(args, context, config); } catch (error) { + // An expected limit, such as a local URL the hosted MCP cannot reach, the plan's scan + // limit, or a re-run that cannot start: its message says what to do instead, so it is a + // plain result, not an error to report. + if (error instanceof ScanRefusedError) { + trackMCP( + "startAccessibilityScan", + server.server.getClientVersion()!, + error, + config, + ); + return { content: [{ type: "text", text: error.message }] }; + } return handleMCPError("startAccessibilityScan", server, config, error); } } +function startNewScan( + args: ScanArgs, + context: ScanProgressContext, + config: BrowserStackConfig, +): Promise { + return runAccessibilityScan( + args.name, + args.pageURL, + context, + config, + args.authConfigId, + { + wcagVersion: args.wcagVersion, + bestPractices: args.bestPractices, + needsReview: args.needsReview, + advancedRules: args.advancedRules, + experimentalRules: args.experimentalRules, + }, + args.localIdentifier, + ); +} + function validateFormAuthArgs(args: AuthConfigArgs): args is FormAuthArgs { return ( args.type === "form" && @@ -1559,19 +1592,6 @@ async function executeGetAuthConfig( } } -/** - * Where to see a scan in the dashboard: its own page, as the scans list links it. A scan's - * page is named by its build, so a link made from the scan's name does not open it. - */ -function scanDashboardNote( - dashboardLink: string | undefined, - scanId: string, -): string { - return dashboardLink - ? `Its page in the Accessibility dashboard: ${dashboardLink}` - : `See it under Website Scanner in the Accessibility dashboard (scan ID ${scanId}).`; -} - /** Why the scan's name differs from the one asked for, as the start of a sentence. */ function renameReason( name: string, @@ -1586,9 +1606,6 @@ function renameReason( return `A scan named "${name}" already exists on this account`; } -/** How many failed pages a failure result lists before summing up the rest. */ -const FAILED_PAGES_LISTED = 10; - /** * A failed run, with the reason the API gives for it: the run's own and each page's. It used * to say only "The dashboard says why", which left an agent nothing to tell the user. @@ -1601,24 +1618,10 @@ function createScanFailureResponse( dashboardLink: string | undefined, details: ScanFailureDetails = { pages: [] }, ): CallToolResult { - const { reason, pages } = details; - const listed = pages.slice(0, FAILED_PAGES_LISTED); - const why = [ - reason ? `Why: ${reason}${/[.!?]$/.test(reason) ? "" : "."}` : "", - pages.length - ? [ - `${pages.length} ${pages.length === 1 ? "page" : "pages"} could not be scanned:`, - ...listed.map((page) => `- ${page.pageUrl}: ${page.reason}`), - ...(pages.length > listed.length - ? [`- and ${pages.length - listed.length} more.`] - : []), - ].join("\n") - : "", - ].filter(Boolean); return createErrorResponse( [ `❌ Accessibility scan "${name}" failed with status: ${status}. Scan ID: ${scanId} and Scan Run ID: ${scanRunId}.`, - ...(why.length ? why : ["The dashboard says why."]), + ...failureReasons(details), scanDashboardNote(dashboardLink, scanId), ].join("\n"), ); @@ -1635,11 +1638,12 @@ function createScanStillRunningResponse( scanRunId: string, dashboardLink: string | undefined, waitedMs: number, + rerun = false, ): CallToolResult { return createSuccessResponse([ `Accessibility scan "${name}" is still running after ${Math.round(waitedMs / 60000)} minutes, so this call stopped waiting for it. The scan was not cancelled: it carries on, and its issues can be read once it finishes. ${scanDashboardNote(dashboardLink, scanId)}`, `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, - `To read its issues, call fetchAccessibilityIssues with scanId "${scanId}" and scanRunId "${scanRunId}" once it has finished. Do not start the scan again: that starts a second scan.`, + `To read its issues, call fetchAccessibilityIssues with scanId "${scanId}" and scanRunId "${scanRunId}" once it has finished. ${rerun ? "Do not re-run the scan before this run finishes." : "Do not start the scan again: that starts a second scan."}`, ]); } @@ -1657,6 +1661,8 @@ function createScanSuccessResponse( const messages = [ `Accessibility scan "${name}" completed. ${scanDashboardNote(dashboardLink, scanId)}`, `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, + // The parser leaves an error page's rows out of the count; this says why. + ...errorPagesNote(report.error_pages ?? [], "scan again"), scanSettingsNote(settings, applied), fullReportLine(reportUrl), `We found ${report.total_issues} issues. Below are the details of the ${report.page_length} most critical issues.`, @@ -1679,69 +1685,49 @@ function createScanSuccessResponse( return createSuccessResponse(messages); } -async function runAccessibilityScan( - name: string, - pageURL: string, - context: ScanProgressContext, - config: BrowserStackConfig, - authConfigId?: number, - settings: ScanSettings = {}, - localIdentifier?: string, -): Promise { - const scanner = await initializeScanner(config); - - const startResp = await scanner.startScan( - name, - [pageURL], - authConfigId, - settings, - localIdentifier, - ); - const scanId = startResp.data!.id; - const scanRunId = startResp.data!.scanRunId; - const scanName = startResp.scanName ?? name; - // Said first, so the name asked for is not taken for the scan's. - const withRenameNote = (result: CallToolResult): CallToolResult => - scanName === name - ? result - : { - ...result, - content: [ - { - type: "text" as const, - text: `${renameReason(name, startResp.nameChanges ?? ["taken"])}, so this scan was started as "${scanName}".\n`, - }, - ...result.content, - ], - }; - - await notifyScanProgress( - context, - `Accessibility scan "${scanName}" started`, - 0, - ); +type ScanOutcome = "completed" | "still running" | "failed"; +/** + * Waits for a run that has started and gives its result: its issues, a failure with the + * reasons the API gives, or its ids once the wait runs out. The same for a new scan and a + * re-run, so a re-run reads like the scan it repeats. + */ +async function scanOutcome( + scanner: AccessibilityScanner, + config: BrowserStackConfig, + context: ScanProgressContext, + scan: { + name: string; + scanId: string; + scanRunId: string; + settings: ScanSettings; + rerun?: boolean; + }, +): Promise<[CallToolResult, ScanOutcome]> { + const { name: scanName, scanId, scanRunId, settings } = scan; let status: string; try { status = await scanner.waitUntilComplete(scanId, scanRunId, context); } catch (error) { if (!(error instanceof ScanTimeoutError)) throw error; - return withRenameNote( + return [ createScanStillRunningResponse( scanName, scanId, scanRunId, await scanner.dashboardLink(scanId), error.waitedMs, + scan.rerun, ), - ); + "still running", + ]; } if (status !== "completed") { const [dashboardLink, details] = await Promise.all([ scanner.dashboardLink(scanId), scanner.failureDetails(scanId, scanRunId), ]); - return withRenameNote( + return [ createScanFailureResponse( scanName, status, @@ -1750,7 +1736,8 @@ async function runAccessibilityScan( dashboardLink, details, ), - ); + "failed", + ]; } const [applied, dashboardLink] = await Promise.all([ @@ -1766,7 +1753,7 @@ async function runAccessibilityScan( ); } catch (error) { if (error instanceof ExportPendingError) { - return withRenameNote( + return [ createSuccessResponse([ `Accessibility scan "${scanName}" completed. ${scanDashboardNote(dashboardLink, scanId)}`, `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, @@ -1781,12 +1768,13 @@ async function runAccessibilityScan( "", ), ]), - ); + "completed", + ]; } throw error; } - return withRenameNote( + return [ createScanSuccessResponse( scanName, await parseAccessibilityReportFromCSV(reportLink), @@ -1797,6 +1785,150 @@ async function runAccessibilityScan( settings, applied, ), + "completed", + ]; +} + +async function runAccessibilityScan( + name: string, + pageURL: string, + context: ScanProgressContext, + config: BrowserStackConfig, + authConfigId?: number, + settings: ScanSettings = {}, + localIdentifier?: string, +): Promise { + const scanner = await initializeScanner(config); + + const startResp = await scanner.startScan( + name, + [pageURL], + authConfigId, + settings, + localIdentifier, + ); + const scanId = startResp.data!.id; + const scanRunId = startResp.data!.scanRunId; + const scanName = startResp.scanName ?? name; + + await notifyScanProgress( + context, + `Accessibility scan "${scanName}" started`, + 0, + ); + + const [result] = await scanOutcome(scanner, config, context, { + name: scanName, + scanId, + scanRunId, + settings, + }); + // The rename is said first, so the name asked for is not taken for the scan's. + return withNotes( + result, + scanName === name + ? [] + : [ + `${renameReason(name, startResp.nameChanges ?? ["taken"])}, so this scan was started as "${scanName}".`, + ], + // A local page this MCP tunnelled itself is saved with an identifier nobody was shown, + // so a re-run could not reach it: no hint to re-run it. + isLocalURL(pageURL) && !localIdentifier ? [] : [rescanHint(String(scanId))], + ); +} + +/** + * A re-run of the scan the caller names, when it is for this same page: a new run of that + * scan, with its saved settings, which can be compared with its earlier runs. A scan for + * another page gets a new scan, as without scanId, and settings passed that the saved ones + * do not match stop the run, since a re-run cannot take them. + */ +async function rerunAccessibilityScan( + args: ScanArgs & { scanId: string }, + context: ScanProgressContext, + config: BrowserStackConfig, +): Promise { + const scanner = await initializeScanner(config); + const { scanId, pageURL } = args; + const saved = await scanner.savedScan(scanId); + if (saved.urlList.length !== 1 || !sameScanPage(saved.urlList[0], pageURL)) { + return withNotes(await startNewScan(args, context, config), [ + otherPageNote(scanId, saved.urlList, pageURL), + ]); + } + + // The saved scan reaches its page through a tunnel whose identifier the API does not give + // back: without one passed, the re-run could not reach the page. + if (saved.localTestingEnabled && args.localIdentifier === undefined) { + return withNotes(await startNewScan(args, context, config), [ + localScanNotRerunNote(scanId), + ]); + } + + const name = saved.name ?? args.name; + // The saved name is the account's, which another user may have set: quoted, not inlined. + const nameLine = saved.name + ? [`Scan name: ${wrapUntrusted("saved scan name", saved.name)}`] + : []; + const differences = savedSettingDifferences(args, saved); + if (differences.length) { + const one = differences.length === 1; + // A plain result: nothing failed, and the call only has to be made again one way or + // the other. + return createSuccessResponse([ + [ + `Scan ${scanId} was not re-run: a re-run uses the scan's saved settings, and ${one ? "this one differs" : "these differ"} from what was passed:`, + ...differences.map((difference) => `- ${difference}`), + `To re-run it with its saved settings, call startAccessibilityScan again with scanId "${scanId}" and without ${one ? "that setting" : "those settings"}. To scan with the settings passed, call it without scanId: that starts a new scan, whose runs can't be compared with this scan's.`, + ...nameLine, + ].join("\n"), + ]); + } + + const { scanRunId } = await scanner.rerunScan(scanId); + const passedName = args.name.trim(); + const unchecked = uncheckedSettings(args, saved); + const lead = [ + `Started a new run (${scanRunId}) of scan ${scanId} with its saved settings.`, + ...nameLine, + ...(passedName && + passedName !== name && + cleanScanName(passedName, "") !== name + ? [ + `The scan keeps its saved name; the name "${passedName}" was not used.`, + ] + : []), + ...(saved.localTestingEnabled + ? [localRerunNote(args.localIdentifier)] + : []), + ...(unchecked.length + ? [ + `${unchecked.join(", ")} could not be checked against the scan's saved settings, which the API does not give back; the run uses the saved ones.`, + ] + : []), + ...unusedLoginAndTunnel(args, saved), + ]; + await notifyScanProgress( + context, + `Run ${scanRunId} of accessibility scan "${name}" started`, + 0, + ); + const [result, outcome] = await scanOutcome(scanner, config, context, { + name, + scanId, + scanRunId, + settings: saved.settings, + rerun: true, + }); + const compare = compareHint(scanId, scanRunId); + return withNotes( + result, + lead, + outcome === "completed" + ? [compare] + : outcome === "still running" + ? [`Once it finishes: ${compare[0].toLowerCase()}${compare.slice(1)}`] + : [], ); } @@ -1832,21 +1964,27 @@ export default function addAccessibilityTools( "startAccessibilityScan", "Scan a web page URL for accessibility issues and get a CSV report. For a mobile app, run it on App Automate with accessibility on (setupBrowserStackAppAutomateTests), then fetchAccessibilityIssues.", { - name: z.string().describe("Name of the accessibility scan"), - pageURL: z + name: z.string().describe("Name of the scan"), + pageURL: z.string().describe("URL of the page to scan"), + scanId: z .string() - .describe("The web page URL to scan for accessibility issues"), + .trim() + .min(1) + .optional() + .describe( + "Only to re-run an earlier scan of this same page (e.g. after a fix), with its saved settings, so runs can be compared.", + ), authConfigId: z .number() .optional() - .describe("Optional auth config ID for authenticated scans"), + .describe("Auth config ID, for a page behind a login"), localIdentifier: z .string() .trim() .min(1) .optional() .describe( - "Identifier of a BrowserStack Local tunnel you started, for localhost or internal URLs.", + "ID of a BrowserStack Local tunnel you started, for localhost or internal URLs.", ), advancedRules: z .boolean() @@ -1893,13 +2031,11 @@ export default function addAccessibilityTools( "createAccessibilityAuthConfig", "Create an authentication configuration for accessibility scans. Supports both form-based and basic authentication.", { - name: z.string().describe("Name for the auth configuration"), + name: z.string().describe("Name for the auth config"), type: z .enum(["form", "basic"]) - .describe( - "Authentication type: 'form' for form-based auth, 'basic' for HTTP basic auth", - ), - url: z.string().describe("URL of the authentication page"), + .describe("'form' for a login form, 'basic' for HTTP basic auth"), + url: z.string().describe("URL of the login page"), username: z .string() .optional() @@ -1985,7 +2121,7 @@ export default function addAccessibilityTools( "getAccessibilityAuthConfig", "Get an accessibility auth config by ID." + NEEDS_A11Y_CONFIG_ID, { - configId: z.number().describe("ID of the auth configuration to retrieve"), + configId: z.number().describe("ID of the auth config"), }, { title: "Get Accessibility Auth Config", @@ -2049,7 +2185,7 @@ export default function addAccessibilityTools( .number() .optional() .describe( - "Pagination offset (default 0); with compareWith, issues already listed.", + "The next_page value from the previous answer (default 0), not an issue index.", ), pageUrl: z .string() @@ -2059,7 +2195,7 @@ export default function addAccessibilityTools( .string() .optional() .describe( - '"previous", an older scanRunId, or an earlier app build id/link; lists new and resolved issues.', + '"previous" (any report), or a run of the same scan or a web/app build: its id or link.', ), exportTaskId: z .string() diff --git a/src/tools/accessiblity-utils/app-report-fetcher.ts b/src/tools/accessiblity-utils/app-report-fetcher.ts index 58511489..01b09301 100644 --- a/src/tools/accessiblity-utils/app-report-fetcher.ts +++ b/src/tools/accessiblity-utils/app-report-fetcher.ts @@ -2,7 +2,7 @@ import { apiClient } from "../../lib/apiClient.js"; import { SessionType } from "../../lib/constants.js"; import { sessionsListUrl } from "../automate-utils/list-session-ids.js"; import { getAutomationBaseUrl } from "../rca-agent-utils/constants.js"; -import { mostCommon, pageOf, sortBySeverity } from "./paginate.js"; +import { mostCommon, pageOf, sortBySeverity, type Page } from "./paginate.js"; import { getRegionalHost, isRegionalHostname, @@ -24,6 +24,30 @@ export const manualApiBase = () => const SNAPSHOT_PAGE_SIZE = 50; export const MAX_SNAPSHOT_PAGES = 20; +/** + * Manual reports are listed newest first, 100 a page; a report and the one before it are + * found among the newest pages. The list's search matches a report's name, app or package. + */ +const MAX_MANUAL_REPORT_PAGES = 3; +const MANUAL_REPORT_PAGE_SIZE = 100; + +/** A manual report as the report list gives it. */ +interface ManualReportRow { + id: string; + created_at?: string; + /** complete | in_progress */ + status?: string; + app_details?: { name?: string; package_name?: string }; +} + +/** An earlier manual report of the same app. */ +export interface EarlierManualReport { + reportId: string; + /** When it was recorded, as the API writes it. */ + time?: string; + link: string; +} + /** A dashboard link names a build by number; a build someone just opened is among the newest. */ export const MAX_APP_BUILD_LIST_PAGES = 3; const BUILD_LIST_PAGE_SIZE = 30; @@ -442,6 +466,58 @@ export function compactAppIssues(all: AppAccessibilityIssue[]): { return { rows, context: { rules, screens } }; } +/** + * A value's length as an answer prints it: indented JSON one level inside its list (an + * issue in `issues`, a rule in `rules`, a screenshot in `screens`), after its key in an + * object, with the separator after it. + */ +function printedLength(value: unknown, key?: string): number { + const text = JSON.stringify(value, null, 2) ?? ""; + const keyed = key === undefined ? 0 : JSON.stringify(key).length + 2; + return keyed + text.length + 4 * text.split("\n").length + 2; +} + +/** + * How many characters of an app page's issues, rules and screens, as printed, one answer + * holds by default. The rest of the answer (its summary, notes and the untrusted-data + * wrappers) adds about 2,000 to 4,000, so the whole stays under about 40,000: within Claude + * Code's 50,000-character limit on a tool's answer, above which it saves the answer to a + * file instead of showing it. + */ +export const APP_PAGE_CHARACTERS = 35_000; + +/** + * One page of an app build's or manual report's issues. The answer prints them as indented + * JSON with the rules and screens they name, so each issue counts against the page as it is + * printed, with its rule's text and its screen's screenshot the first time the page names + * them. Counting the bare issues alone let a 70-issue report's page reach 51,000 characters. + * The cursor still counts the bare issues. + */ +function appIssuePage( + rows: AppIssueRow[], + { rules, screens }: AppIssuePageContext, + cursor: number, + maxCharacterLength = APP_PAGE_CHARACTERS, +): Page { + const named = new Set(); + return pageOf(rows, { + nextPage: cursor, + maxCharacterLength, + extraLength: (row) => { + let more = printedLength(row) - JSON.stringify(row).length; + if (!named.has(`rule:${row.rule_id}`)) { + named.add(`rule:${row.rule_id}`); + more += printedLength(rules[row.rule_id], row.rule_id); + } + if (screens[row.screen] && !named.has(`screen:${row.screen}`)) { + named.add(`screen:${row.screen}`); + more += printedLength(screens[row.screen], row.screen); + } + return more; + }, + }); +} + /** The rules and screens a page's issues name, and nothing else. */ export function contextOf( records: AppIssueRow[], @@ -696,6 +772,17 @@ function buildOfDashboardLink( }; } +/** + * A lookup that could not be answered (an HTTP failure other than the API's 404), as opposed + * to an answer about the build: a caller probing an id can try the next API instead. + */ +export class BuildLookupError extends Error { + constructor(message: string) { + super(message); + this.name = "BuildLookupError"; + } +} + export class AppAccessibilityApi { constructor( private readonly headers: () => Record | undefined, @@ -768,6 +855,19 @@ export class AppAccessibilityApi { : this.buildOfNativeBuild(ids.buildId!); } + /** + * The accessibility build of an App Automate build named by its own id (its hashed_id), as + * for its link; undefined when App Automate knows no build by that id. + */ + async buildIdOfAppAutomateBuild( + buildId: string, + ): Promise { + const session = await this.anySdkSession(buildId); + if (session) return this.buildOfSdkSession(session); + const native = await this.nativeBuild(buildId); + return native ? this.buildOfNativeBuild(buildId, native) : undefined; + } + private async buildOfSdkSession(sessionId: string): Promise { const resp = await apiClient.get({ url: `${getAutomationBaseUrl()}/ext/v1/sessions/${encodeURIComponent(sessionId)}/testCode`, @@ -801,7 +901,7 @@ export class AppAccessibilityApi { raise_error: false, }); if (!resp.ok && resp.status !== 404) { - throw new Error( + throw new BuildLookupError( `Could not list the sessions of App Automate build ${buildId}: HTTP ${resp.status}`, ); } @@ -815,8 +915,11 @@ export class AppAccessibilityApi { * builds; the name, read from the account's build list, tells runs that started together * apart, and finds a run older than those. */ - private async buildOfNativeBuild(buildId: string): Promise { - const native = await this.nativeBuild(buildId); + private async buildOfNativeBuild( + buildId: string, + known?: { project?: string; startedAt?: number }, + ): Promise { + const native = known ?? (await this.nativeBuild(buildId)); if (!native?.project) { throw new Error( `No session found for App Automate build ${buildId}, and it is not an Espresso or XCUITest build with a project, so it cannot be mapped to its accessibility report. Pass the build id the BrowserStack SDK printed as "testhub id" as buildId, or the build's App Accessibility dashboard link as reportUrl`, @@ -868,7 +971,7 @@ export class AppAccessibilityApi { } // Each framework's API answers 404 for the other's builds. if (resp.status !== 404) { - throw new Error( + throw new BuildLookupError( `Could not read App Automate build ${buildId} from the ${framework} API: HTTP ${resp.status}`, ); } @@ -979,6 +1082,64 @@ export class AppAccessibilityApi { return resp.data; } + /** The manual reports whose name, app or package contains `search`, newest first, a page at a time. */ + private async *manualReports( + search: string, + ): AsyncGenerator { + for (let page = 1; page <= MAX_MANUAL_REPORT_PAGES; page++) { + const data = await this.listing( + `${manualApiBase()}/reports?report_name=${encodeURIComponent(search)}&page_size=${MANUAL_REPORT_PAGE_SIZE}&page=${page}`, + `page ${page} of the manual App Accessibility reports`, + ); + yield data.reports ?? []; + if (!data.info?.next) return; + } + } + + /** + * The newest finished manual report of the same app recorded before this one, or + * undefined when none is among the newest. The report's details do not name its app: the + * report list does, where the report is found by its name; the app's reports are then + * listed by its package (or, without one, its name). + */ + async previousManualReport( + reportId: string, + ): Promise { + const name: string | undefined = (await this.manualReport(reportId)) + ?.meta_data?.name; + if (!name) return undefined; + let own: ManualReportRow | undefined; + for await (const rows of this.manualReports(name)) { + own = rows.find((row) => row.id === reportId); + if (own) break; + } + const pkg = own?.app_details?.package_name; + const app = pkg ?? own?.app_details?.name; + if (!own || !app) return undefined; + const sameApp = (row: ManualReportRow) => + pkg + ? row.app_details?.package_name === pkg + : row.app_details?.name === app; + const recorded = (row: ManualReportRow) => Date.parse(row.created_at ?? ""); + for await (const rows of this.manualReports(app)) { + const earlier = rows.find( + (row) => + row.id !== reportId && + sameApp(row) && + row.status !== "in_progress" && + recorded(row) < recorded(own), + ); + if (earlier) { + return { + reportId: earlier.id, + time: earlier.created_at, + link: `${appDashboard()}/report/${earlier.id}`, + }; + } + } + return undefined; + } + /** * A manual report's screens as the dashboard reads them, duplicates left out and each * issue with its element, fix and measurements; undefined while that route answers an @@ -1063,7 +1224,7 @@ export class AppAccessibilityApi { const summary = detail.data?.issue_summary ?? {}; const all = flattenAppSnapshots(details); const { rows, context } = compactAppIssues(all); - const page = pageOf(rows, { nextPage: cursor, maxCharacterLength }); + const page = appIssuePage(rows, context, cursor, maxCharacterLength); return { ...page, ...contextOf(page.records, context), @@ -1282,7 +1443,7 @@ export class AppAccessibilityApi { await this.readBuild(buildId); const all = flattenAppSnapshots(details); const { rows, context } = compactAppIssues(all); - const page = pageOf(rows, { nextPage: cursor, maxCharacterLength }); + const page = appIssuePage(rows, context, cursor, maxCharacterLength); return { ...page, ...contextOf(page.records, context), diff --git a/src/tools/accessiblity-utils/compare-lists.ts b/src/tools/accessiblity-utils/compare-lists.ts new file mode 100644 index 00000000..8bd246db --- /dev/null +++ b/src/tools/accessiblity-utils/compare-lists.ts @@ -0,0 +1,131 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { + ExportPendingError, + type AccessibilityReportFetcher, + type IssueSource, +} from "./report-fetcher.js"; +import { + compareIssues, + loadMainIssues, + type RunIssue, +} from "./report-parser.js"; +import { wrapUntrusted } from "../../lib/untrusted-content.js"; +import { + createSuccessResponse, + exportProgress, + formatDuration, + type ScanProgressContext, +} from "./shared.js"; + +/* + * What a comparison of two web reports (Website Scanner runs, or Automated Tests builds) + * shares: the two exports matched by rule, element and page, the lists it gives, and the + * pages that did not load in either. + */ + +/** + * The pages of the newer run that did not load (an error page was scanned in their place), + * or whose earlier issues all gave way to new ones. Neither the scan nor the dashboard says + * so, and their real issues would otherwise read as fixed. + */ +export function pagesNotLoadedNote( + diff: { + not_checked?: unknown[]; + failed_pages?: string[]; + not_checked_earlier?: unknown[]; + failed_pages_earlier?: string[]; + changed_pages?: string[]; + }, + later: string, + earlier: string, + noun: "run" | "build", +): string[] { + const pages = (urls: string[]) => + wrapUntrusted("report page addresses", JSON.stringify(urls)); + // Worded for one page or several, one issue or several. + const one = (count: number, singular: string, plural: string) => + count === 1 ? singular : plural; + const they = (n: number) => one(n, "it", "they"); + const didNotLoad = (urls: string[], run: string) => { + const n = urls.length; + return `⚠️ ${one(n, "1 page", `${n} pages`)} did not load in ${run}: ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place. The ${one(n, "error page's", "error pages'")} issues are not counted as`; + }; + // A page that failed in both runs has no issues of its own in either. + const foundThere = (count: number, run: string, notCountedAs: string) => + count > 0 + ? `, and the ${one(count, "1 issue", `${count} issues`)} found there in ${run} ${one(count, "is", "are")} not counted as ${notCountedAs}` + : ""; + const failed = diff.failed_pages ?? []; + const failedEarlier = diff.failed_pages_earlier ?? []; + const changed = diff.changed_pages ?? []; + return [ + ...(failed.length > 0 + ? [ + `${didNotLoad(failed, later)} new${foundThere(diff.not_checked?.length ?? 0, earlier, "resolved (listed as not checked)")}. ${noun === "run" ? "Scan" : "Run the tests"} again once ${they(failed.length)} ${one(failed.length, "loads", "load")}. ${pages(failed)}`, + ] + : []), + ...(failedEarlier.length > 0 + ? [ + `${didNotLoad(failedEarlier, earlier)} resolved${foundThere(diff.not_checked_earlier?.length ?? 0, later, `new (listed as not checked against the earlier ${noun})`)}. Compare with a ${noun} in which ${they(failedEarlier.length)} loaded. ${pages(failedEarlier)}`, + ] + : []), + ...(changed.length > 0 + ? [ + `⚠️ None of the earlier issues on ${one(changed.length, "1 page remain, and it has", `${changed.length} pages remain, and they have`)} new ones: ${they(changed.length)} may not have loaded, or loaded something else, in ${later}. Check ${one(changed.length, "it", "them")} before calling those issues fixed. ${pages(changed)}`, + ] + : []), + ]; +} + +/** At most this many issues of each list are listed in a comparison. */ +export const COMPARE_LIST_LIMIT = 50; + +/** + * One list of a comparison, at most COMPARE_LIST_LIMIT of it, quoted as data: its rows come + * from the reports. `total` counts the whole list where only its first issues were read; + * `rest` says where the ones not listed are. + */ +export function comparedList( + issues: RunIssue[], + total = issues.length, + rest = "", +): string { + const shown = issues.slice(0, COMPARE_LIST_LIMIT); + const more = total - shown.length; + return ( + wrapUntrusted("accessibility comparison", JSON.stringify(shown, null, 2)) + + (more > 0 ? `\n(${more} more not listed${rest}.)` : "") + ); +} + +/** + * Two reports' CSV exports, read and matched by rule, element and page; or, while an export + * is still being generated, the answer to give instead. + */ +export async function diffExports( + reportFetcher: AccessibilityReportFetcher, + later: IssueSource, + earlier: IssueSource, + what: "runs" | "builds", + context?: ScanProgressContext, +): Promise< + | { links: [string, string]; diff: ReturnType } + | CallToolResult +> { + let links: [string, string]; + try { + links = await Promise.all([ + reportFetcher.getReportLink(later, { onWait: exportProgress(context) }), + reportFetcher.getReportLink(earlier, {}), + ]); + } catch (error) { + if (error instanceof ExportPendingError) { + return createSuccessResponse([ + `The CSV exports of these ${what} are still being generated (waited ${formatDuration(error.waitedMs)}). Call fetchAccessibilityIssues again with the same arguments in about a minute; the exports continue where they are.`, + ]); + } + throw error; + } + const [after, before] = await Promise.all(links.map(loadMainIssues)); + return { links, diff: compareIssues(before, after) }; +} diff --git a/src/tools/accessiblity-utils/compare-with.ts b/src/tools/accessiblity-utils/compare-with.ts new file mode 100644 index 00000000..146e6b7a --- /dev/null +++ b/src/tools/accessiblity-utils/compare-with.ts @@ -0,0 +1,141 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import type { + AccessibilityReportFetcher, + IssueSource, +} from "./report-fetcher.js"; +import { asSentence } from "./error-messages.js"; +import { createSuccessResponse, describeSource, sourceArgs } from "./shared.js"; + +/* + * What fetchAccessibilityIssues's compareWith names: the Website Scanner run it resolves to, + * the report before a Workflow Analyzer or manual app report ("previous"), or the answer + * when there is nothing to compare yet or no built-in compare for the two reports. + */ + +/** What a report compareWith has no compare for says first. */ +export const NO_BUILT_IN_COMPARE = { + workflow: "Workflow Analyzer reports have no built-in compare.", + appReport: "App Accessibility manual reports have no built-in compare.", +} as const; + +/** + * Two reports compareWith has no compare for are compared by reading both. This is said as + * guidance, not as an error, so the agent goes straight to the two reads. An app issue is + * matched by its element, not its screen, as a fix can renumber the screens. + */ +export function readBothToCompare( + lead: string, + app: boolean, + reads?: [string, string], +): CallToolResult { + const match = app + ? "rule and component (the element's id or class)" + : "rule and CSS selector on each page"; + const calls = reads + ? ` (one call with ${reads[0]}; the other with ${reads[1]})` + : ""; + return createSuccessResponse([ + `${lead} Read both reports with fetchAccessibilityIssues${calls} and match issues by ${match}; an issue in the later report only is new, in the earlier only is fixed, in both is still open.`, + ]); +} + +/** + * The answer when a run or build to compare is still processing: its issues are still being + * added, so a comparison would count the ones not added yet as resolved or new, and read as + * a clean result. Calling again at once gets the same answer, so it hands back to the user. + */ +export function notReadyToCompare( + which: string, + status: string, + preface: string[] = [], +): CallToolResult { + return createSuccessResponse([ + ...preface, + `⏳ ${which} has not finished yet (status: ${status}), so nothing was compared: its issues are still being added, and a comparison now would count the ones not added yet as resolved or new. Tell the user it is still processing and to ask again in a few minutes; do not call fetchAccessibilityIssues again in this turn.`, + ]); +} + +/** + * compareWith "previous" on a report with no built-in compare: the report before it is found + * here (a Workflow Analyzer report's of the same page, or else the same site; a manual app + * report's of the same app) and named with the call that reads it. Agents left to find it + * searched for a way to list reports, and one gave no answer. + */ +export async function previousReportToCompare( + reportFetcher: AccessibilityReportFetcher, + source: { kind: "workflow" | "appReport"; reportId: string }, +): Promise { + const app = source.kind === "appReport"; + const earlier: + | { reportId: string; time?: string; link: string; samePage?: boolean } + | undefined = await ( + app + ? reportFetcher.previousManualReport(source.reportId) + : reportFetcher.previousWorkflowReport(source.reportId) + ).catch(() => undefined); + const lead = NO_BUILT_IN_COMPARE[source.kind]; + if (!earlier) { + return readBothToCompare( + `${lead} No earlier ${app ? "manual report of the same app" : "Workflow Analyzer report of the same page or site"} is among the account's newest reports, so ask the user which report to compare it with.`, + app, + ); + } + const which = app + ? "manual report of the same app" + : earlier.samePage + ? "report of the same page" + : "report of the same site (no earlier one scanned the same page)"; + return readBothToCompare( + `${lead} The newest ${which} before it is ${earlier.reportId}${earlier.time ? `, ${app ? "recorded" : "started"} ${earlier.time}` : ""}: ${earlier.link}`, + app, + [ + sourceArgs(source), + sourceArgs({ kind: source.kind, reportId: earlier.reportId }), + ], + ); +} + +type ScanRunSource = { kind: "scan"; scanId: string; scanRunId: string }; + +/** + * The run of this scan that compareWith names: its scanRunId, or its report link or build id, + * which are resolved as a source is. A link or id that names anything else (another scan's + * run, a build, a report) has no built-in compare with a Scanner run: that is answered with + * how to compare them by reading both, naming each read's arguments. + */ +export async function scanRunToCompare( + reportFetcher: AccessibilityReportFetcher, + source: ScanRunSource, + compareWith: string, +): Promise<{ scanRunId: string } | CallToolResult> { + const link = /^https?:\/\//i.test(compareWith); + if (!link && !/^[a-z0-9]{40}$/i.test(compareWith)) { + return { scanRunId: compareWith }; + } + let other: IssueSource; + try { + other = await reportFetcher.resolveBuild( + link + ? await reportFetcher.sourceFromReportUrl(compareWith) + : { kind: "build", buildId: compareWith }, + { orScanRun: true }, + ); + // A run is compared by its scan's ids, which only the scan list maps a build id to. + if (other.kind === "scanBuild") { + other = await reportFetcher.findScanRun(other); + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + return createSuccessResponse([ + `compareWith could not be matched to a Website Scanner run: ${asSentence(reason)} To compare with another run of scan ${source.scanId}, pass its scanRunId or "previous". To compare with another scan's run, read both reports with fetchAccessibilityIssues and match issues by rule and CSS selector on each page; an issue in the later report only is new, in the earlier only is fixed, in both is still open.`, + ]); + } + if (other.kind === "scan" && other.scanId === source.scanId) { + return { scanRunId: other.scanRunId }; + } + return readBothToCompare( + `compareWith names ${describeSource(other)}, not a run of scan ${source.scanId}: only runs of one scan are compared here.`, + false, + [sourceArgs(source), sourceArgs(other)], + ); +} diff --git a/src/tools/accessiblity-utils/error-messages.ts b/src/tools/accessiblity-utils/error-messages.ts new file mode 100644 index 00000000..563d3f5b --- /dev/null +++ b/src/tools/accessiblity-utils/error-messages.ts @@ -0,0 +1,34 @@ +/* + * How an error is put to the agent: as a sentence, and, for a refusal the user resolves on + * their own account, as what to tell them instead of a fault to report. + */ + +/** + * The message as a sentence: a full stop is added only when it does not already end in + * one (or in "!", "?" or "…", before any closing quote or bracket), so a message that + * ends in a full stop does not end in "..". + */ +export function asSentence(message: string): string { + const text = message.trim(); + return /[.!?…]["'”’)\]]*$/.test(text) ? text : `${text}.`; +} + +/** + * Refusals the user resolves on their own account, with what to tell them. The API's own + * words ("You do not have access to External APIs", "user info missing") read like faults, + * so agents retried, renamed the scan, or reported them with submitFeedback. Each pattern + * is the API's message as logged in telemetry. + */ +export function accountRefusal(message: string): string | undefined { + const scanLimit = message.match(/make over (\d+) scans/i); + if (scanLimit) { + return `This account has reached its plan's limit of ${scanLimit[1]} saved Website Scanner scans, so no new scan can be started. To scan again, delete scans that are no longer needed under Website Scanner in the Accessibility dashboard (https://accessibility.browserstack.com/site-scanner), or upgrade the plan. This is a plan limit, not a fault: tell the user, and do not retry or start the scan under another name.`; + } + if (/access to External APIs/i.test(message)) { + return "This account's BrowserStack plan does not include the External APIs add-on, which the MCP's accessibility tools need: they work through the Accessibility API. Scans can still be run and read in the Accessibility dashboard (https://accessibility.browserstack.com). To use them from here, the account admin can add External APIs to the plan through BrowserStack sales. This is a plan limit, not a fault: tell the user, and do not retry."; + } + if (/user info missing/i.test(message)) { + return "Website Scanner has not set this user up yet: it does that the first time they open Website Scanner in the Accessibility dashboard, and refuses its API until then. Ask the user to open https://accessibility.browserstack.com/site-scanner once while signed in to BrowserStack, then start the scan again."; + } + return undefined; +} diff --git a/src/tools/accessiblity-utils/expert-query.ts b/src/tools/accessiblity-utils/expert-query.ts new file mode 100644 index 00000000..97258164 --- /dev/null +++ b/src/tools/accessiblity-utils/expert-query.ts @@ -0,0 +1,57 @@ +/* A question for accessibilityExpert, fitted to the length the knowledge base takes. */ + +/** The longest question the accessibility knowledge base takes; longer ones are refused. */ +export const EXPERT_QUERY_MAX_CHARS = 500; + +/** The first `max` UTF-16 units of `text`, without splitting a surrogate pair at the cut. */ +function headOf(text: string, max: number): string { + if (text.length <= max) return text; + const code = text.charCodeAt(max - 1); + // A high surrogate at the cut would be left without its pair: cut before it. + return text.slice(0, code >= 0xd800 && code <= 0xdbff ? max - 1 : max); +} + +/** The opening of `text` within `max` characters, cut at a word break when one is near. */ +function cutAtWord(text: string, max: number): string { + if (text.length <= max) return text.trimEnd(); + const head = headOf(text, max); + const lastBreak = head.search(/\s\S*$/); + return (lastBreak > max * 0.6 ? head.slice(0, lastBreak) : head).trimEnd(); +} + +/** + * A question the knowledge base will take. It refuses anything over 500 characters + * ("Query is too long"), and agents writing a detailed question hit that: two refusals on + * 4 Nov alone. Agents put the question itself last, after the context, so a longer one + * keeps its last sentence and as much of its opening as fits around it. A question of one + * sentence, or one whose last sentence is too long for that, keeps only its opening. + */ +export function fitExpertQuery(query: string): { + query: string; + shortened: boolean; + kept?: "opening" | "opening and last sentence"; +} { + if (query.length <= EXPERT_QUERY_MAX_CHARS) + return { query, shortened: false }; + const text = query.trim(); + // Sentence ends in Latin and CJK punctuation; 。？！ take no space after them. + const sentences = text.match( + /[^.?!。？！]+(?:[.?!。？！]+["')\]」』]*\s*|$)/g, + ) ?? [text]; + const last = sentences[sentences.length - 1].trim(); + const gap = " … "; + if (sentences.length < 2 || last.length > EXPERT_QUERY_MAX_CHARS * 0.6) { + return { + query: cutAtWord(text, EXPERT_QUERY_MAX_CHARS), + shortened: true, + kept: "opening", + }; + } + const opening = text.slice(0, text.length - last.length); + const room = EXPERT_QUERY_MAX_CHARS - last.length - gap.length; + return { + query: `${cutAtWord(opening, room)}${gap}${last}`, + shortened: true, + kept: "opening and last sentence", + }; +} diff --git a/src/tools/accessiblity-utils/report-answers.ts b/src/tools/accessiblity-utils/report-answers.ts new file mode 100644 index 00000000..fe19c3eb --- /dev/null +++ b/src/tools/accessiblity-utils/report-answers.ts @@ -0,0 +1,252 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import type { ScanFailureDetails } from "./scanner.js"; +import { + ScanNotInListError, + type AccessibilityReportFetcher, + type IssueSource, +} from "./report-fetcher.js"; +import type { ErrorPage } from "./report-parser.js"; +import { wrapUntrusted } from "../../lib/untrusted-content.js"; +import { BrowserStackConfig } from "../../lib/types.js"; +import { + createErrorResponse, + createSuccessResponse, + describeSource, + initializeScanner, + scanDashboardNote, + sourceArgs, +} from "./shared.js"; + +/* + * What fetchAccessibilityIssues and startAccessibilityScan say when a report has no full + * result to give: a run that failed or was cancelled, a report still finalizing, a run + * still going with no report yet (or stuck), a build id no API knows, and the pages a run + * did not scan or that showed an error page. + */ + +/** + * Of PROCESSING_STATUSES, the processing after the run: a web report's advanced issues while + * "finalizing", an app build's rules while "rule_processing". It usually ends within a few + * minutes, but a first read no longer waits for it inside the call. It used to wait up to + * five minutes and then read the export, and a client cuts off a call that long: the + * claude.ai connector aborts a tool call silent for 305 s, progress notifications or not. + * Every such call left the agent with nothing, and one reported a fix as verified that had + * added 26 issues. So the read says the report is not ready, and to call again, which any + * client handles. + */ +export const FINALIZING_STATUSES = new Set(["finalizing", "rule_processing"]); + +/** + * The answer for a first read of a report still finalizing: not ready yet, and when to come + * back. Not an error, since nothing failed, but it says outright that no issues were read, + * so an empty answer cannot be taken for a clean page. + */ +export function stillFinalizingResponse( + source: IssueSource, + status: string, + preface: string[], + extraArgs: string, +): CallToolResult { + const state = + status === "rule_processing" + ? "is still running its accessibility rules (status: rule_processing)" + : "is still finalizing (status: finalizing): BrowserStack is still adding its issues"; + return createSuccessResponse([ + ...preface, + `⏳ The report for ${describeSource(source)} ${state}. This usually takes a few minutes after the run ends. No issues have been read yet, so this is not a result: do not tell the user the page is clean or the fix is verified.`, + `Call fetchAccessibilityIssues again with ${sourceArgs(source)}${extraArgs} in a minute or two. If it still says this after about 10 minutes, tell the user the report is stuck instead of calling again.`, + ]); +} + +/** + * A build id no API knows, after the scan list was searched for it (`scanList`) or the API + * that reads runs by build id said it has none. The id is the user's to check, so this is a + * plain answer naming what to pass instead, not a fault to report. + */ +export function buildIdNotFound( + buildId: string, + scanList?: ScanNotInListError, +): CallToolResult { + // A hashed_id's shape was also asked of Automate and App Automate. + const builds = /^[0-9a-f]{40}$/.test(buildId) + ? "Automated Tests (web), App Accessibility, Automate, App Automate" + : "Automated Tests (web), App Accessibility"; + const where = scanList + ? `No ${builds.replace(/, (?=[^,]*$)/, " or ")} build has id "${buildId}", and no Website Scanner run ${scanList.capped ? `among the most recent scans (${scanList.pages} pages)` : "in the scan list"} has it as its build id.` + : `No ${builds} or Website Scanner build with id "${buildId}" is visible to this account.`; + return createSuccessResponse([ + `${where} Check the id, or pass instead the report's dashboard link as reportUrl, a Website Scanner run's scanId and scanRunId, or the id the BrowserStack SDK printed as "testhub id" at the end of the run.`, + ]); +} + +/** How long a run can stay "running" before it is taken for stuck rather than slow. */ +const STUCK_AFTER_MS = 24 * 60 * 60 * 1000; + +/** + * A run or build still processing whose report could not be exported: it has no report yet. + * Not an error, since nothing failed; one that started over a day ago is probably stuck, and + * calling again will not change that. + */ +export async function noReportYetResponse( + reportFetcher: AccessibilityReportFetcher, + source: IssueSource, + status: string, + preface: string[], +): Promise { + const started = await reportFetcher.reportStartedAt(source); + const stuck = + started !== undefined && Date.now() - Date.parse(started) > STUCK_AFTER_MS; + const again = + source.kind === "scan" || source.kind === "scanBuild" + ? "scan the page again" + : source.kind === "workflow" + ? "run Workflow Analyzer again" + : "run the tests again"; + return createSuccessResponse([ + ...preface, + `⏳ ${describeSource(source)} is still running (status: ${status}${started ? `, started ${started}` : ""}) and has no report yet, so no issues were read: this is not a result.`, + stuck + ? `It started more than 24 hours ago, so it is probably stuck rather than still running: tell the user, and that they should ${again} and read the new one, instead of calling fetchAccessibilityIssues again for this one.` + : "Tell the user it is still running and to ask again in a few minutes, once it finishes; do not call fetchAccessibilityIssues again in this turn.", + ]); +} + +/** + * Pages that did not load: an error page (Chrome's, or BrowserStack Local's when the tunnel + * is not connected) was scanned in their place, and its issues came back as theirs. The + * report parser leaves them out of the issues and the total; this says so. + */ +export function errorPagesNote(pages: ErrorPage[], again: string): string[] { + if (pages.length === 0) return []; + const one = (n: number, singular: string, plural: string) => + n === 1 ? singular : plural; + const n = pages.length; + const issues = pages.reduce((sum, p) => sum + p.issues, 0); + const counted = + issues > 0 + ? ` The ${one(issues, "1 issue", `${issues} issues`)} found there ${one(issues, "is", "are")} from the error ${one(n, "page", "pages")}, not your ${one(n, "page", "pages")}, so ${one(issues, "it is", "they are")} left out of the issues and the total below.` + : ""; + return [ + `⚠️ ${one(n, "1 page", `${n} pages`)} did not load; ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place.${counted} Check the ${one(n, "page is", "pages are")} reachable (and the Local tunnel, for localhost), then ${again}. ${wrapUntrusted("report page addresses", JSON.stringify(pages.map((p) => p.url)))}`, + ]; +} + +/** + * Why a run that failed has nothing to read, or undefined where "failed" is not about the + * report: an app build's status is its tests' result, and its scans are read either way. + */ +export async function failedRunResponse( + source: IssueSource, + status: "failed" | "cancelled", + config: BrowserStackConfig, + preface: string[], +): Promise { + switch (source.kind) { + case "scan": + return failedScanRunResponse( + config, + source.scanId, + source.scanRunId, + preface, + (pages) => + `0 issues does not mean the ${pages > 1 ? "pages are" : "page is"} clean.`, + status === "cancelled" + ? "Scan again and read the new run." + : "Fix the cause and scan again.", + status, + ); + // The API reads a build that completed with no test cases as failed. + case "build": + return createErrorResponse( + [ + ...preface, + `Build ${source.buildId} has no accessibility results (status: failed). This usually means no test ran through the BrowserStack SDK: a plain script instead of a test framework, or a session that failed to start. Run the tests again through the SDK and read the new build's testhub id.`, + ].join("\n"), + ); + // A report fails only when BrowserStack stops processing it. + case "workflow": + return createErrorResponse( + [ + ...preface, + `Workflow Analyzer report ${source.reportId} has no accessibility results (status: failed): its processing did not complete, so 0 issues would not mean the pages are clean. Scan the workflow again with Workflow Analyzer and read the new report.`, + ].join("\n"), + ); + default: + return undefined; + } +} + +/** How many failed pages a failure result lists before summing up the rest. */ +const FAILED_PAGES_LISTED = 10; + +/** + * "N pages :" and a line per page with its reason, the first FAILED_PAGES_LISTED of + * them. The lines are the scan logs' (the page's address, the site's HTTP status, where it + * redirected), so they are quoted as data. + */ +export function pageReasons( + pages: ScanFailureDetails["pages"], + what: [one: string, several: string], +): string { + const listed = pages.slice(0, FAILED_PAGES_LISTED); + const more = pages.length - listed.length; + return [ + `${pages.length} ${pages.length === 1 ? `page ${what[0]}` : `pages ${what[1]}`}:`, + wrapUntrusted( + "page scan log", + listed.map((page) => `- ${page.pageUrl}: ${page.reason}`).join("\n"), + ), + ...(more > 0 ? [`And ${more} more ${more === 1 ? "page" : "pages"}.`] : []), + ].join("\n"); +} + +/** Why a run failed, in the API's words (quoted as data): the run's own reason and each page's. */ +export function failureReasons({ + reason, + pages, +}: ScanFailureDetails): string[] { + const why = [ + reason ? `Why: ${wrapUntrusted("scan failure reason", reason)}` : "", + pages.length + ? pageReasons(pages, ["could not be scanned", "could not be scanned"]) + : "", + ].filter(Boolean); + return why.length ? why : ["The dashboard says why."]; +} + +/** + * A scan run that failed, read or compared. A run fails only when it scanned no page (or + * the plan refused it), and its export is then an empty report: read as one, it said "0 + * issues ✅ All issues retrieved", and compared, every issue of the other run read as + * resolved or new. + */ +export async function failedScanRunResponse( + config: BrowserStackConfig, + scanId: string, + scanRunId: string, + preface: string[], + meaning: (pages: number) => string, + next: string, + status: "failed" | "cancelled" = "failed", +): Promise { + const scanner = await initializeScanner(config); + const cancelled = status === "cancelled"; + // A cancelled run has no failure to explain. + const [dashboardLink, details] = await Promise.all([ + scanner.dashboardLink(scanId), + cancelled + ? Promise.resolve({ pages: [] }) + : scanner.failureDetails(scanId, scanRunId), + ]); + return createErrorResponse( + [ + ...preface, + cancelled + ? `Scan run ${scanRunId} of scan ${scanId} was cancelled, so it has no results: ${meaning(details.pages.length)}` + : `Scan run ${scanRunId} of scan ${scanId} failed, so no pages were checked: ${meaning(details.pages.length)}`, + ...(cancelled ? [] : failureReasons(details)), + next, + scanDashboardNote(dashboardLink, scanId), + ].join("\n"), + ); +} diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index 9b3f91a0..a4d60f0e 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -1,13 +1,17 @@ import { apiClient } from "../../lib/apiClient.js"; import { AppAccessibilityApi, + BuildLookupError, manualReportIdFromDashboardLink, manualReportStatus, type AppBuildIssues, type AppBuildRef, type AppBuildReport, type AppManualReport, + type EarlierManualReport, } from "./app-report-fetcher.js"; +import { onErrorPage, samePage, type RunIssue } from "./report-parser.js"; +import { sortBySeverity } from "./paginate.js"; import { SessionType } from "../../lib/constants.js"; import { sessionsListUrl } from "../automate-utils/list-session-ids.js"; import { getAutomationBaseUrl } from "../rca-agent-utils/constants.js"; @@ -70,6 +74,22 @@ export class ScanRunNotFoundError extends Error { } } +/** + * No scan the scan list was searched across has a run with this build id; `capped` when + * the search stopped at the page limit rather than at the end of the list. + */ +export class ScanNotInListError extends Error { + constructor( + readonly capped: boolean, + readonly pages: number, + ) { + super( + `No Website Scanner scan${capped ? ` among the most recent scans (${pages} pages)` : ""} matches this link. Pass scanId and scanRunId instead`, + ); + this.name = "ScanNotInListError"; + } +} + /** The dashboard link of a run named by its build id, without the extra query a copied link carries. */ export function scanRunLink(source: ScanBuildSource): string { return `${dashboard()}/site-scanner/projects/${encodeURIComponent(source.scanName ?? "")}/builds?thBuildId=${encodeURIComponent(source.thBuildId)}`; @@ -81,13 +101,52 @@ export function scanRunLink(source: ScanBuildSource): string { * link someone just opened will be. It runs at the list's default page size (~1.2 s a page): * at limit=50 or 100 the cursor skips scans, so a larger page cannot be used to go faster. */ -export const MAX_SCAN_LIST_PAGES = 15; +export const MAX_SCAN_LIST_PAGES = 25; +/** + * Runs already found by their build id. A build id names one fixed run, so the mapping never + * changes: a link read again, or both sides of a compare, skip the scan-list walk. Keyed by + * the account, so one account never sees another's, and bounded, oldest dropped first. + */ +export const foundScanRuns = new Map< + string, + { scanId: string; scanRunId: string } +>(); +const MAX_FOUND_SCAN_RUNS = 200; + /** How many pages of a scan's runs are read to find the run before a given one. */ const MAX_SCAN_RUN_PAGES = 10; /** A build whose status is checked is one just run, so it sits at the top of its project's list. */ const MAX_BUILD_LIST_PAGES = 3; +/** + * The Workflow Analyzer report list holds the whole account's reports, newest first, with no + * page or date: a report is found in its newest pages, at the list's largest page size, and + * the pages of the earlier ones after it are read from their summaries, a batch at a time. + */ +const MAX_WORKFLOW_LIST_PAGES = 3; +const WORKFLOW_LIST_PAGE_SIZE = 99; +const MAX_WORKFLOW_CANDIDATES = 30; +const WORKFLOW_SUMMARY_BATCH = 10; + +/** An earlier Workflow Analyzer report: of the same page, or else of the same site. */ +export interface EarlierWorkflowReport { + reportId: string; + /** When it started, as the API writes it. */ + time?: string; + samePage: boolean; + link: string; +} + +/** A page's site: its host, without a leading "www.". */ +function siteOf(url: string): string | undefined { + try { + return new URL(url).hostname.toLowerCase().replace(/^www\./, ""); + } catch { + return undefined; + } +} + interface ScanListPage { scans?: { id: number; @@ -100,8 +159,9 @@ interface ScanListPage { /** * How long one call waits on a CSV export. Export time grows with the report's rows: a few - * seconds for hundreds, ~17 s for 18k rows, over 2 minutes for 30k issues. A call stays well - * under the 60 s request timeout MCP clients default to; an export still running after this + * seconds for hundreds, ~17 s for 18k rows, over 2 minutes for 30k issues. The wait itself + * stays under the 60 s request timeout MCP clients default to, though finding a link's run + * and comparing two reports can add to a call's time; an export still running after this * is handed back to be continued by the next call rather than started again. */ export const EXPORT_WAIT_MS = 45_000; @@ -178,6 +238,77 @@ export interface RunChanges { resolvedIssues: number; } +/** An Automated Tests (web) build, as its overview names it. */ +export interface WebBuild { + buildId: string; + name: string; + projectName?: string; + buildNumber?: number; + /** When the build started, as the API writes it (ISO 8601). */ + startedAt?: string; +} + +/** The compare API's three lists, in its own words (`retained` is still open). */ +export type CompareStatus = "new" | "resolved" | "retained"; +const COMPARE_STATUSES: CompareStatus[] = ["new", "retained", "resolved"]; + +/** Two Automated Tests builds as the compare API compares them. */ +export interface WebBuildComparison { + /** The older build; `autoResolved` when the API chose it (no baseReport was passed). */ + base: WebBuild & { autoResolved: boolean }; + after: WebBuild; + counts: Record; + /** The first issues of each list, up to the list limit, most severe first. */ + issues: Record; +} + +/** Why the compare API gave no comparison, in its words. */ +export interface WebCompareRefusal { + refused: string; + /** The build has no earlier completed run with its name in its project. */ + noPrevious?: true; +} + +/** + * How long a call to the compare API is waited on. The API stops itself at 30 s and answers + * 504 "Request timed out"; this covers a gateway that never answers. + */ +export const WEB_COMPARE_TIMEOUT_MS = 40_000; +/** The most issues the compare API hands out in one call. */ +const WEB_COMPARE_PAGE = 100; + +/** + * A compare API row as a comparison lists it. Its key is the API's own (signatureKey), which + * tells apart more than rule, element and page: a changed element's markup, for one. + */ +function comparedIssue(row: any): RunIssue { + const selector = [row?.target ?? []].flat().filter(Boolean).join(" "); + const issue_type = row?.ruleData?.id ?? ""; + const affected_page = row?.url ?? ""; + return { + key: + row?.signatureKey || + [issue_type, selector || row?.componentId || "", affected_page].join("|"), + issue_type, + severity: (row?.impact || "unknown").trim(), + component: row?.componentId ?? "", + ...(selector ? { selector } : {}), + affected_page, + ...(onErrorPage(row?.html) ? { on_error_page: true as const } : {}), + }; +} + +/** A build as the compare API's summary names it. */ +function comparedBuild(report: any): WebBuild { + return { + buildId: report?.id, + name: report?.buildName, + projectName: report?.projectName, + buildNumber: report?.buildNumber, + startedAt: report?.createdOn, + }; +} + /** The thBuildId a dashboard report link carries, if any. */ function thBuildIdOf(link: string | undefined): string | null { try { @@ -264,7 +395,7 @@ export class AccessibilityReportFetcher { private auth: { username: string; password: string } | undefined; private readonly app = new AppAccessibilityApi(() => this.authHeaders()); /** Web builds whose overview was read, by id, so a status check need not read it again. */ - private readonly webBuildProjects = new Map(); + private readonly webBuilds = new Map(); public setAuth(auth: { username: string; password: string }): void { this.auth = auth; @@ -338,9 +469,11 @@ export class AccessibilityReportFetcher { return this.buildOfAutomateSession(automate.sessionId); } if (automate?.buildId) { - return this.buildOfAutomateSession( - await this.anyAutomateSession(automate.buildId), - ); + const session = await this.automateSession(automate.buildId); + if (!session) { + throw new Error(`No Automate build with id ${automate.buildId}`); + } + return this.buildOfAutomateSession(session); } const thBuildId = url?.searchParams.get("thBuildId"); if (url && thBuildId && url.pathname.includes("/automated-tests/")) { @@ -386,18 +519,49 @@ export class AccessibilityReportFetcher { } const web = resp.ok ? resp.data?.data : undefined; if (web) { - if (web.projectName) { - this.webBuildProjects.set(source.buildId, web.projectName); - } + this.rememberWebBuild(source.buildId, web); return source; } const meta = await this.app.buildMeta(source.buildId); if (meta) return { kind: "appBuild", buildId: source.buildId }; + // An Automate or App Automate build's own id, its hashed_id (40 hex characters; a TestHub + // id has other letters too), as the SDK prints it at the end of a run: its sessions name + // the accessibility build they ran in, as for the build's link. Asked before the scan list. + // A probe that could not be answered (not a 404) moves on to the next API, and is said + // if none knows the id; once a session is found, what it says is the answer. + const probeFailures: string[] = []; + if (/^[0-9a-f]{40}$/.test(source.buildId)) { + let session: string | undefined; + try { + session = await this.automateSession(source.buildId); + } catch (error) { + probeFailures.push( + error instanceof Error ? error.message : String(error), + ); + } + if (session) return this.buildOfAutomateSession(session); + let appBuildId: string | undefined; + try { + appBuildId = await this.app.buildIdOfAppAutomateBuild(source.buildId); + } catch (error) { + if (!(error instanceof BuildLookupError)) throw error; + probeFailures.push(error.message); + } + if (appBuildId) return { kind: "appBuild", buildId: appBuildId }; + } // A Website Scanner run's build id (a scanner link's thBuildId) is a TestHub id too, which // only the scanner's issues route knows: the caller reads it there, and says so if not. - if (orScanRun) return { kind: "scanBuild", thBuildId: source.buildId }; + // A 40-hex id is an Automate id, not a TestHub one: when Automate could not be asked about + // it, that is the answer, not a missing scan run. + if (orScanRun && !probeFailures.length) { + return { kind: "scanBuild", thBuildId: source.buildId }; + } throw new Error( - `No Automated Tests (web) or App Accessibility build with id "${source.buildId}". Pass the id the BrowserStack SDK printed as "testhub id" at the end of the run (40 lowercase letters and digits), or the report's dashboard link as reportUrl`, + `No Automated Tests (web) or App Accessibility build with id "${source.buildId}". Pass the id the BrowserStack SDK printed as "testhub id" at the end of the run (40 lowercase letters and digits), or the report's dashboard link as reportUrl${ + probeFailures.length + ? ` (Automate could not be asked about it: ${probeFailures.join("; ")})` + : "" + }`, ); } @@ -429,16 +593,26 @@ export class AccessibilityReportFetcher { return this.app.appReportIssues(reportId, cursor, maxCharacterLength); } - /** Any one session of an Automate build; every session of an SDK run names the same build. */ - private async anyAutomateSession(automateBuildId: string): Promise { + /** The newest manual report of the same app recorded before this one, if any. */ + previousManualReport( + reportId: string, + ): Promise { + return this.app.previousManualReport(reportId); + } + + /** + * Any one session of an Automate build; every session of an SDK run names the same build. + * Undefined when Automate does not know the build (its 404). + */ + private async automateSession( + automateBuildId: string, + ): Promise { const resp = await apiClient.get({ url: `${sessionsListUrl(SessionType.Automate, automateBuildId)}?limit=1`, headers: this.authHeaders(), raise_error: false, }); - if (resp.status === 404) { - throw new Error(`No Automate build with id ${automateBuildId}`); - } + if (resp.status === 404) return undefined; if (!resp.ok) { throw new Error( `Could not list the sessions of Automate build ${automateBuildId}: HTTP ${resp.status}`, @@ -507,6 +681,9 @@ export class AccessibilityReportFetcher { scanId: string; scanRunId: string; }> { + const cacheKey = `${this.auth?.username ?? ""}|${thBuildId}`; + const found = foundScanRuns.get(cacheKey); + if (found) return { kind: "scan", ...found }; let nextPage: string | undefined; let sameName: number | undefined; for (let page = 0; page < MAX_SCAN_LIST_PAGES; page++) { @@ -525,11 +702,14 @@ export class AccessibilityReportFetcher { `Website Scanner scan ${scan.id} has no completed run to read yet; wait for it to finish, or pass scanId and scanRunId`, ); } - return { - kind: "scan", - scanId: String(scan.id), - scanRunId: String(scanRunId), - }; + const run = { scanId: String(scan.id), scanRunId: String(scanRunId) }; + foundScanRuns.delete(cacheKey); + foundScanRuns.set(cacheKey, run); + for (const oldest of foundScanRuns.keys()) { + if (foundScanRuns.size <= MAX_FOUND_SCAN_RUNS) break; + foundScanRuns.delete(oldest); + } + return { kind: "scan", ...run }; } if (sameName === undefined && scanName && scan.name === scanName) { sameName = scan.id; @@ -547,12 +727,7 @@ export class AccessibilityReportFetcher { ); } // nextPage is still set only when the walk stopped at the page cap, not at the end. - const searched = nextPage - ? ` among the most recent scans (${MAX_SCAN_LIST_PAGES} pages)` - : ""; - throw new Error( - `No Website Scanner scan${searched} matches this link. Pass scanId and scanRunId instead`, - ); + throw new ScanNotInListError(nextPage !== undefined, MAX_SCAN_LIST_PAGES); } /** @@ -597,7 +772,10 @@ export class AccessibilityReportFetcher { } taskId = initData.data.task_id; } - rememberExport(key, { taskId }); + // The task of a report still adding issues is not kept either: continued after the report + // finished, it would hand back the snapshot taken while it was processing as final. + if (options.stillProcessing) reportExports.delete(key); + else rememberExport(key, { taskId }); const started = Date.now(); for (let attempt = 0; ; attempt++) { @@ -638,28 +816,43 @@ export class AccessibilityReportFetcher { } } - /** The run before `scanRunId` in its scan's history, or undefined for the first run. */ - async previousScanRun( + /** + * A scan's runs, newest first, and whether the whole list was read: a list cut short at + * the page limit, or by a page that could not be read, may not hold a run that is the scan's. + */ + async scanRuns( scanId: string, - scanRunId: string, - ): Promise { - // The runs come a page at a time; the one before may be on a later page. + ): Promise<{ runs: ScanRun[]; complete: boolean }> { + // The runs come a page at a time. const runs: ScanRun[] = []; const base = `${apiBase()}/website-scanner/v1/scans/${encodeURIComponent(scanId)}/scan_runs`; let nextPage: string | undefined; + let complete = false; for (let page = 0; page < MAX_SCAN_RUN_PAGES; page++) { const data = await this.getData( nextPage ? `${base}?nextPage=${encodeURIComponent(nextPage)}` : base, ); runs.push(...(data?.scanRuns ?? [])); - if (!data?.pagination?.hasNext || !data.pagination.nextPage) break; + if (!data?.pagination?.hasNext || !data.pagination.nextPage) { + complete = data !== undefined; + break; + } nextPage = data.pagination.nextPage; } - const newestFirst = [...runs].sort((a, b) => - b.scanDate.localeCompare(a.scanDate), - ); - const at = newestFirst.findIndex((run) => String(run.id) === scanRunId); - return at >= 0 ? newestFirst[at + 1] : undefined; + return { + runs: runs.sort((a, b) => b.scanDate.localeCompare(a.scanDate)), + complete, + }; + } + + /** The run before `scanRunId` in its scan's history, or undefined for the first run. */ + async previousScanRun( + scanId: string, + scanRunId: string, + ): Promise { + const { runs } = await this.scanRuns(scanId); + const at = runs.findIndex((run) => String(run.id) === scanRunId); + return at >= 0 ? runs[at + 1] : undefined; } /** The dashboard's new / retained / resolved counts against the run before this one. */ @@ -716,15 +909,122 @@ export class AccessibilityReportFetcher { } } + /** A Workflow Analyzer report's start and the pages it scanned, from its summary. */ + private async workflowReportPages( + reportId: string, + ): Promise<{ time?: string; pages: string[] } | undefined> { + const summary = ( + await this.getData( + `${apiBase()}/workflow-analyzer/v1/reports/${encodeURIComponent(reportId)}`, + ) + )?.summary; + if (!summary) return undefined; + const rows: { url?: unknown }[] = summary.issueCount?.issueCountByUrl ?? []; + return { + time: summary.time, + pages: rows + .map((row) => row?.url) + .filter((url): url is string => typeof url === "string"), + }; + } + + /** + * The newest Workflow Analyzer report before this one that scanned one of its pages, or + * else one on its site; undefined when none is among the account's newest reports. + */ + async previousWorkflowReport( + reportId: string, + ): Promise { + const own = await this.workflowReportPages(reportId); + if (!own?.pages.length) return undefined; + // Newest first: the completed reports listed after this one are earlier. + const earlier: string[] = []; + let reached = false; + let nextPage: string | undefined; + for ( + let page = 0; + page < MAX_WORKFLOW_LIST_PAGES && + earlier.length < MAX_WORKFLOW_CANDIDATES; + page++ + ) { + const data = await this.getData( + `${apiBase()}/workflow-analyzer/v1/reports?limit=${WORKFLOW_LIST_PAGE_SIZE}${ + nextPage ? `&next_page=${encodeURIComponent(nextPage)}` : "" + }`, + ); + for (const row of data?.reports ?? []) { + if (String(row.id) === reportId) reached = true; + else if (reached && row.status === "completed") { + earlier.push(String(row.id)); + } + } + if (!data?.pagination?.hasNext || !data.pagination.nextPage) break; + nextPage = data.pagination.nextPage; + } + const pages = new Set(own.pages.map(samePage)); + const sites = new Set( + own.pages.map(siteOf).filter((site) => site !== undefined), + ); + const onSite = (url: string) => { + const site = siteOf(url); + return site !== undefined && sites.has(site); + }; + const started = (time?: string) => Date.parse(time ?? ""); + let sameSite: EarlierWorkflowReport | undefined; + const candidates = earlier.slice(0, MAX_WORKFLOW_CANDIDATES); + for (let i = 0; i < candidates.length; i += WORKFLOW_SUMMARY_BATCH) { + const batch = candidates.slice(i, i + WORKFLOW_SUMMARY_BATCH); + const summaries = await Promise.all( + batch.map((id) => this.workflowReportPages(id)), + ); + for (const [j, summary] of summaries.entries()) { + if (!summary || started(summary.time) >= started(own.time)) continue; + const report = { + reportId: batch[j], + time: summary.time, + link: `${dashboard()}/workflow-analyzer/report?ids=${encodeURIComponent(batch[j])}`, + }; + if (summary.pages.some((url) => pages.has(samePage(url)))) { + return { ...report, samePage: true }; + } + if (!sameSite && summary.pages.some(onSite)) { + sameSite = { ...report, samePage: false }; + } + } + } + return sameSite; + } + + /** + * When the run or build behind a report started, as its API writes it (ISO 8601), or + * undefined when it cannot be read. + */ + async reportStartedAt(source: IssueSource): Promise { + try { + switch (source.kind) { + case "build": + return (await this.webBuild(source.buildId))?.startedAt; + case "scan": + return (await this.scanRuns(source.scanId)).runs.find( + (run) => String(run.id) === source.scanRunId, + )?.scanDate; + case "workflow": + return ( + await this.getData( + `${apiBase()}/workflow-analyzer/v1/reports/${encodeURIComponent(source.reportId)}`, + ) + )?.summary?.time; + default: + return undefined; + } + } catch { + return undefined; + } + } + private async buildStatus(buildId: string): Promise { const q = encodeURIComponent; - const project = - this.webBuildProjects.get(buildId) ?? - ( - await this.getData( - `${apiBase()}/automated-tests/v1/builds/${q(buildId)}`, - ) - )?.projectName; + const project = (await this.webBuild(buildId))?.projectName; if (!project) return undefined; let nextPage: string | undefined; for (let page = 0; page < MAX_BUILD_LIST_PAGES; page++) { @@ -741,6 +1041,178 @@ export class AccessibilityReportFetcher { return undefined; } + private rememberWebBuild(buildId: string, overview: any): WebBuild { + const build: WebBuild = { + buildId, + name: overview.name, + projectName: overview.projectName, + buildNumber: overview.buildNumber, + startedAt: overview.time, + }; + this.webBuilds.set(buildId, build); + return build; + } + + /** An Automated Tests build's name, project, number and start, from its overview. */ + async webBuild(buildId: string): Promise { + const known = this.webBuilds.get(buildId); + if (known) return known; + const overview = await this.getData( + `${apiBase()}/automated-tests/v1/builds/${encodeURIComponent(buildId)}`, + ); + return overview ? this.rememberWebBuild(buildId, overview) : undefined; + } + + /** + * The run before a build that the compare API would pick: the newest earlier build with + * the same name in its project whose accessibility run completed (`failed` in the build + * list is one that completed with no test cases). Found in the project's newest builds. + */ + async previousWebBuild( + build: WebBuild, + ): Promise<(WebBuild & { status: string }) | undefined> { + if (!build.projectName) return undefined; + const q = encodeURIComponent; + let passed = false; + let nextPage: string | undefined; + for (let page = 0; page < MAX_BUILD_LIST_PAGES; page++) { + const data = await this.getData( + `${apiBase()}/automated-tests/v1/builds?project_name=${q(build.projectName)}&limit=100${ + nextPage ? `&next_page=${q(nextPage)}` : "" + }`, + ); + for (const row of data?.builds ?? []) { + if (row.id === build.buildId) passed = true; + else if ( + passed && + row.name === build.name && + (row.status === "completed" || row.status === "failed") + ) { + return { + buildId: row.id, + name: row.name, + projectName: build.projectName, + buildNumber: row.buildNumber, + startedAt: row.time, + status: row.status, + }; + } + } + if (!data?.pagination?.has_next || !data.pagination.next_page) break; + nextPage = data.pagination.next_page; + } + return undefined; + } + + /** + * Two builds of one build name and project, compared by the Automated Tests compare API + * as the dashboard compares them: `newReport` against `baseReport`, or against the run + * before it when no base is given. The API takes them in that order and does not check it. + * The counts are the API's; each list holds up to `listLimit` issues, read from the first + * page and, for a list that page does not cover, a page of that list alone. The API refuses + * builds of different names or projects, builds over its issue-count threshold and calls + * that outlast its 30 s budget; any refusal is handed back in its words. + */ + async compareWebBuilds( + newReport: string, + baseReport: string | undefined, + listLimit: number, + ): Promise { + const call = async ( + params: Record, + ): Promise => { + const url = `${apiBase()}/automated-tests/v1/builds/compare/issues?${new URLSearchParams(params)}`; + let resp; + try { + resp = await apiClient.get({ + url, + headers: this.authHeaders(), + raise_error: false, + timeout: WEB_COMPARE_TIMEOUT_MS, + }); + } catch (error) { + const timedOut = /timeout|ECONNABORTED/i.test( + error instanceof Error + ? `${error.message} ${(error as any).code}` + : "", + ); + return { + refused: timedOut + ? `no answer within ${WEB_COMPARE_TIMEOUT_MS / 1000} s` + : error instanceof Error + ? error.message + : "the request failed", + }; + } + const body = resp.data; + if (resp.ok && body?.data?.summary) return body.data; + const reason = + (typeof body?.message === "string" && body.message) || + [body?.error].flat().filter(Boolean).join("; ") || + `HTTP ${resp.status}`; + return { + refused: reason, + ...(resp.status === 404 && /no previous completed run/i.test(reason) + ? { noPrevious: true as const } + : {}), + }; + }; + + const first = await call({ + newReport, + ...(baseReport ? { baseReport } : {}), + limit: String(WEB_COMPARE_PAGE), + }); + if ("refused" in first) return first; + const summary = first.summary; + const counts: Record = { + new: Number(summary.newIssues) || 0, + retained: Number(summary.retainedIssues) || 0, + resolved: Number(summary.resolvedIssues) || 0, + }; + const base = comparedBuild(summary.baseReport); + const rowsOf = (data: any, status: CompareStatus) => + (data?.issues ?? []) + .filter((row: any) => row?.status === status) + .map(comparedIssue); + const issues = {} as Record; + for (const status of COMPARE_STATUSES) { + issues[status] = rowsOf(first, status); + } + // The first page runs new, then still open, then resolved: a list it cuts short is read + // on its own, against the base the first call settled on. + const short = COMPARE_STATUSES.filter( + (status) => issues[status].length < Math.min(counts[status], listLimit), + ); + const pages = await Promise.all( + short.map((status) => + call({ + newReport, + baseReport: base.buildId ?? baseReport ?? "", + status, + limit: String(Math.max(10, Math.min(listLimit, WEB_COMPARE_PAGE))), + }), + ), + ); + for (const [i, status] of short.entries()) { + const page = pages[i]; + if ("refused" in page) return page; + issues[status] = rowsOf(page, status); + } + for (const status of COMPARE_STATUSES) { + issues[status] = sortBySeverity(issues[status].slice(0, listLimit)); + } + return { + base: { + ...base, + autoResolved: Boolean(summary.baseReport?.autoResolved), + }, + after: comparedBuild(summary.newReport), + counts, + issues, + }; + } + /** The `data` of a successful API response, or undefined. */ private async getData(url: string): Promise { const resp = await apiClient.get({ diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 25db2258..f301da07 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -113,8 +113,16 @@ type PaginatedResult = { hidden_issues: number | null; /** Set when the report was read for one page. */ page?: PageSummary; + /** + * Set when pages did not load: an error page was scanned in their place. Their rows are + * left out of everything above, as they are the error page's issues, not the page's. + */ + error_pages?: ErrorPage[]; }; +/** A page that did not load, and how many issues the error page in its place would have added. */ +export type ErrorPage = { url: string; issues: number }; + const DUPLICATE_COLUMN = "Duplicate Issue of"; const NEEDS_REVIEW_COLUMN = "Needs review"; const REVIEW_STATUS_COLUMN = "Needs Review Status"; @@ -134,7 +142,7 @@ function isMainIssue(row: Record): boolean { } /** A page address as the report writes it, give or take host case, trailing slash and hash. */ -function samePage(url: string): string { +export function samePage(url: string): string { try { const u = new URL(url.trim()); return `${u.protocol}//${u.host.toLowerCase()}${u.pathname.replace(/\/+$/, "")}${u.search}`; @@ -427,6 +435,8 @@ export type RunIssue = { issue_type: string; severity: string; component: string; + /** The element's CSS selector, where the key is not made from it. */ + selector?: string; affected_page: string; /** Found on an error page scanned in the page's place: the page did not load. */ on_error_page?: true; @@ -441,15 +451,56 @@ export type RunIssue = { const ERROR_PAGE_SNIPPET = /dashboard-privoxy-page|id="main-frame-error"|chrome-error:\/\/|class="neterror[" ]/i; +/** Whether an issue's HTML snippet is from an error page scanned in place of a page that did not load. */ +export function onErrorPage(snippet: string | undefined): boolean { + return ERROR_PAGE_SNIPPET.test(snippet ?? ""); +} + +/** + * The report's rows without those of pages that did not load, and those pages. A page is one + * when any row under its address shows an error page (as compareIssues decides it), and all + * its rows are then the error page's: counted as the page's issues, they would say the page + * was checked. `issues` is how many of them the total would have counted. + */ +function setApartErrorPages( + rows: Record[], + grouped: boolean, +): { rows: Record[]; error_pages: ErrorPage[] } { + const failed = new Map(); + for (const row of rows) { + if (!onErrorPage(row["HTML snippet"])) continue; + const url = row["Affected page"] ?? ""; + const key = samePage(url); + if (!failed.has(key)) failed.set(key, { url, issues: 0 }); + } + if (failed.size === 0) return { rows, error_pages: [] }; + const kept: Record[] = []; + for (const row of rows) { + const page = failed.get(samePage(row["Affected page"] ?? "")); + if (!page) kept.push(row); + else if ((!grouped || isMainIssue(row)) && !isHidden(row)) page.issues++; + } + return { rows: kept, error_pages: [...failed.values()] }; +} + /** A page whose earlier issues are all gone and new ones appeared: worth a second look. */ const CHANGED_PAGE_MIN_ISSUES = 2; -/** A report's main issues that the dashboard counts (not hidden), keyed for comparing runs. */ +/** + * A report's main issues that the dashboard counts (not hidden), keyed for comparing runs. + * In a grouped report an error page's issues on a second page are duplicates of those on the + * first, so those duplicates are kept too, marked: they show that page did not load either, + * and as their page is set apart they are never counted. + */ export async function loadMainIssues(reportLink: string): Promise { const { parsed, columns } = await loadReport(reportLink); const grouped = columns.includes(DUPLICATE_COLUMN); return parsed - .filter((row) => (!grouped || isMainIssue(row)) && !isHidden(row)) + .filter( + (row) => + (!grouped || isMainIssue(row) || onErrorPage(row["HTML snippet"])) && + !isHidden(row), + ) .map((row) => { const element = (row["CSS selector"] || row["Component"] || "").trim(); return { @@ -462,7 +513,7 @@ export async function loadMainIssues(reportLink: string): Promise { severity: (row["Severity"] || "unknown").trim(), component: row["Component"], affected_page: row["Affected page"], - ...(ERROR_PAGE_SNIPPET.test(row["HTML snippet"] ?? "") + ...(onErrorPage(row["HTML snippet"]) ? { on_error_page: true as const } : {}), }; @@ -491,8 +542,8 @@ function issuesByPage( } /** - * Which issues of `current` are new since `base`, which of `base` are gone, and how many - * remain, decided page by page. A page that did not load in one of the runs (an error page + * Which issues of `current` are new since `base`, which of `base` are gone, and which remain + * (`still_open`, as `current` has them; `retained` counts them), decided page by page. A page that did not load in one of the runs (an error page * was scanned under its address) is set apart. Not loaded in `current`: its earlier issues * were not checked again, so they are `not_checked` rather than resolved, and the error * page's issues are not new. Not loaded in `base`: the error page's issues are not resolved, @@ -507,6 +558,7 @@ export function compareIssues( added: RunIssue[]; resolved: RunIssue[]; retained: number; + still_open: RunIssue[]; not_checked: RunIssue[]; failed_pages: string[]; not_checked_earlier: RunIssue[]; @@ -517,12 +569,12 @@ export function compareIssues( const after = issuesByPage(current); const added: RunIssue[] = []; const resolved: RunIssue[] = []; + const stillOpen: RunIssue[] = []; const notChecked: RunIssue[] = []; const notCheckedEarlier: RunIssue[] = []; const failedPages: string[] = []; const failedPagesEarlier: string[] = []; const changedPages: string[] = []; - let retained = 0; for (const key of new Set([...before.keys(), ...after.keys()])) { const earlier = before.get(key); const later = after.get(key); @@ -544,7 +596,9 @@ export function compareIssues( ); added.push(...pageAdded); resolved.push(...pageResolved); - retained += (later?.issues.length ?? 0) - pageAdded.length; + stillOpen.push( + ...(later?.issues ?? []).filter((issue) => earlierKeys.has(issue.key)), + ); if ( earlier && earlier.issues.length >= CHANGED_PAGE_MIN_ISSUES && @@ -557,7 +611,8 @@ export function compareIssues( return { added: sortBySeverity(added), resolved: sortBySeverity(resolved), - retained, + retained: stillOpen.length, + still_open: sortBySeverity(stillOpen), not_checked: sortBySeverity(notChecked), failed_pages: failedPages, not_checked_earlier: sortBySeverity(notCheckedEarlier), @@ -574,12 +629,21 @@ export async function parseAccessibilityReportFromCSV( pageUrl, }: PaginationOptions = {}, ): Promise { - // 1) Download & parse - const { parsed, columns } = await loadReport(reportLink); + // 1) Download & parse, setting apart the pages that did not load + const loaded = await loadReport(reportLink); + const { columns } = loaded; + const { rows: parsed, error_pages } = setApartErrorPages( + loaded.parsed, + columns.includes(DUPLICATE_COLUMN), + ); if (pageUrl) { const { all, page } = pageIssues(parsed, columns, pageUrl); const slice = pageWithRules(all, parsed, { maxCharacterLength, nextPage }); + // Read for one page, only that page not loading matters. + const notLoaded = error_pages.filter( + (p) => samePage(p.url) === samePage(pageUrl), + ); return { ...slice, total_issues: all.length, @@ -587,6 +651,7 @@ export async function parseAccessibilityReportFromCSV( unlinked_duplicates: 0, hidden_issues: null, page, + ...(notLoaded.length ? { error_pages: notLoaded } : {}), }; } @@ -656,5 +721,6 @@ export async function parseAccessibilityReportFromCSV( hidden_issues: columns.includes(STATUS_COLUMN) ? mainRows.length - visibleMainRows.length : null, + ...(error_pages.length ? { error_pages } : {}), }; } diff --git a/src/tools/accessiblity-utils/scan-rerun.ts b/src/tools/accessiblity-utils/scan-rerun.ts new file mode 100644 index 00000000..aa759939 --- /dev/null +++ b/src/tools/accessiblity-utils/scan-rerun.ts @@ -0,0 +1,201 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import type { SavedScan, ScanSettings } from "./scanner.js"; +import { onOff, wcagName } from "./shared.js"; +import { wrapUntrusted } from "../../lib/untrusted-content.js"; + +/* + * Re-running a scan by its scanId: whether it is for the page asked for, how the settings + * passed differ from its saved ones, and the notes and hints said with a scan's result. + */ + +/** startAccessibilityScan's arguments, as the call passed them: a setting left out is undefined. */ +export type ScanArgs = { + name: string; + pageURL: string; + scanId?: string; + authConfigId?: number; + localIdentifier?: string; +} & ScanSettings; + +/** + * The result with lines said before and after it. Each text block is made to end in a + * newline, as createSuccessResponse's do, so a link that ends one block is not run into + * the next by a client that joins blocks with no separator. + */ +export function withNotes( + result: CallToolResult, + lead: string[], + tail: string[] = [], +): CallToolResult { + if (!lead.length && !tail.length) return result; + const block = (text: string) => ({ + type: "text" as const, + text: text.endsWith("\n") ? text : `${text}\n`, + }); + return { + ...result, + content: [ + ...lead.map(block), + ...result.content.map((c) => (c.type === "text" ? block(c.text) : c)), + ...tail.map(block), + ], + }; +} + +/** + * Said with every new scan. Re-scanning a page used to start another scan, and runs of two + * scans cannot be compared; a run added to this scan can. + */ +export function rescanHint(scanId: string): string { + return `To scan this same page again later (e.g. after a fix), call startAccessibilityScan with scanId "${scanId}"; that adds a run to this scan so the runs can be compared with compareWith "previous".`; +} + +export function compareHint(scanId: string, scanRunId: string): string { + return `To see what changed since the previous run, call fetchAccessibilityIssues with scanId "${scanId}", scanRunId "${scanRunId}" and compareWith "previous".`; +} + +/** Why a scan given by id is not re-run for this page: it was saved for another. */ +export function otherPageNote( + scanId: string, + urlList: string[], + pageURL: string, +): string { + if (!urlList.length) { + return `Scan ${scanId}'s pages could not be read, so it could not be checked to be for ${pageURL}; a new scan was started.`; + } + // The saved pages are the account's, which another user may have set: quoted as data. + const pages = wrapUntrusted( + "saved scan pages", + JSON.stringify(urlList.slice(0, 3)), + ); + if (urlList.length === 1) { + return `Scan ${scanId} is for another page, not ${pageURL}, so a new scan was started. Its page: ${pages}`; + } + const more = urlList.length > 3 ? ` (the first 3 of ${urlList.length})` : ""; + return `Scan ${scanId} is for ${urlList.length} pages, not just ${pageURL}, so a new scan was started. Its pages${more}: ${pages}`; +} + +/** + * The settings a call passed that differ from the scan's saved ones. A re-run takes no + * settings, so running it anyway would quietly drop what was asked for. Only settings the + * call passed are compared (none has a default in the schema), and only against a saved + * value the overview gives: it gives whether a login or Local is saved, not which one. + */ +export function savedSettingDifferences( + args: ScanArgs, + saved: SavedScan, +): string[] { + const differ = ( + label: string, + passed: T | undefined, + savedValue: T | undefined, + show: (value: T) => string, + ) => + passed !== undefined && savedValue !== undefined && passed !== savedValue + ? [`${label}: saved ${show(savedValue)}, passed ${show(passed)}`] + : []; + const { settings } = saved; + return [ + ...differ("WCAG version", args.wcagVersion, settings.wcagVersion, wcagName), + ...differ( + "advanced rules", + args.advancedRules, + settings.advancedRules, + onOff, + ), + ...differ( + "experimental rules", + args.experimentalRules, + settings.experimentalRules, + onOff, + ), + ...differ( + "best practices", + args.bestPractices, + settings.bestPractices, + onOff, + ), + ...differ("needs review", args.needsReview, settings.needsReview, onOff), + ...(args.authConfigId !== undefined && saved.authEnabled === false + ? [ + `login: the scan has no auth config saved, passed authConfigId ${args.authConfigId}`, + ] + : []), + ...(args.localIdentifier !== undefined && + saved.localTestingEnabled === false + ? [ + `BrowserStack Local: the scan is saved without a tunnel, passed localIdentifier "${args.localIdentifier}"`, + ] + : []), + ]; +} + +/** Settings a call passed that the overview gives no saved value to check them against. */ +export function uncheckedSettings(args: ScanArgs, saved: SavedScan): string[] { + const { settings } = saved; + return [ + args.wcagVersion !== undefined && settings.wcagVersion === undefined + ? "wcagVersion" + : "", + args.advancedRules !== undefined && settings.advancedRules === undefined + ? "advancedRules" + : "", + args.experimentalRules !== undefined && + settings.experimentalRules === undefined + ? "experimentalRules" + : "", + args.bestPractices !== undefined && settings.bestPractices === undefined + ? "bestPractices" + : "", + args.needsReview !== undefined && settings.needsReview === undefined + ? "needsReview" + : "", + ].filter(Boolean); +} + +/** + * A login or Local tunnel passed to a re-run, which the run cannot take: it logs in and + * tunnels with the scan's saved ones, which the overview does not name. Said plainly, so a + * run with the old login is not taken for one with the login passed. + */ +export function unusedLoginAndTunnel( + args: ScanArgs, + saved: SavedScan, +): string[] { + return [ + // Which login is saved is not given: only whether one is. + ...(args.authConfigId !== undefined && saved.authEnabled !== false + ? [ + `authConfigId ${args.authConfigId} was not used: a re-run logs in with the scan's saved login, which the API does not name. To scan with login ${args.authConfigId}, call startAccessibilityScan without scanId (that starts a new scan).`, + ] + : []), + // Said in the Local note when the scan is saved with a tunnel. + ...(args.localIdentifier !== undefined && + saved.localTestingEnabled === undefined + ? [ + `localIdentifier "${args.localIdentifier}" was not used: a re-run uses the scan's saved BrowserStack Local settings, which the API does not give back. To scan through that tunnel, call startAccessibilityScan without scanId (that starts a new scan).`, + ] + : []), + ]; +} + +/** + * Why a scan saved with BrowserStack Local is not re-run when no localIdentifier is passed: + * its tunnel's identifier is not given back, and one this MCP started was never shown. + */ +export function localScanNotRerunNote(scanId: string): string { + return `Scan ${scanId} reaches its page through BrowserStack Local under an identifier the API does not give back, so it can't be re-run from here: a new scan was started, whose runs can't be compared with scan ${scanId}'s. To re-run scan ${scanId}, start BrowserStack Local yourself with the identifier the scan was created with and pass it as localIdentifier.`; +} + +/** + * A re-run reaches a local page through the tunnel the scan was saved with, by its + * identifier, which the overview does not give back: the tunnel has to be running under + * that identifier, or the run cannot reach the page. + */ +export function localRerunNote(localIdentifier: string | undefined): string { + return `This scan reaches its page through BrowserStack Local, and the run uses the scan's saved Local settings: a tunnel must be running with the same local identifier the scan was created with${ + localIdentifier + ? ` (the API does not give the saved identifier back, so it could not be checked against "${localIdentifier}")` + : " (the API does not give the saved identifier back)" + }. If the run cannot reach the page, start that tunnel and re-run.`; +} diff --git a/src/tools/accessiblity-utils/scanner.ts b/src/tools/accessiblity-utils/scanner.ts index 04d19993..12d59cc9 100644 --- a/src/tools/accessiblity-utils/scanner.ts +++ b/src/tools/accessiblity-utils/scanner.ts @@ -25,6 +25,10 @@ export interface ScanFailureDetails { pages: { pageUrl: string; status: string; reason: string }[]; } +/** The scans API on the account's region (GRR). */ +const scansApi = () => + `${getRegionalHost("accessibilityApi")}/api/website-scanner/v1/scans`; + /** How long startAccessibilityScan waits for a scan to finish before handing back its ids. */ export const SCAN_WAIT_MS = 5 * 60 * 1000; @@ -42,6 +46,85 @@ export class ScanTimeoutError extends Error { } } +/** + * A refusal whose message is the whole answer, given as a plain result: not as a failure to + * start the scan with the advice to open an issue (or to send feedback, which an error + * result gets), since nothing is broken and the message says what to do instead. + */ +export class ScanRefusedError extends Error { + constructor(message: string) { + super(message); + this.name = "ScanRefusedError"; + } +} + +/** + * central-scanner's refusal of a scan over the plan's limit, "You don't have access to make + * over 50 scans". It read as a failure to report on GitHub, with no hint of the limit. + */ +const SCAN_LIMIT_REFUSAL = /make over (\d+) scans/i; + +/** The plan-limit refusal, with the limit the API named (50 on the plans seen so far). */ +export function scanLimitMessage(limit?: string): string { + return `${limit ? `Your plan allows ${limit} saved scans and this account has reached that limit` : "This account has reached your plan's limit of saved scans"}. Delete scans you no longer need in the dashboard, or upgrade the plan to create more.`; +} + +/** Hosts on the user's own machine, which a scan reaches as bs-local.com through the tunnel. */ +const LOCAL_HOSTS = new Set(["127.0.0.1", "localhost", "0.0.0.0"]); +const BS_LOCAL_DOMAIN = "bs-local.com"; + +/** The URL as a scan is created with it, and so as the scan saves it. */ +export function scanUrl(url: string): string { + try { + const parsed = new URL(url); + if (LOCAL_HOSTS.has(parsed.hostname)) { + parsed.hostname = BS_LOCAL_DOMAIN; + return parsed.toString(); + } + return url; + } catch (e) { + logger.warn(`[AccessibilityScan] Invalid URL skipped: ${e}`); + return url; + } +} + +/** + * Whether a scan's saved URL is this page: the page is put as a scan would save it + * (scanUrl), and the case of the host and a trailing slash are not told apart. + */ +export function samePage(savedUrl: string, pageURL: string): boolean { + const key = (url: string) => { + const trimmed = url.trim(); + try { + const parsed = new URL(scanUrl(trimmed)); + parsed.pathname = parsed.pathname.replace(/\/+$/, ""); + return parsed.toString(); + } catch { + return trimmed.replace(/\/+$/, ""); + } + }; + return key(savedUrl) === key(pageURL); +} + +/** A scan id this account cannot see, or that is no scan's. */ +function notVisible(scanId: string): string { + return `No scan with id ${scanId} is visible to this account, so nothing was started. Leave out scanId to start a new scan.`; +} + +/** + * A scan's saved setup, from its overview: what a re-run of it runs with. A setting left out + * is one the overview does not give. + */ +export interface SavedScan { + name?: string; + urlList: string[]; + settings: ScanSettings; + /** Whether a login (auth config) is saved; which one, the overview does not say. */ + authEnabled?: boolean; + /** Whether the scan goes through BrowserStack Local; its identifier is not given. */ + localTestingEnabled?: boolean; +} + /** How to scan through a tunnel the user starts, which the scan API takes by identifier. */ const OWN_TUNNEL_HINT = "Start BrowserStack Local yourself (the BrowserStack Local app, or BrowserStackLocal --key --local-identifier ) and pass that name as localIdentifier."; @@ -122,6 +205,10 @@ function nameIsTaken(err: any): boolean { function scanStartError(err: any): Error { // apiClient throws generic errors, try to extract message. const reason = refusalReason(err); + const limit = SCAN_LIMIT_REFUSAL.exec(String(reason ?? "")); + if (err?.response?.status === 402 && limit) { + return new ScanRefusedError(scanLimitMessage(limit[1])); + } if (err?.response?.status === 422 && !reason) { return new Error( "A scan with this name already exists. please update the name and run again.", @@ -177,22 +264,22 @@ export class AccessibilityScanner { // Check if any URL is local const hasLocal = urlList.some(isLocalURL); const localIdentifier = ownTunnel ?? randomUUID(); - const localHosts = new Set(["127.0.0.1", "localhost", "0.0.0.0"]); - const BS_LOCAL_DOMAIN = "bs-local.com"; // A tunnel the user started is theirs to run: none is started here, and none is killed, // as killing every BrowserStackLocal process would take theirs down with it. if (!ownTunnel) { + // Both refusals are limits of how this MCP runs, not failures: the scan API takes the + // identifier of a tunnel the user started. if (config.USE_OWN_LOCAL_BINARY_PROCESS && hasLocal) { - throw new Error( - `Cannot start scan with local URLs when using own BrowserStack Local binary process. ${OWN_TUNNEL_HINT}`, + throw new ScanRefusedError( + `This BrowserStack MCP is set to use your own BrowserStack Local (USE_OWN_LOCAL_BINARY_PROCESS), so it starts no tunnel for a local URL. ${OWN_TUNNEL_HINT} Or scan a public URL.`, ); } // The hosted server cannot start a tunnel on the user's machine, but the scan API // takes the identifier of one the user started. if (config.REMOTE_MCP && hasLocal) { - throw new Error( + throw new ScanRefusedError( `This hosted BrowserStack MCP runs on BrowserStack's servers, so it cannot open a tunnel to a local URL on your machine. ${OWN_TUNNEL_HINT} Or scan a public URL.`, ); } @@ -208,19 +295,7 @@ export class AccessibilityScanner { } } - const transformedUrlList = urlList.map((url) => { - try { - const parsed = new URL(url); - if (localHosts.has(parsed.hostname)) { - parsed.hostname = BS_LOCAL_DOMAIN; - return parsed.toString(); - } - return url; - } catch (e) { - logger.warn(`[AccessibilityScan] Invalid URL skipped: ${e}`); - return url; - } - }); + const transformedUrlList = urlList.map(scanUrl); const scanSettings = Object.fromEntries( Object.entries(settings).filter(([, value]) => value !== undefined), @@ -379,24 +454,9 @@ export class AccessibilityScanner { scanId: string, scanRunId: string, ): Promise { - const read = (url: string) => - apiClient - .get({ - url, - headers: { - Authorization: - "Basic " + - Buffer.from( - `${this.auth?.username}:${this.auth?.password}`, - ).toString("base64"), - }, - raise_error: false, - }) - .catch(() => undefined); - const base = `${getRegionalHost("accessibilityApi")}/api/website-scanner/v1/scans`; - const [list, logs] = await Promise.all([ - read(base), - read(`${base}/${scanId}/scan_runs/${scanRunId}/scan_logs`), + const [list, pages] = await Promise.all([ + this.readQuietly(scansApi()), + this.unscannedPages(scanId, scanRunId), ]); const last = ( list?.ok @@ -412,8 +472,25 @@ export class AccessibilityScanner { last.error.trim() ? last.error.trim() : undefined; - const entries: any[] = logs?.ok ? (logs.data?.data?.scanLogs ?? []) : []; - const pages = entries + return { ...(reason ? { reason } : {}), pages }; + } + + /** + * The pages of a run whose scan log is not a success, each with the log's reason. Only + * the run's logs are read, not the scans list failureDetails also reads: a completed run + * names the pages it could not scan here too, and they are otherwise simply absent from + * its report. A read that fails gives none. + */ + async unscannedPages( + scanId: string, + scanRunId: string, + ): Promise { + const logs = await this.readQuietly( + `${scansApi()}/${encodeURIComponent(scanId)}/scan_runs/${encodeURIComponent(scanRunId)}/scan_logs`, + ); + const entries: unknown = logs?.ok ? logs.data?.data?.scanLogs : undefined; + if (!Array.isArray(entries)) return []; + return entries .filter((entry) => entry && entry.status !== "success") .map((entry) => { const said = [entry.description, entry.error] @@ -431,7 +508,185 @@ export class AccessibilityScanner { String(entry.status ?? "unknown"), }; }); - return { ...(reason ? { reason } : {}), pages }; + } + + /** + * A GET of the scanner API, or undefined when it fails: for reads that only add detail to + * an answer, so their failure must not become the answer. + */ + private async readQuietly(url: string) { + try { + return await apiClient.get({ + url, + headers: { + Authorization: + "Basic " + + Buffer.from( + `${this.auth?.username}:${this.auth?.password}`, + ).toString("base64"), + }, + raise_error: false, + }); + } catch { + return undefined; + } + } + + private authHeaders(): Record { + return { + Authorization: + "Basic " + + Buffer.from(`${this.auth?.username}:${this.auth?.password}`).toString( + "base64", + ), + }; + } + + /** + * A scan's saved setup, which a re-run of it runs with. An id that is no scan this account + * can see is a refusal to say as it is; the API takes only a number. + */ + async savedScan(scanId: string): Promise { + if (!/^\d+$/.test(scanId)) throw new ScanRefusedError(notVisible(scanId)); + const resp = await apiClient.get({ + url: `${scansApi()}/${scanId}/overview`, + headers: this.authHeaders(), + raise_error: false, + }); + if (resp.status === 404) throw new ScanRefusedError(notVisible(scanId)); + if (!resp.ok) { + const reason = refusalReason({ response: resp }); + throw new Error( + `Could not read scan ${scanId} to check it is for this page (${reason || `HTTP ${resp.status}`}); nothing was started`, + ); + } + const data = resp.data?.data ?? {}; + const info = data.scanConfigInfo ?? {}; + const flag = (value: unknown) => + typeof value === "boolean" ? value : undefined; + const settings: ScanSettings = { + ...(typeof info.wcagVersion === "string" && info.wcagVersion + ? { wcagVersion: info.wcagVersion } + : {}), + ...(flag(info.bestPractices) !== undefined + ? { bestPractices: info.bestPractices } + : {}), + ...(flag(info.needsReview) !== undefined + ? { needsReview: info.needsReview } + : {}), + ...(flag(info.advanced) !== undefined + ? { advancedRules: info.advanced } + : {}), + ...(flag(info.experimental) !== undefined + ? { experimentalRules: info.experimental } + : {}), + }; + return { + ...(typeof info.name === "string" && info.name + ? { name: info.name } + : {}), + urlList: Array.isArray(data.urlList) ? data.urlList.map(String) : [], + settings, + authEnabled: flag(info.authEnabled), + localTestingEnabled: flag(info.localTestingEnabled), + }; + } + + /** + * Starts a new run of a saved scan. The endpoint takes no settings: the run has the + * scan's saved ones, Local and login included. Its refusals are said as they are. + */ + async rerunScan(scanId: string): Promise<{ scanRunId: string }> { + let data: any; + try { + const response = await apiClient.post({ + url: `${scansApi()}/${scanId}/new_scan`, + headers: { + ...this.authHeaders(), + "Content-Type": "application/json", + }, + body: {}, + }); + data = response.data; + } catch (err: any) { + throw await this.rerunRefusal(scanId, err); + } + const scanRunId = data?.data?.scanRunId; + if (!data?.success || scanRunId === undefined || scanRunId === null) { + throw new Error( + `Unable to re-run scan ${scanId}: ${data?.error || JSON.stringify(data)}`, + ); + } + return { scanRunId: String(scanRunId) }; + } + + private async rerunRefusal(scanId: string, err: any): Promise { + const status = err?.response?.status; + const reason = refusalReason(err); + const said = String(reason ?? ""); + if (status === 422 && /already in progress/i.test(said)) { + const run = await this.runInProgress(scanId); + return new ScanRefusedError( + run + ? `A run of scan ${scanId} is still going (run ${run.id}, ${run.status}), so a new run was not started. Read that run's issues with fetchAccessibilityIssues (scanId "${scanId}", scanRunId "${run.id}") once it finishes, or wait for it to finish and call startAccessibilityScan with scanId "${scanId}" again for another run.` + : `A run of scan ${scanId} is still going, so a new run was not started; its run id could not be read (the scan's page in the dashboard shows it). Wait for it to finish, then read its issues with fetchAccessibilityIssues, or call startAccessibilityScan with scanId "${scanId}" again for another run.`, + ); + } + if (status === 403 && /auth config/i.test(said)) { + return new ScanRefusedError( + `Scan ${scanId} can't be re-run: the login config (auth config) it was saved with has been deleted. Start a new scan instead: call startAccessibilityScan without scanId, with an authConfigId that exists if the page needs a login.`, + ); + } + if (status === 404) return new ScanRefusedError(notVisible(scanId)); + if (typeof status === "number" && status >= 400 && status < 500 && said) { + // 402 is the plan: its message names what the plan does not include. + return new ScanRefusedError( + `Scan ${scanId} was not re-run: ${said}${/[.!?]$/.test(said) ? "" : "."}`, + ); + } + return new Error( + `Failed to re-run scan ${scanId}: ${said || err?.message || String(err)}`, + ); + } + + /** + * The run of a scan that is still going, or undefined when it cannot be told. The scans + * list gives each scan's last run with its status, but only for a scan on its first page; + * the scan's own runs list has every run, newest by date, but no status. + */ + async runInProgress( + scanId: string, + ): Promise<{ id: string; status: string } | undefined> { + const read = (url: string) => + apiClient + .get({ url, headers: this.authHeaders(), raise_error: false }) + .catch(() => undefined); + const going = (status: unknown): status is string => + typeof status === "string" && !FINAL_SCAN_STATUSES.has(status); + const [list, runs] = await Promise.all([ + read(scansApi()), + read(`${scansApi()}/${scanId}/scan_runs`), + ]); + const last = ( + list?.ok + ? ((list.data?.data?.scans ?? []) as any[]).find( + (scan) => String(scan?.id) === String(scanId), + ) + : undefined + )?.lastScanDetails; + if (last?.scanRunId && going(last.scanStatus)) { + return { id: String(last.scanRunId), status: last.scanStatus }; + } + const newest = ( + runs?.ok ? ((runs.data?.data?.scanRuns ?? []) as any[]) : [] + ) + .filter((run) => run?.id !== undefined) + .sort((a, b) => String(b.scanDate).localeCompare(String(a.scanDate)))[0]; + if (!newest) return undefined; + const status = ( + await read(`${scansApi()}/${scanId}/scan_runs/${newest.id}/status`) + )?.data?.data?.status; + return going(status) ? { id: String(newest.id), status } : undefined; } async pollStatus( diff --git a/src/tools/accessiblity-utils/shared.ts b/src/tools/accessiblity-utils/shared.ts new file mode 100644 index 00000000..722742f2 --- /dev/null +++ b/src/tools/accessiblity-utils/shared.ts @@ -0,0 +1,184 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { AccessibilityScanner } from "./scanner.js"; +import { + EXPORT_WAIT_MS, + scanRunLink, + type IssueSource, +} from "./report-fetcher.js"; +import { getBrowserStackAuth } from "../../lib/get-auth.js"; +import { BrowserStackConfig } from "../../lib/types.js"; + +/* + * Helpers of accessibility.ts that the answer and compare modules beside this file use too: + * the response builders, how a report source is named, progress notifications, the scanner + * client and the scan-settings names. They are moved here unchanged, so that those modules + * and accessibility.ts do not import each other. + */ + +interface AuthCredentials { + username: string; + password: string; +} + +export interface ScanProgressContext { + sendNotification: (notification: any) => Promise; + _meta?: { + progressToken?: string | number; + }; +} + +export function setupAuth(config: BrowserStackConfig): AuthCredentials { + const authString = getBrowserStackAuth(config); + const [username, password] = authString.split(":"); + return { username, password }; +} + +export function createErrorResponse(message: string): CallToolResult { + return { + content: [ + { + type: "text", + text: message, + }, + ], + isError: true, + }; +} + +/** + * Each message ends with a newline: clients that join text blocks with no separator + * would otherwise run a link at the end of one block into the next block's first word. + */ +export function createSuccessResponse(messages: string[]): CallToolResult { + return { + content: messages.map((text) => ({ + type: "text" as const, + text: `${text}\n`, + })), + }; +} + +export async function notifyScanProgress( + context: ScanProgressContext, + message: string, + progress = 0, +): Promise { + await context.sendNotification({ + method: "notifications/progress", + params: { + progressToken: context._meta?.progressToken?.toString() ?? "NOT_FOUND", + message, + progress, + total: 100, + }, + }); +} + +export async function initializeScanner( + config: BrowserStackConfig, +): Promise { + const scanner = new AccessibilityScanner(); + const auth = setupAuth(config); + scanner.setAuth(auth); + return scanner; +} + +/** The arguments that name the same source again, for the next-page hint. */ +export function sourceArgs(source: IssueSource): string { + switch (source.kind) { + case "scan": + return `scanId: "${source.scanId}", scanRunId: "${source.scanRunId}"`; + // A run read by its build id is named by its link again, or, given as a build id, by that. + case "scanBuild": + return source.scanName !== undefined + ? `reportUrl: "${scanRunLink(source)}"` + : `buildId: "${source.thBuildId}"`; + case "build": + case "appBuild": + return `buildId: "${source.buildId}"`; + case "workflow": + return `workflowReportId: "${source.reportId}"`; + case "appReport": + return `appReportId: "${source.reportId}"`; + } +} + +/** What a dashboard link resolved to, in words. */ +export function describeSource(source: IssueSource): string { + switch (source.kind) { + case "scan": + return `Website Scanner scan ${source.scanId}, run ${source.scanRunId}`; + case "scanBuild": + return `Website Scanner run with build id ${source.thBuildId}`; + case "build": + return `Automated Tests build ${source.buildId}`; + case "workflow": + return `Workflow Analyzer report ${source.reportId}`; + case "appBuild": + return `App Accessibility build ${source.buildId}`; + case "appReport": + return `App Accessibility manual report ${source.reportId}`; + } +} + +/** Statuses during which a report's issues are still being added (`rule_processing`: an app build's). */ +export const PROCESSING_STATUSES = new Set([ + "queued", + "running", + "processing", + "finalizing", + "rule_processing", + "in_progress", +]); + +export function formatDuration(ms: number): string { + const seconds = Math.round(ms / 1000); + return seconds < 60 + ? `${seconds}s` + : `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, "0")}s`; +} + +/** Progress while a report's CSV export is generated, so clients see the call is alive. */ +export function exportProgress( + context: ScanProgressContext | undefined, +): (waitedMs: number) => Promise { + return async (waitedMs) => { + if (!context) return; + await notifyScanProgress( + context, + `Generating the report's CSV export (large reports take longer): ${formatDuration(waitedMs)} so far.`, + Math.min(99, Math.round((waitedMs / EXPORT_WAIT_MS) * 100)), + ); + }; +} + +/** The WCAG versions and levels Website Scanner checks against, and RGAA 4.1, by name. */ +export const WCAG_VERSIONS = { + wcag2a: "WCAG 2.0 A", + wcag2aa: "WCAG 2.0 AA", + wcag2aaa: "WCAG 2.0 AAA", + wcag21a: "WCAG 2.1 A", + wcag21aa: "WCAG 2.1 AA", + wcag21aaa: "WCAG 2.1 AAA", + wcag22a: "WCAG 2.2 A", + wcag22aa: "WCAG 2.2 AA", + wcag22aaa: "WCAG 2.2 AAA", + rgaa41: "RGAA 4.1", +} as const; + +export const wcagName = (value: string) => + WCAG_VERSIONS[value as keyof typeof WCAG_VERSIONS] ?? value; +export const onOff = (value: boolean) => (value ? "on" : "off"); + +/** + * Where to see a scan in the dashboard: its own page, as the scans list links it. A scan's + * page is named by its build, so a link made from the scan's name does not open it. + */ +export function scanDashboardNote( + dashboardLink: string | undefined, + scanId: string, +): string { + return dashboardLink + ? `Its page in the Accessibility dashboard: ${dashboardLink}` + : `See it under Website Scanner in the Accessibility dashboard (scan ID ${scanId}).`; +} diff --git a/src/tools/accessiblity-utils/web-build-compare.ts b/src/tools/accessiblity-utils/web-build-compare.ts new file mode 100644 index 00000000..f802bea0 --- /dev/null +++ b/src/tools/accessiblity-utils/web-build-compare.ts @@ -0,0 +1,390 @@ +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import type { + AccessibilityReportFetcher, + IssueSource, + WebBuild, + WebBuildComparison, +} from "./report-fetcher.js"; +import type { compareIssues, RunIssue } from "./report-parser.js"; +import { wrapUntrusted } from "../../lib/untrusted-content.js"; +import { + COMPARE_LIST_LIMIT, + comparedList, + diffExports, + pagesNotLoadedNote, +} from "./compare-lists.js"; +import { notReadyToCompare, readBothToCompare } from "./compare-with.js"; +import { + createErrorResponse, + createSuccessResponse, + describeSource, + PROCESSING_STATUSES, + type ScanProgressContext, +} from "./shared.js"; + +/* fetchAccessibilityIssues's compareWith on an Automated Tests (web) build. */ + +/** A web build in prose: its number and id. Its name is the user's own, so it goes in data. */ +function webBuildRef(build: WebBuild): string { + return `build ${build.buildNumber !== undefined ? `#${build.buildNumber} ` : ""}(${build.buildId}${build.startedAt ? `, started ${build.startedAt}` : ""})`; +} + +/** Web builds' names, projects, numbers, ids and starts, quoted as data. */ +function webBuildsData(builds: Record): string { + const data = Object.fromEntries( + Object.entries(builds).map(([role, b]) => [ + role, + { + name: b.name, + project: b.projectName, + build_number: b.buildNumber, + id: b.buildId, + started: b.startedAt, + }, + ]), + ); + return `Builds: ${wrapUntrusted("Automated Tests builds", JSON.stringify(data, null, 2))}`; +} + +/** + * A build the build list calls `failed` completed with no test cases, so it has no + * accessibility results: there is nothing to compare it with, which is said as guidance. + */ +function noResultsToCompare( + build: WebBuild, + builds: Record, + previous = false, +): CallToolResult { + const ref = webBuildRef(build); + const which = previous + ? `The run before this build with the same build name and project ("previous") is ${ref}, which` + : `${ref[0].toUpperCase()}${ref.slice(1)}`; + return createSuccessResponse([ + `${which} has no accessibility results: its status is failed, which Automated Tests gives a build that completed with no test cases. There is nothing to compare; compare with another build, or run the tests again with accessibility on.`, + webBuildsData(builds), + ]); +} + +/** The three lists of a web build comparison, and what is set apart from them. */ +interface WebCompareLists { + counts: { new: number; fixed: number; open: number }; + new: RunIssue[]; + fixed: RunIssue[]; + open: RunIssue[]; + /** Where only the first issues of a list were read: its count, to say how many more. */ + totals?: { new: number; fixed: number; open: number }; + /** What did not load, said before the lists. */ + warnings: string[]; + /** The issues set apart because their page did not load, listed after them. */ + setApart: string[]; + /** Where the issues not listed are. */ + rest: string; + closing: string[]; +} + +function webComparisonMessages( + head: string[], + lists: WebCompareLists, +): CallToolResult { + const { counts, totals = lists.counts } = lists; + return createSuccessResponse([ + ...head, + `${counts.new} new, ${counts.fixed} fixed, ${counts.open} still open.`, + ...lists.warnings, + `New (only in the later build) (${counts.new}): ${comparedList(lists.new, totals.new, lists.rest)}`, + `Fixed (only in the earlier build) (${counts.fixed}): ${comparedList(lists.fixed, totals.fixed, lists.rest)}`, + `Still open (in both) (${counts.open}): ${comparedList(lists.open, totals.open, lists.rest)}`, + ...lists.setApart, + ...lists.closing, + ]); +} + +/** + * The compare API's comparison as lists. Issues found on an error page are set apart: the + * page did not load, so they say nothing about the page. The API counts them in its lists. + */ +function apiComparisonLists(comparison: WebBuildComparison): WebCompareLists { + const { counts, issues } = comparison; + const loaded = (rows: RunIssue[]) => rows.filter((r) => !r.on_error_page); + // A list's issues not read count as more; those set apart are listed below. + const more = (rows: RunIssue[], count: number) => + loaded(rows).length + Math.max(0, count - rows.length); + const apart = [ + ...issues.new.map((r) => ({ ...r, list: "new" })), + ...issues.resolved.map((r) => ({ ...r, list: "fixed" })), + ...issues.retained.map((r) => ({ ...r, list: "still open" })), + ].filter((r) => r.on_error_page); + return { + counts: { + new: counts.new, + fixed: counts.resolved, + open: counts.retained, + }, + new: loaded(issues.new), + fixed: loaded(issues.resolved), + open: loaded(issues.retained), + totals: { + new: more(issues.new, counts.new), + fixed: more(issues.resolved, counts.resolved), + open: more(issues.retained, counts.retained), + }, + warnings: + apart.length > 0 + ? [ + `⚠️ ${apart.length === 1 ? "1 issue listed was" : `${apart.length} issues listed were`} found on an error page (BrowserStack Local's connection page or Chrome's network error page) scanned in place of a page that did not load. ${apart.length === 1 ? "It is" : "They are"} set apart below as "page did not load", not listed as new, fixed or still open; the counts above include ${apart.length === 1 ? "it" : "them"}. Run the tests again once the page loads.`, + ] + : [], + setApart: + apart.length > 0 + ? [`Page did not load (${apart.length}): ${comparedList(apart)}`] + : [], + rest: "", + closing: [], + }; +} + +/** Two web builds' CSV exports matched by rule, element and page, as lists. */ +function diffComparisonLists( + { + links, + diff, + }: { links: [string, string]; diff: ReturnType }, + before: WebBuild, + after: WebBuild, +): WebCompareLists { + // Two builds of different names can share a number, so the id is said too. + const label = (b: WebBuild) => + `build ${b.buildNumber !== undefined ? `#${b.buildNumber} ` : ""}(${b.buildId})`; + return { + counts: { + new: diff.added.length, + fixed: diff.resolved.length, + open: diff.still_open.length, + }, + new: diff.added, + fixed: diff.resolved, + open: diff.still_open, + warnings: pagesNotLoadedNote(diff, label(after), label(before), "build"), + setApart: [ + ...(diff.not_checked.length + ? [ + `Not checked (page did not load in ${label(after)}): ${comparedList(diff.not_checked)}`, + ] + : []), + ...(diff.not_checked_earlier.length + ? [ + `Not checked against the earlier build (page did not load in ${label(before)}): ${comparedList(diff.not_checked_earlier)}`, + ] + : []), + ], + rest: "; see the CSV links", + closing: [ + `CSV of each build, valid for about 12 hours: later ${links[0]} , earlier ${links[1]}`, + ], + }; +} + +/** How an issue list is matched when the compare API does not compare the two builds. */ +const MATCHED_BY_ELEMENT = + "so the issue lists were matched by rule, element and page."; + +/** + * An Automated Tests build against another (its id or report link) or "previous", from the + * older build ("before") to the newer ("after"), whichever order they were passed in. Two + * builds of one build name and project are compared by the Automated Tests compare API, as + * the dashboard compares them; any other pair, or one the API refuses or does not answer in + * time, by matching their issue exports by rule, element and page, as Scanner runs are. + */ +export async function compareWebBuilds( + reportFetcher: AccessibilityReportFetcher, + source: { kind: "build"; buildId: string }, + compareWith: string, + preface: string[], + context?: ScanProgressContext, +): Promise { + const self = await reportFetcher.webBuild(source.buildId); + if (!self) { + throw new Error(`Could not read Automated Tests build ${source.buildId}`); + } + let other: WebBuild | undefined; + if (compareWith !== "previous") { + let named: IssueSource; + try { + named = await reportFetcher.resolveBuild( + /^https?:\/\//i.test(compareWith) + ? await reportFetcher.sourceFromReportUrl(compareWith) + : { kind: "build", buildId: compareWith }, + ); + } catch (error) { + return createErrorResponse( + `compareWith: ${error instanceof Error ? error.message : String(error)}.`, + ); + } + if (named.kind !== "build") { + return readBothToCompare( + `compareWith names ${describeSource(named)}, which is not an Automated Tests build; an Automated Tests build is compared here only with another ("previous", or another build's id or link).`, + false, + ); + } + if (named.buildId === source.buildId) { + return createErrorResponse( + `compareWith names this build itself. Pass another build's id or link, or "previous".`, + ); + } + other = await reportFetcher.webBuild(named.buildId); + if (!other) { + throw new Error(`Could not read Automated Tests build ${named.buildId}`); + } + } + + const [selfStatus, otherStatus] = await Promise.all([ + reportFetcher.reportStatus(source), + other + ? reportFetcher.reportStatus({ kind: "build", buildId: other.buildId }) + : undefined, + ]); + // The older build is "before", whichever was passed as compareWith: by start, or by build + // number when a start is missing and both are runs of one build name and project. With + // neither, the order passed is kept, and said. + const started = (b: WebBuild) => Date.parse(b.startedAt ?? ""); + let reversed = false; + let orderUnknown = false; + if (other) { + const [otherAt, selfAt] = [started(other), started(self)]; + if (!Number.isNaN(otherAt) && !Number.isNaN(selfAt)) { + reversed = otherAt > selfAt; + } else if ( + typeof other.buildNumber === "number" && + typeof self.buildNumber === "number" && + other.name === self.name && + other.projectName === self.projectName + ) { + reversed = other.buildNumber > self.buildNumber; + } else { + orderUnknown = true; + } + } + let before = reversed ? self : other; + const after = reversed ? other! : self; + const pair = (b: WebBuild | undefined): Record => + b ? { before: b, after } : { after }; + for (const [build, status] of [ + [self, selfStatus], + [other, otherStatus], + ] as const) { + if (build && status === "failed") { + return noResultsToCompare(build, pair(before)); + } + } + const notReady = (build: WebBuild, status: string | undefined) => + status && PROCESSING_STATUSES.has(status) + ? notReadyToCompare( + `Automated Tests ${webBuildRef(build)}`, + status, + preface, + ) + : undefined; + const unfinished = + notReady(self, selfStatus) ?? (other && notReady(other, otherStatus)); + if (unfinished) return unfinished; + + const head = (base: WebBuild, lines: string[]) => [ + ...preface, + `Compared Automated Tests builds: before ${webBuildRef(base)}, after ${webBuildRef(after)}.`, + ...(reversed + ? ["compareWith was the newer build, so it is used as 'after'."] + : []), + ...(orderUnknown + ? [ + "The builds' start times could not be read, so compareWith was taken as the earlier build. If it is the newer one, swap them: new and fixed issues are then the other way around.", + ] + : []), + ...lines, + webBuildsData({ before: base, after }), + ]; + + let method: string; + if ( + !before || + (before.name === after.name && before.projectName === after.projectName) + ) { + const compared = await reportFetcher.compareWebBuilds( + after.buildId, + before?.buildId, + COMPARE_LIST_LIMIT, + ); + if (!("refused" in compared)) { + // The API's summary leaves the project out; the overview has it, and the status check + // below reads the overview anyway. + const base: WebBuild = + before ?? + (await reportFetcher.webBuild(compared.base.buildId)) ?? + compared.base; + if (!before) { + const baseStatus = await reportFetcher.reportStatus({ + kind: "build", + buildId: base.buildId, + }); + if (baseStatus === "failed") { + return noResultsToCompare(base, pair(base), true); + } + const baseUnfinished = notReady(base, baseStatus); + if (baseUnfinished) return baseUnfinished; + } + return webComparisonMessages( + head(base, [ + ...(before + ? [] + : [ + `compareWith "previous": the compare API used the run before it with the same build name and project, ${webBuildRef(base)}.`, + ]), + "Same build name and project, so this is the Automated Tests compare API's comparison, as the dashboard shows it: issues are matched by the dashboard's own issue key (key below), not by rule, element and page alone.", + ]), + apiComparisonLists(compared), + ); + } + if (compared.noPrevious) { + return createErrorResponse( + `${webBuildRef(after)} is the first completed run of its build name in its project, so there is nothing earlier to compare it with. Pass another build's id or link as compareWith instead.`, + ); + } + if (!before) { + const previous = await reportFetcher.previousWebBuild(after); + if (!previous) { + return createErrorResponse( + `The compare API gave no comparison (${compared.refused}), and no earlier completed run of this build's name is among its project's newest builds. Pass the earlier build's id or link as compareWith instead.`, + ); + } + if (previous.status === "failed") { + return noResultsToCompare(previous, pair(previous), true); + } + before = previous; + } + method = `The compare API gave no comparison (${compared.refused}), ${MATCHED_BY_ELEMENT}`; + } else { + method = + before.name !== after.name + ? `Different build names, ${MATCHED_BY_ELEMENT}` + : `Different projects, ${MATCHED_BY_ELEMENT}`; + } + + const read = await diffExports( + reportFetcher, + { kind: "build", buildId: after.buildId }, + { kind: "build", buildId: before.buildId }, + "builds", + context, + ); + if ("content" in read) return read; + return webComparisonMessages( + head(before, [ + ...(compareWith === "previous" + ? [ + `compareWith "previous": the run before it with the same build name and project, ${webBuildRef(before)}.`, + ] + : []), + method, + ]), + diffComparisonLists(read, before, after), + ); +} diff --git a/tests/tools/accessibility.test.ts b/tests/tools/accessibility.test.ts index 72b5711e..2d948eda 100644 --- a/tests/tools/accessibility.test.ts +++ b/tests/tools/accessibility.test.ts @@ -12,6 +12,7 @@ vi.mock("../../src/tools/accessiblity-utils/scanner", () => ({ startScan: vi.fn().mockResolvedValue({ id: "scan-1", scanRunId: "run-1" }), waitUntilComplete: vi.fn().mockResolvedValue({ status: "completed" }), })), + ScanRefusedError: class extends Error {}, })); vi.mock("../../src/tools/accessiblity-utils/report-fetcher", () => ({ AccessibilityReportFetcher: vi.fn().mockImplementation(() => ({ diff --git a/tests/tools/accessibilityAppCompareOutput.test.ts b/tests/tools/accessibilityAppCompareOutput.test.ts index d85016fa..c5cdd5e0 100644 --- a/tests/tools/accessibilityAppCompareOutput.test.ts +++ b/tests/tools/accessibilityAppCompareOutput.test.ts @@ -28,6 +28,14 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ previousAppBuild = m.previousAppBuild; }, })); +// A completed scan's read also reads its scan logs, for the pages it could not scan. +vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ + AccessibilityScanner: class { + setAuth = vi.fn(); + unscannedPages = vi.fn(async () => []); + }, + ScanTimeoutError: class extends Error {}, +})); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ parseAccessibilityReportFromCSV: m.parse, })); @@ -489,16 +497,22 @@ describe("fetchAccessibilityIssues comparing app builds", () => { expect(text(result)).toMatch(/itself/); }); - it("compares scan runs and app builds only, not web builds", async () => { - m.resolveBuild.mockImplementation(async (source: unknown) => source); - m.reportStatus.mockResolvedValue("completed"); + it("says to read both when compareWith names a report that is not an app build", async () => { + m.sourceFromReportUrl.mockResolvedValue({ + kind: "appReport", + reportId: "6d1f0a2b-3c4d-4e5f-8a9b-0c1d2e3f4a5b", + }); const result = await fetchIssues()({ - buildId: "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja", - compareWith: "previous", + buildId: BUILD_ID, + compareWith: + "https://app-accessibility.browserstack.com/report/6d1f0a2b-3c4d-4e5f-8a9b-0c1d2e3f4a5b", }); - expect(result.isError).toBe(true); - expect(text(result)).toMatch(/Website Scanner runs and App Accessibility builds/); + expect(result.isError).toBeFalsy(); + expect(text(result).trim()).toBe( + 'compareWith names App Accessibility manual report 6d1f0a2b-3c4d-4e5f-8a9b-0c1d2e3f4a5b, which is not an App Accessibility build; an app build is compared here only with another ("previous", or an earlier build\'s id or link). Read both reports with fetchAccessibilityIssues and match issues by rule and component (the element\'s id or class); an issue in the later report only is new, in the earlier only is fixed, in both is still open.', + ); + expect(m.appBuildIssues).not.toHaveBeenCalled(); }); }); diff --git a/tests/tools/accessibilityAutomateBuildId.test.ts b/tests/tools/accessibilityAutomateBuildId.test.ts new file mode 100644 index 00000000..cc09cfa6 --- /dev/null +++ b/tests/tools/accessibilityAutomateBuildId.test.ts @@ -0,0 +1,178 @@ +import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; + +// A build id that is an Automate or App Automate build's own hashed_id (40 hex characters), +// as the SDK prints it at the end of a run, is read as the accessibility build that run made. +// Neither the web nor the app Accessibility API knows such an id, so Automate is asked, then +// App Automate. A probe that can't be answered (anything but a 404) moves on to the next, +// and is named if nothing knows the id; once a session is found, what it says is the answer. + +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: vi.fn() }, +})); + +import { apiClient } from "../../src/lib/apiClient.js"; +import { + AccessibilityReportFetcher, + foundScanRuns, +} from "../../src/tools/accessiblity-utils/report-fetcher.js"; + +const get = apiClient.get as Mock; + +const HASHED = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c"; // the Automate / App Automate build +const SESSION = "1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d"; +const WEB_BUILD = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; // the accessibility build it made +const APP_BUILD = "a9appbuildgggggggggggggggggggggggggggggg"; + +type Reply = { ok: boolean; status: number; data: unknown }; +const reply = (status: number, data: unknown = null): Reply => ({ + ok: status >= 200 && status < 300, + status, + data, +}); +const notFound = reply(404, { message: "not found" }); + +/** What each API answers; anything not set is a 404. */ +let answers: { + automateSessions?: Reply; + appAutomateSessions?: Reply; + testCode?: Reply; + native?: Reply; +}; + +function route({ url }: { url: string }): Reply { + const path = new URL(url).pathname; + if (path.endsWith(`/automated-tests/v1/builds/${WEB_BUILD}`)) { + return reply(200, { success: true, data: { id: WEB_BUILD } }); + } + if (path === `/automate/builds/${HASHED}/sessions.json`) { + return answers.automateSessions ?? notFound; + } + if (path === `/app-automate/builds/${HASHED}/sessions.json`) { + return answers.appAutomateSessions ?? notFound; + } + if (path.endsWith(`/ext/v1/sessions/${SESSION}/testCode`)) { + return answers.testCode ?? notFound; + } + if (/\/app-automate\/(espresso|xcuitest)\/v2\/builds\//.test(path)) { + return answers.native ?? notFound; + } + // The web and app Accessibility APIs don't know a hashed_id. + return notFound; +} + +const sessions = [{ automation_session: { hashed_id: SESSION } }]; +const paths = () => get.mock.calls.map(([o]) => new URL(o.url).pathname); + +function fetcher() { + const f = new AccessibilityReportFetcher(); + f.setAuth({ username: "u", password: "p" }); + return f; +} +const resolve = (buildId: string, orScanRun = true) => + fetcher().resolveBuild({ kind: "build", buildId }, { orScanRun }); + +beforeEach(() => { + get.mockReset(); + get.mockImplementation(async (o: { url: string }) => route(o)); + foundScanRuns.clear(); + answers = {}; +}); + +describe("a 40-hex Automate or App Automate build id passed as buildId", () => { + it("reads an Automate build's id as the accessibility build its sessions ran in", async () => { + answers.automateSessions = reply(200, sessions); + answers.testCode = reply(200, { build_uuid: WEB_BUILD }); + + await expect(resolve(HASHED)).resolves.toEqual({ + kind: "build", + buildId: WEB_BUILD, + }); + expect(paths()).not.toContain(`/app-automate/builds/${HASHED}/sessions.json`); + }); + + it("asks App Automate when Automate does not know the id", async () => { + answers.appAutomateSessions = reply(200, sessions); + answers.testCode = reply(200, { build_uuid: APP_BUILD }); + + await expect(resolve(HASHED)).resolves.toEqual({ + kind: "appBuild", + buildId: APP_BUILD, + }); + expect(paths()).toContain(`/automate/builds/${HASHED}/sessions.json`); + }); + + it.each([ + ["refuses the request (403)", reply(403, { message: "forbidden" })], + ["fails (500)", reply(500, { message: "boom" })], + ["lists no sessions for the build", reply(200, [])], + ])( + "moves on to App Automate when Automate %s", + async (_case, automate) => { + answers.automateSessions = automate; + answers.appAutomateSessions = reply(200, sessions); + answers.testCode = reply(200, { build_uuid: APP_BUILD }); + + await expect(resolve(HASHED)).resolves.toEqual({ + kind: "appBuild", + buildId: APP_BUILD, + }); + }, + ); + + it.each([ + ["refuses the request (403)", reply(403, { message: "forbidden" }), "HTTP 403"], + ["lists no sessions for the build", reply(200, []), "has no sessions"], + ])( + "names why Automate could not be asked, when Automate %s and nothing else knows the id", + async (_case, automate, said) => { + answers.automateSessions = automate; + + const error = await resolve(HASHED).catch((e) => e); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain(`No Automated Tests (web) or App Accessibility build with id "${HASHED}"`); + expect(error.message).toContain("(Automate could not be asked about it: "); + expect(error.message).toContain(said); + // A 40-hex id is an Automate id: it is not then sent on as a Website Scanner run's. + expect(paths().some((p) => p.includes("website-scanner"))).toBe(false); + }, + ); + + it("names an App Automate lookup that failed too", async () => { + answers.automateSessions = reply(503); + answers.appAutomateSessions = reply(502); + + const error = await resolve(HASHED, false).catch((e) => e); + + expect(error.message).toContain(`Could not list the sessions of Automate build ${HASHED}: HTTP 503`); + expect(error.message).toContain(`Could not list the sessions of App Automate build ${HASHED}: HTTP 502`); + }); + + it("takes an id nothing knows as a Website Scanner run's build id when every API answered", async () => { + await expect(resolve(HASHED)).resolves.toEqual({ + kind: "scanBuild", + thBuildId: HASHED, + }); + }); + + it("returns what a session that was found says, rather than moving on", async () => { + answers.automateSessions = reply(200, sessions); + answers.testCode = reply(200, { build_uuid: undefined }); + answers.appAutomateSessions = reply(200, sessions); + + await expect(resolve(HASHED)).rejects.toThrow( + `Automate session ${SESSION} is not linked to a BrowserStack SDK build`, + ); + expect(paths()).not.toContain(`/app-automate/builds/${HASHED}/sessions.json`); + }); + + it("does not ask Automate about an id that is not 40 hex characters", async () => { + const scannerBuildId = "zz7ocvkf03ilbyscee9azhiqh1q6cgcqbsjhijdm"; + + await expect(resolve(scannerBuildId)).resolves.toEqual({ + kind: "scanBuild", + thBuildId: scannerBuildId, + }); + expect(paths().some((p) => p.includes("/automate/builds/"))).toBe(false); + }); +}); diff --git a/tests/tools/accessibilityExpertQuery.test.ts b/tests/tools/accessibilityExpertQuery.test.ts new file mode 100644 index 00000000..eac8c925 --- /dev/null +++ b/tests/tools/accessibilityExpertQuery.test.ts @@ -0,0 +1,115 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// The accessibility knowledge base refuses a question over 500 characters ("Query is too +// long"). A longer one is shortened: its last sentence (the question itself, which agents put +// last) and as much of its opening as fits, or only its opening when it has one sentence or +// a last sentence too long to keep. The note says which was sent. + +const rag = vi.hoisted(() => ({ query: vi.fn() })); +vi.mock("../../src/tools/accessiblity-utils/accessibility-rag.js", () => ({ + queryAccessibilityRAG: rag.query, +})); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); +vi.mock("../../src/index.js", () => ({ trackMCP: vi.fn(), PackageJsonVersion: "0.0.0" })); + +import addAccessibilityTools from "../../src/tools/accessibility.js"; +import { + EXPERT_QUERY_MAX_CHARS, + fitExpertQuery, +} from "../../src/tools/accessiblity-utils/expert-query.js"; + +const context = (n: number) => + Array.from({ length: n }, (_, i) => `Context sentence number ${i} about the checkout form.`).join(" "); +const QUESTION = "How do I fix the missing form labels?"; + +/** True when a string holds a high surrogate without the low one after it, or the reverse. */ +const hasLoneSurrogate = (s: string) => + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + it("leaves a question within the limit as it is", () => { + expect(fitExpertQuery(QUESTION)).toEqual({ query: QUESTION, shortened: false }); + }); + + it("keeps the last sentence and as much of the opening as fits", () => { + const long = `${context(15)} ${QUESTION}`; + + const fitted = fitExpertQuery(long); + + expect(long.length).toBeGreaterThan(EXPERT_QUERY_MAX_CHARS); + expect(fitted.shortened).toBe(true); + expect(fitted.kept).toBe("opening and last sentence"); + expect(fitted.query.length).toBeLessThanOrEqual(EXPERT_QUERY_MAX_CHARS); + expect(fitted.query.startsWith("Context sentence number 0")).toBe(true); + expect(fitted.query.endsWith(` … ${QUESTION}`)).toBe(true); + }); + + it("keeps only the opening of a question with one long sentence, and says so", () => { + const fitted = fitExpertQuery(`${"word ".repeat(150)}end`); + + expect(fitted.kept).toBe("opening"); + expect(fitted.query.length).toBeLessThanOrEqual(EXPERT_QUERY_MAX_CHARS); + expect(fitted.query).not.toContain("…"); + }); + + it("splits Chinese and Japanese text at 。？！, so the question at the end is kept", () => { + const zhQuestion = "如何修复缺少的表单标签？"; + const long = `${"这是关于结账表单的背景说明。".repeat(40)}${zhQuestion}`; + + const fitted = fitExpertQuery(long); + + expect(fitted.kept).toBe("opening and last sentence"); + expect(fitted.query.endsWith(zhQuestion)).toBe(true); + expect(fitted.query.length).toBeLessThanOrEqual(EXPERT_QUERY_MAX_CHARS); + }); + + it("never cuts between the two halves of a character outside the BMP", () => { + // 499 ASCII characters, then emoji: a cut at 500 would land inside the first one. + const fitted = fitExpertQuery(`${"a".repeat(EXPERT_QUERY_MAX_CHARS - 1)}${"😀".repeat(20)}`); + + expect(fitted.shortened).toBe(true); + expect(hasLoneSurrogate(fitted.query)).toBe(false); + expect(fitted.query.length).toBeLessThanOrEqual(EXPERT_QUERY_MAX_CHARS); + }); +}); + +describe("accessibilityExpert with a long question", () => { + let ask: (query: string) => Promise; + + beforeEach(() => { + rag.query.mockReset(); + rag.query.mockResolvedValue({ content: [{ type: "text", text: "Answer." }] }); + let handler: (...args: any[]) => any = () => undefined; + const server: any = { + tool: vi.fn((...args: any[]) => { + if (args[0] === "accessibilityExpert") handler = args[args.length - 1]; + }), + server: { getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }) }, + }; + addAccessibilityTools(server, { "browserstack-username": "u", "browserstack-access-key": "k" } as any); + ask = (query) => handler({ query }); + }); + + it.each([ + ["opening and last sentence", `${context(15)} ${QUESTION}`], + ["opening", `${"word ".repeat(150)}end`], + ])("says the %s was sent, matching what was cut", async (kept, query) => { + const result = await ask(query); + + const sent: string = rag.query.mock.calls[0][0]; + expect(result.content[0].text).toBe( + `The question was ${query.length} characters; the knowledge base takes at most ${EXPERT_QUERY_MAX_CHARS}, so ${sent.length} were sent: its ${kept}. If the answer misses the point, ask again more briefly.`, + ); + expect(result.content.at(-1).text).toBe("Answer."); + }); + + it("adds no note to a question within the limit", async () => { + const result = await ask(QUESTION); + + expect(rag.query.mock.calls[0][0]).toBe(QUESTION); + expect(result.content).toEqual([{ type: "text", text: "Answer." }]); + }); +}); diff --git a/tests/tools/accessibilityFailedAndSkippedPages.test.ts b/tests/tools/accessibilityFailedAndSkippedPages.test.ts new file mode 100644 index 00000000..39828ea4 --- /dev/null +++ b/tests/tools/accessibilityFailedAndSkippedPages.test.ts @@ -0,0 +1,311 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// What fetchAccessibilityIssues says of a run that did not check its pages: a run that +// failed, pages a completed scan could not scan, and pages that showed an error page that +// was scanned in their place. Each used to read as a clean or complete report. + +const m = vi.hoisted(() => ({ + get: vi.fn(), + getReportLink: vi.fn(), + reportStatus: vi.fn(), + resolveBuild: vi.fn(), + previousScanRun: vi.fn(), + changesSinceLastRun: vi.fn(), + appBuildReport: vi.fn(), + unscannedPages: vi.fn(), + failureDetails: vi.fn(), + dashboardLink: vi.fn(), +})); + +// The CSV export is served here and read by the real report parser. +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: m.get, post: vi.fn() }, +})); +vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ + AccessibilityReportFetcher: class { + setAuth = vi.fn(); + getReportLink = m.getReportLink; + reportStatus = m.reportStatus; + resolveBuild = m.resolveBuild; + previousScanRun = m.previousScanRun; + changesSinceLastRun = m.changesSinceLastRun; + appBuildReport = m.appBuildReport; + }, + EXPORT_WAIT_MS: 45_000, + ExportPendingError: class extends Error {}, +})); +vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ + AccessibilityScanner: class { + setAuth = vi.fn(); + unscannedPages = m.unscannedPages; + failureDetails = m.failureDetails; + dashboardLink = m.dashboardLink; + }, + ScanTimeoutError: class extends Error {}, +})); +vi.mock("../../src/lib/get-auth.js", () => ({ + getBrowserStackAuth: vi.fn().mockReturnValue("fake-user:fake-key"), +})); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); +vi.mock("../../src/index.js", () => ({ trackMCP: vi.fn(), PackageJsonVersion: "0.0.0" })); + +import addAccessibilityTools from "../../src/tools/accessibility.js"; + +function fetchIssues() { + let handler: (...args: any[]) => any = () => undefined; + const server: any = { + tool: vi.fn((...args: any[]) => { + if (args[0] === "fetchAccessibilityIssues") handler = args[args.length - 1]; + }), + server: { getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }) }, + }; + addAccessibilityTools(server, { + "browserstack-username": "fake-user", + "browserstack-access-key": "fake-key", + } as any); + return (args: Record) => + handler(args, { sendNotification: vi.fn(), _meta: {} }); +} + +const text = (result: any) => + result.content.map((c: { text: string }) => c.text).join("\n"); + +/** The content of the first untrusted-data block with this label. */ +function quoted(out: string, label: string): string { + const block = out.match( + new RegExp(`«UNTRUSTED ${label} ([0-9a-f]+)»\\n([\\s\\S]*?)\\n«END UNTRUSTED \\1»`), + ); + return block?.[2] ?? ""; +} + +const RUN = { scanId: "4300543", scanRunId: "2420156" }; +const BUILD_ID = "xuychmygbnlorshyefzt7m9mtuqamfs4qbeswfiu"; + +const COLUMNS = ["Issue type", "Severity", "Component", "Issue description", "Affected page", "HTML snippet", "CSS selector", "How to fix this issue", "Issue link", "Duplicate Issue of", "Issue status"]; +const csv = (rows: Record[]) => + [COLUMNS, ...rows.map((r) => COLUMNS.map((c) => r[c] ?? ""))] + .map((cells) => cells.map((v) => `"${v.replace(/"/g, '""')}"`).join(",")) + .join("\n"); +const issue = (page: string, type: string, snippet = "") => ({ + "Issue type": type, + Severity: "serious", + Component: type, + "Issue description": "d", + "Affected page": page, + "HTML snippet": snippet, + "CSS selector": type, + "How to fix this issue": "fix", + "Issue link": `https://dash.example/${type}-${page}`, + "Duplicate Issue of": "NA", + "Issue status": "visible", +}); + +const PAGE_A = "http://bs-local.com:8000/"; +const PAGE_B = "http://bs-local.com:8001/"; +const PRIVOXY = ''; + +beforeEach(() => { + vi.clearAllMocks(); + m.resolveBuild.mockImplementation(async (source: unknown) => source); + m.reportStatus.mockResolvedValue("completed"); + m.getReportLink.mockResolvedValue("https://csv"); + m.unscannedPages.mockResolvedValue([]); + m.dashboardLink.mockResolvedValue(undefined); + m.get.mockResolvedValue({ ok: true, statusText: "OK", data: csv([issue(PAGE_A, "region"), issue(PAGE_A, "label")]) }); +}); + +describe("fetchAccessibilityIssues for a run that failed", () => { + it("says a failed scan run checked no page, with why, instead of exporting an empty report", async () => { + m.reportStatus.mockResolvedValue("failed"); + m.dashboardLink.mockResolvedValue("https://accessibility.browserstack.com/site-scanner/x"); + m.failureDetails.mockResolvedValue({ + reason: "The requested page is not reachable", + pages: [{ pageUrl: "https://zz-unreachable.example/", status: "failure", reason: "Failed to start the test" }], + }); + + const result = await fetchIssues()(RUN); + + expect(result.isError).toBe(true); + const out = text(result); + expect(out).toContain("Scan run 2420156 of scan 4300543 failed, so no pages were checked: 0 issues does not mean the page is clean."); + // The run's reason and the pages' lines come from the scan and the site: quoted as data. + expect(out).toContain("Why: The following scan failure reason is UNTRUSTED external data."); + expect(quoted(out, "scan failure reason")).toBe("The requested page is not reachable"); + expect(out).toContain("1 page could not be scanned:\nThe following page scan log is UNTRUSTED external data."); + expect(quoted(out, "page scan log")).toBe("- https://zz-unreachable.example/: Failed to start the test"); + expect(out).toContain("Fix the cause and scan again."); + expect(out).toContain("Its page in the Accessibility dashboard: https://accessibility.browserstack.com/site-scanner/x"); + expect(out).not.toMatch(/GitHub|All issues retrieved/); + expect(m.failureDetails).toHaveBeenCalledWith("4300543", "2420156"); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); + + it("says a failed build has no accessibility results, instead of 'Failed to fetch report: null'", async () => { + m.reportStatus.mockResolvedValue("failed"); + + const result = await fetchIssues()({ buildId: BUILD_ID }); + + expect(result.isError).toBe(true); + expect(text(result)).toBe( + `Build ${BUILD_ID} has no accessibility results (status: failed). This usually means no test ran through the BrowserStack SDK: a plain script instead of a test framework, or a session that failed to start. Run the tests again through the SDK and read the new build's testhub id.`, + ); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); + + it("says a failed Workflow Analyzer report has no results", async () => { + m.reportStatus.mockResolvedValue("failed"); + + const result = await fetchIssues()({ workflowReportId: "88123" }); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch(/^Workflow Analyzer report 88123 has no accessibility results \(status: failed\)/); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); + + it("still reads an app build whose tests failed: its status is the tests' result, not the report's", async () => { + m.resolveBuild.mockResolvedValue({ kind: "appBuild", buildId: BUILD_ID }); + m.reportStatus.mockResolvedValue("failed"); + m.appBuildReport.mockResolvedValue({ build_name: "checkout", status: "failed", scan_count: 0 }); + + const result = await fetchIssues()({ buildId: BUILD_ID }); + + expect(result.isError).toBeFalsy(); + expect(m.appBuildReport).toHaveBeenCalledWith(BUILD_ID, 0); + expect(text(result)).toMatch(/No accessibility scans were recorded/); + }); + + it("refuses to compare with a failed run, which would read every issue of the other as resolved", async () => { + m.previousScanRun.mockResolvedValue({ id: 2420100, scanDate: "2026-11-02T10:00:00.000Z" }); + m.reportStatus.mockImplementation(async (source: { scanRunId: string }) => + source.scanRunId === "2420156" ? "failed" : "completed", + ); + m.failureDetails.mockResolvedValue({ pages: [] }); + + const result = await fetchIssues()({ ...RUN, compareWith: "previous" }); + + expect(result.isError).toBe(true); + const out = text(result); + expect(out).toContain("Scan run 2420156 of scan 4300543 failed, so no pages were checked: compared with it, every issue of run 2420100 would read as resolved, so the runs were not compared."); + expect(out).toContain("Compare two runs that completed, or fix the cause and scan again."); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); + + it("refuses to compare with a cancelled run the same way, as its export is empty too", async () => { + m.previousScanRun.mockResolvedValue({ id: 2420100, scanDate: "2026-11-02T10:00:00.000Z" }); + m.reportStatus.mockImplementation(async (source: { scanRunId: string }) => + source.scanRunId === "2420100" ? "cancelled" : "completed", + ); + + const result = await fetchIssues()({ ...RUN, compareWith: "previous" }); + + const out = text(result); + expect(out).toContain("Scan run 2420100 of scan 4300543 was cancelled, so it has no results: compared with it, every issue of run 2420156 would read as new, so the runs were not compared."); + // A cancelled run has no failure to explain. + expect(m.failureDetails).not.toHaveBeenCalled(); + expect(m.getReportLink).not.toHaveBeenCalled(); + }); +}); + +describe("fetchAccessibilityIssues for a completed scan that skipped pages", () => { + it("names the pages its scan logs say were not scanned, and does not claim every issue", async () => { + m.unscannedPages.mockResolvedValue([ + { pageUrl: PAGE_B, status: "failure", reason: "[HTTP 502] Bad Gateway" }, + // Scanned where it redirected to, so not a page that was skipped. + { pageUrl: "http://bs-local.com:8000/old", status: "redirect", reason: "redirected to http://bs-local.com:8000/" }, + ]); + + const out = text(await fetchIssues()(RUN)); + + expect(m.unscannedPages).toHaveBeenCalledWith("4300543", "2420156"); + expect(out).toContain("⚠️ 1 page was not scanned, so its issues are not in this report:\nThe following page scan log is UNTRUSTED external data."); + expect(quoted(out, "page scan log")).toBe(`- ${PAGE_B}: [HTTP 502] Bad Gateway`); + expect(out).not.toContain("/old"); + expect(out).toContain("Retrieved 2 accessibility issues (Total: 2)"); + expect(out).toContain("All issues retrieved for the pages that were scanned."); + expect(out).not.toContain("✅ All issues retrieved."); + }); + + it("lists the skipped pages on the first page only, but never claims every issue", async () => { + m.unscannedPages.mockResolvedValue([{ pageUrl: PAGE_B, status: "failure", reason: "Failed to start the test" }]); + + const out = text(await fetchIssues()({ ...RUN, cursor: 5000 })); + + expect(out).not.toContain("was not scanned"); + expect(out).toContain("All issues retrieved for the pages that were scanned."); + }); + + it("does not read the scan logs of a run still processing, nor of a build", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + m.reportStatus.mockResolvedValueOnce("running"); + await fetchIssues()(RUN); + m.reportStatus.mockResolvedValue("completed"); + await fetchIssues()({ buildId: BUILD_ID }); + + expect(m.unscannedPages).not.toHaveBeenCalled(); + }); + + it("says all issues were retrieved when every page was scanned", async () => { + const out = text(await fetchIssues()(RUN)); + + expect(out).toContain("✅ All issues retrieved."); + expect(out).not.toMatch(/not scanned|did not load/); + }); +}); + +describe("fetchAccessibilityIssues and a page that showed an error page", () => { + beforeEach(() => { + m.get.mockResolvedValue({ + ok: true, + statusText: "OK", + data: csv([ + issue(PAGE_A, "region"), + issue(PAGE_A, "label"), + // BrowserStack Local's "not connected" page, scanned under the page's address. + issue(PAGE_B, "keyboard-focus-visible", PRIVOXY), + issue(PAGE_B, "decorative-image", 'BrowserStack Logo'), + ]), + }); + }); + + it("leads with the page that did not load and leaves the error page's issues out of the list and the total", async () => { + const out = text(await fetchIssues()(RUN)); + + expect(out).toContain( + "⚠️ 1 page did not load; an error page was scanned in its place. The 2 issues found there are from the error page, not your page, so they are left out of the issues and the total below. Check the page is reachable (and the Local tunnel, for localhost), then scan again.", + ); + // The address comes from the report, so it is quoted as data. + expect(out.indexOf(PAGE_B)).toBeGreaterThan(out.indexOf("«UNTRUSTED")); + expect(out.indexOf("did not load")).toBeLessThan(out.indexOf("Retrieved")); + expect(out).toContain("Retrieved 2 accessibility issues (Total: 2)"); + expect(out).toContain("Summary of the 2 issues: 2 serious."); + expect(out).not.toMatch(/keyboard-focus-visible|decorative-image|dashboard-privoxy-page/); + expect(out).toContain("All issues retrieved for the pages that were scanned."); + }); + + it("does not also list the page as not scanned when its scan log failed too", async () => { + m.unscannedPages.mockResolvedValue([{ pageUrl: "http://BS-LOCAL.com:8001", status: "failure", reason: "[HTTP 502] Bad Gateway" }]); + + const out = text(await fetchIssues()(RUN)); + + expect(out).toContain("1 page did not load"); + expect(out).not.toContain("not scanned"); + }); + + it("says a build's page did not load, and to run the tests again", async () => { + const out = text(await fetchIssues()({ buildId: BUILD_ID })); + + expect(out).toContain("Check the page is reachable (and the Local tunnel, for localhost), then run the tests again."); + expect(out).toContain("Retrieved 2 accessibility issues (Total: 2)"); + }); + + it("read for the page that did not load, says so rather than that the page has no issues", async () => { + const out = text(await fetchIssues()({ ...RUN, pageUrl: PAGE_B })); + + expect(out).toContain("1 page did not load"); + expect(out).toContain("Retrieved 0 accessibility issues (Total: 0)"); + expect(out).not.toContain("No issue in this report is on"); + }); +}); diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index bffbf768..b7255c06 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -8,6 +8,7 @@ const m = vi.hoisted(() => ({ sourceFromReportUrl: vi.fn(), reportStatus: vi.fn(), previousScanRun: vi.fn(), + scanRuns: vi.fn(), changesSinceLastRun: vi.fn(), parse: vi.fn(), loadMainIssues: vi.fn(), @@ -23,6 +24,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ reportStatus = m.reportStatus; resolveBuild = vi.fn(async (source: unknown) => source); previousScanRun = m.previousScanRun; + scanRuns = m.scanRuns; changesSinceLastRun = m.changesSinceLastRun; findScanRun = m.findScanRun; }, @@ -36,6 +38,14 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ } }, })); +// A completed scan's read also reads its scan logs, for the pages it could not scan. +vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ + AccessibilityScanner: class { + setAuth = vi.fn(); + unscannedPages = vi.fn(async () => []); + }, + ScanTimeoutError: class extends Error {}, +})); vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ parseAccessibilityReportFromCSV: m.parse, loadMainIssues: m.loadMainIssues, @@ -196,6 +206,14 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { it("compares with a run it is given, counting by matching the issues", async () => { m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.scanRuns.mockResolvedValue({ + runs: [ + { id: 5520318, scanDate: "2026-10-22T09:00:00.000Z" }, + { id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }, + { id: 5520281, scanDate: "2026-10-20T08:00:00.000Z" }, + ], + complete: true, + }); const result = await fetchIssues()({ ...RUN, compareWith: "5520281" }); @@ -261,10 +279,19 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { expect(out).toMatch(/Not checked against the earlier run \(page did not load in run 5520296\)/); }); - it("compares Website Scanner runs only", async () => { - const result = await fetchIssues()({ buildId: "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja", compareWith: "previous" }); + // A report with no built-in compare is answered with how to compare it by hand, as + // guidance rather than an error, so the agent goes straight to reading both reports. + const READ_BOTH = + " Read both reports with fetchAccessibilityIssues and match issues by rule and CSS selector on each page; an issue in the later report only is new, in the earlier only is fixed, in both is still open."; - expect(result.isError).toBe(true); - expect(text(result)).toMatch(/Website Scanner/); + it("says to read both Workflow Analyzer reports and compare them, as guidance rather than an error", async () => { + const result = await fetchIssues()({ workflowReportId: "277527", compareWith: "277401" }); + + expect(result.isError).toBeFalsy(); + expect(text(result).trim()).toBe(`Workflow Analyzer reports have no built-in compare.${READ_BOTH}`); + expect(text(result)).not.toMatch(/GitHub/); + expect(m.getReportLink).not.toHaveBeenCalled(); + expect(m.reportStatus).not.toHaveBeenCalled(); }); + }); diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index a560875f..114aaaa4 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -14,8 +14,12 @@ import { MAX_SCAN_LIST_PAGES, reportExports, MAX_REPORT_EXPORTS, + foundScanRuns, } from "../../src/tools/accessiblity-utils/report-fetcher.js"; +// Runs found by build id are cached for the process; each test starts without them. +beforeEach(() => foundScanRuns.clear()); + const get = apiClient.get as Mock; const API = "https://api-accessibility.browserstack.com/api"; const BUILD_ID = "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja"; @@ -153,6 +157,19 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { ]); }); + it("remembers the run a link's build id names, so a second lookup does not walk the list again", async () => { + get + .mockResolvedValueOnce(scansPage([scan(1, "other", "aaa", 11)], "tok2")) + .mockResolvedValueOnce(scansPage([scan(1234, "checkout fix", "bbb", 5678)])); + + const first = await fetcher().findScanRun(linkRun("checkout fix", "bbb")); + get.mockClear(); + const again = await fetcher().findScanRun(linkRun("checkout fix", "bbb")); + + expect(again).toEqual(first); + expect(get).not.toHaveBeenCalled(); + }); + it("names the scan when the link is to one of its earlier runs", async () => { get.mockResolvedValueOnce( scansPage([scan(1234, "checkout fix", "newer", 5678)]), @@ -471,6 +488,47 @@ describe("AccessibilityReportFetcher.getReportLink on a large report", () => { expect(urls().filter((u) => u.includes("scan_run_id="))).toHaveLength(2); }); + it("does not keep the task of an export started while the report is processing", async () => { + get.mockResolvedValueOnce(ok({ task_id: "t:1", message: "started" })); + get.mockResolvedValue(pending); + const first = fetcher().getReportLink(SOURCE, { stillProcessing: true }); + const pendingFirst = expect(first).rejects.toBeInstanceOf(ExportPendingError); + await vi.advanceTimersByTimeAsync(EXPORT_WAIT_MS + 10_000); + await pendingFirst; + + // Once the report has finished, the next call starts a new export instead of taking + // the snapshot that one was making as final. + get.mockReset(); + get + .mockResolvedValueOnce(ok({ task_id: "t:2", message: "started" })) + .mockResolvedValueOnce(ok({ reportLink: "https://csv2?Expires=9999999999" })); + await expect(fetcher().getReportLink(SOURCE)).resolves.toBe("https://csv2?Expires=9999999999"); + expect(urls()[0]).toBe( + "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/1234/scan_runs/issues?scan_run_id=5678", + ); + expect(urls().some((u) => u.includes("task_id=t%3A1"))).toBe(false); + }); + + it("reuses a run read by its build id for 5 minutes at most, as its processing status is not checked", async () => { + const BY_BUILD_ID: IssueSource = { kind: "scanBuild", thBuildId: BUILD_ID }; + const expires = Math.floor(Date.now() / 1000) + 12 * 3600; + get + .mockResolvedValueOnce(ok({ task_id: "t:1", message: "started" })) + .mockResolvedValueOnce(ok({ reportLink: `https://csv?Expires=${expires}` })); + + await fetcher().getReportLink(BY_BUILD_ID); + vi.setSystemTime(Date.now() + 4 * 60 * 1000); + await expect(fetcher().getReportLink(BY_BUILD_ID)).resolves.toBe(`https://csv?Expires=${expires}`); + expect(get).toHaveBeenCalledTimes(2); + + vi.setSystemTime(Date.now() + 2 * 60 * 1000); + get + .mockResolvedValueOnce(ok({ task_id: "t:2", message: "started" })) + .mockResolvedValueOnce(ok({ reportLink: "https://csv2?Expires=9999999999" })); + await expect(fetcher().getReportLink(BY_BUILD_ID)).resolves.toBe("https://csv2?Expires=9999999999"); + expect(get).toHaveBeenCalledTimes(4); + }); + it("remembers a bounded number of reports' exports", async () => { get.mockResolvedValue(ok({ reportLink: "https://csv?Expires=9999999999" })); diff --git a/tests/tools/accessibilityReportLinks.test.ts b/tests/tools/accessibilityReportLinks.test.ts index 03f1a6b5..b0b4cae7 100644 --- a/tests/tools/accessibilityReportLinks.test.ts +++ b/tests/tools/accessibilityReportLinks.test.ts @@ -11,11 +11,15 @@ vi.mock("../../src/lib/apiClient.js", () => ({ import { apiClient } from "../../src/lib/apiClient.js"; import { AccessibilityReportFetcher, + foundScanRuns, reportExports, ScanRunNotFoundError, scanRunLink, } from "../../src/tools/accessiblity-utils/report-fetcher.js"; +// Runs found by build id are cached for the process; each test starts without them. +beforeEach(() => foundScanRuns.clear()); + const get = apiClient.get as Mock; const API = "https://api-accessibility.browserstack.com/api"; const DASHBOARD = "https://accessibility.browserstack.com"; diff --git a/tests/tools/accessibilityRescan.test.ts b/tests/tools/accessibilityRescan.test.ts new file mode 100644 index 00000000..79cca8a9 --- /dev/null +++ b/tests/tools/accessibilityRescan.test.ts @@ -0,0 +1,670 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Re-scanning a page after a fix used to start a second scan, and runs of two scans cannot +// be compared. Given the earlier scan's id, startAccessibilityScan starts a new run of that +// scan instead (POST /scans/:id/new_scan), but only when that scan is for the same page. +// And a refusal of a 51st scan read as a failure to report on GitHub. + +const m = vi.hoisted(() => ({ + post: vi.fn(), + get: vi.fn(), + startScan: vi.fn(), + savedScan: vi.fn(), + rerunScan: vi.fn(), + waitUntilComplete: vi.fn(), + runSettings: vi.fn(), + getReportLink: vi.fn(), + parse: vi.fn(), +})); + +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post, get: m.get } })); +vi.mock("../../src/lib/local.js", () => ({ + isLocalURL: (url: string) => new URL(url).hostname === "localhost", + ensureLocalBinarySetup: vi.fn(), + killExistingBrowserStackLocalProcesses: vi.fn(), +})); +vi.mock("../../src/config.js", () => ({ + default: { REMOTE_MCP: false, USE_OWN_LOCAL_BINARY_PROCESS: false }, +})); +vi.mock("../../src/logger.js", () => ({ + default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); +vi.mock("../../src/index.js", () => ({ trackMCP: vi.fn(), PackageJsonVersion: "0.0.0" })); + +const SCANS = "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans"; + +async function scannerModule() { + return vi.importActual( + "../../src/tools/accessiblity-utils/scanner.js", + ); +} + +async function scanner() { + const { AccessibilityScanner } = await scannerModule(); + const s = new AccessibilityScanner(); + s.setAuth({ username: "u", password: "p" }); + return s; +} + +const ok = (data: unknown) => ({ ok: true, status: 200, data: { success: true, data } }); +const refused = (status: number, error: unknown) => ({ + response: { status, data: { success: false, data: null, error } }, +}); + +describe("samePage", () => { + it.each([ + ["https://example.com/", "https://example.com"], + ["https://example.com/", "https://EXAMPLE.com/"], + ["https://example.com/docs/", "https://example.com/docs"], + ["http://bs-local.com:3000/cart", "http://localhost:3000/cart"], + ["http://bs-local.com:3000/", "http://127.0.0.1:3000"], + ])("takes %s and %s for the same page", async (saved, page) => { + expect((await scannerModule()).samePage(saved, page)).toBe(true); + }); + + it.each([ + ["https://example.com/", "https://example.org/"], + ["https://example.com/a", "https://example.com/b"], + ["https://example.com/?q=1", "https://example.com/?q=2"], + ["http://bs-local.com:3000/", "http://localhost:4000/"], + ])("tells %s and %s apart", async (saved, page) => { + expect((await scannerModule()).samePage(saved, page)).toBe(false); + }); +}); + +describe("AccessibilityScanner.savedScan", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it("reads the saved page and settings from the scan's overview", async () => { + m.get.mockResolvedValue( + ok({ + scanConfigInfo: { + name: "zz-home", + wcagVersion: "wcag21aa", + needsReview: true, + bestPractices: false, + advanced: true, + experimental: false, + authEnabled: false, + localTestingEnabled: true, + }, + urlList: ["http://bs-local.com:8799/"], + }), + ); + + const saved = await (await scanner()).savedScan("4304809"); + + expect(m.get.mock.calls[0][0].url).toBe(`${SCANS}/4304809/overview`); + expect(saved).toEqual({ + name: "zz-home", + urlList: ["http://bs-local.com:8799/"], + settings: { + wcagVersion: "wcag21aa", + needsReview: true, + bestPractices: false, + advancedRules: true, + experimentalRules: false, + }, + authEnabled: false, + localTestingEnabled: true, + }); + }); + + it("leaves out a setting the overview does not give", async () => { + m.get.mockResolvedValue(ok({ scanConfigInfo: { name: "old", advanced: true }, urlList: ["https://a.example/"] })); + + const saved = await (await scanner()).savedScan("1"); + + expect(saved.settings).toEqual({ advancedRules: true }); + expect(saved.authEnabled).toBeUndefined(); + expect(saved.localTestingEnabled).toBeUndefined(); + }); + + it("says no scan with that id is visible, for a 404 or an id that is not a number", async () => { + const { ScanRefusedError } = await scannerModule(); + m.get.mockResolvedValue({ ok: false, status: 404, data: { success: false, error: "Scan logs not found" } }); + + const unknown = await (await scanner()).savedScan("999999999").catch((e) => e); + expect(unknown).toBeInstanceOf(ScanRefusedError); + expect(unknown.message).toBe( + "No scan with id 999999999 is visible to this account, so nothing was started. Leave out scanId to start a new scan.", + ); + + m.get.mockClear(); + const named = await (await scanner()).savedScan("zz-rescan-a").catch((e) => e); + expect(named).toBeInstanceOf(ScanRefusedError); + expect(named.message).toContain("No scan with id zz-rescan-a is visible to this account"); + expect(m.get).not.toHaveBeenCalled(); + }); +}); + +describe("AccessibilityScanner.rerunScan", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it("starts a new run of the scan, sending no settings", async () => { + m.post.mockResolvedValue({ data: { success: true, data: { scanRunId: 2425999 } } }); + + await expect((await scanner()).rerunScan("4304819")).resolves.toEqual({ scanRunId: "2425999" }); + expect(m.post.mock.calls[0][0].url).toBe(`${SCANS}/4304819/new_scan`); + expect(m.post.mock.calls[0][0].body).toEqual({}); + }); + + it("names the run still going, from the scans list", async () => { + const { ScanRefusedError } = await scannerModule(); + m.post.mockRejectedValue(refused(422, "A New Scan is already in progress")); + m.get.mockImplementation(async ({ url }: { url: string }) => + url === SCANS + ? ok({ scans: [{ id: 4304819, lastScanDetails: { scanRunId: 2425998, scanStatus: "processing" } }] }) + : ok({ scanRuns: [] }), + ); + + const error = await (await scanner()).rerunScan("4304819").catch((e) => e); + + expect(error).toBeInstanceOf(ScanRefusedError); + expect(error.message).toBe( + 'A run of scan 4304819 is still going (run 2425998, processing), so a new run was not started. Read that run\'s issues with fetchAccessibilityIssues (scanId "4304819", scanRunId "2425998") once it finishes, or wait for it to finish and call startAccessibilityScan with scanId "4304819" again for another run.', + ); + }); + + it("finds the run still going from the scan's runs when the scan is not on the list's first page", async () => { + m.post.mockRejectedValue(refused(422, "A New Scan is already in progress")); + m.get.mockImplementation(async ({ url }: { url: string }) => { + if (url === SCANS) return ok({ scans: [] }); + if (url.endsWith("/scan_runs")) + return ok({ + scanRuns: [ + { id: 10, scanDate: "2026-11-04T10:00:00.000Z" }, + { id: 11, scanDate: "2026-11-04T12:00:00.000Z" }, + ], + }); + if (url.endsWith("/scan_runs/11/status")) return ok({ status: "running" }); + return { ok: false, status: 404, data: {} }; + }); + + const error = await (await scanner()).rerunScan("7").catch((e) => e); + + expect(error.message).toContain("(run 11, running)"); + }); + + it("says a run is still going without naming one it cannot tell", async () => { + m.post.mockRejectedValue(refused(422, "A New Scan is already in progress")); + m.get.mockImplementation(async ({ url }: { url: string }) => { + if (url === SCANS) return ok({ scans: [{ id: 7, lastScanDetails: { scanRunId: 10, scanStatus: "completed" } }] }); + if (url.endsWith("/scan_runs")) return ok({ scanRuns: [{ id: 10, scanDate: "2026-11-04T10:00:00.000Z" }] }); + return ok({ status: "completed" }); + }); + + const error = await (await scanner()).rerunScan("7").catch((e) => e); + + expect(error.message).toContain("A run of scan 7 is still going, so a new run was not started; its run id could not be read"); + expect(error.message).not.toContain("run 10"); + }); + + it("says the scan's login config was deleted, and to start a new scan", async () => { + m.post.mockRejectedValue(refused(403, "Scan cannot be created because auth config has been removed")); + + const error = await (await scanner()).rerunScan("7").catch((e) => e); + + expect(error.message).toBe( + "Scan 7 can't be re-run: the login config (auth config) it was saved with has been deleted. Start a new scan instead: call startAccessibilityScan without scanId, with an authConfigId that exists if the page needs a login.", + ); + }); + + it("says no scan with that id is visible for a 404", async () => { + m.post.mockRejectedValue(refused(404, "Scan Config Not Found")); + + const error = await (await scanner()).rerunScan("7").catch((e) => e); + + expect(error.message).toContain("No scan with id 7 is visible to this account"); + }); + + it("passes the plan's message through", async () => { + m.post.mockRejectedValue(refused(402, "You don't have access to use local testing")); + + const error = await (await scanner()).rerunScan("7").catch((e) => e); + + expect(error.message).toBe("Scan 7 was not re-run: You don't have access to use local testing."); + expect(error.message).not.toMatch(/GitHub/); + }); +}); + +describe("AccessibilityScanner.startScan at the plan's scan limit", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it("says the plan allows 50 saved scans, rather than a failure to report", async () => { + const { ScanRefusedError, scanLimitMessage } = await scannerModule(); + m.post.mockRejectedValue(refused(402, "You don't have access to make over 50 scans")); + + const error = await (await scanner()).startScan("zz", ["https://example.com/"]).catch((e) => e); + + expect(error).toBeInstanceOf(ScanRefusedError); + expect(error.message).toBe( + "Your plan allows 50 saved scans and this account has reached that limit. Delete scans you no longer need in the dashboard, or upgrade the plan to create more.", + ); + expect(scanLimitMessage("50")).toBe(error.message); + expect(m.post).toHaveBeenCalledTimes(1); + }); + + it("gives the limit the API names, not a fixed 50", async () => { + const { scanLimitMessage } = await scannerModule(); + m.post.mockRejectedValue(refused(402, "You don't have access to make over 200 scans")); + + const error = await (await scanner()).startScan("zz", ["https://example.com/"]).catch((e) => e); + + expect(error.message).toBe( + "Your plan allows 200 saved scans and this account has reached that limit. Delete scans you no longer need in the dashboard, or upgrade the plan to create more.", + ); + expect(scanLimitMessage()).toBe( + "This account has reached your plan's limit of saved scans. Delete scans you no longer need in the dashboard, or upgrade the plan to create more.", + ); + }); + + it("keeps any other plan refusal as it was", async () => { + m.post.mockRejectedValue(refused(402, "You don't have access to use local testing")); + + await expect( + (await scanner()).startScan("zz", ["https://example.com/"]), + ).rejects.toThrow("Failed to start scan: You don't have access to use local testing"); + }); +}); + +describe("startAccessibilityScan with scanId", () => { + const SAVED = { + name: "zz-rescan-a", + urlList: ["https://example.com/"], + settings: { + wcagVersion: "wcag21aa", + needsReview: true, + bestPractices: false, + advancedRules: true, + experimentalRules: true, + }, + authEnabled: false, + localTestingEnabled: false, + }; + + beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + const actual = await scannerModule(); + vi.doMock("../../src/tools/accessiblity-utils/scanner.js", () => ({ + ...actual, + AccessibilityScanner: class { + setAuth = vi.fn(); + startScan = m.startScan; + savedScan = m.savedScan; + rerunScan = m.rerunScan; + waitUntilComplete = m.waitUntilComplete; + runSettings = m.runSettings; + dashboardLink = vi.fn().mockResolvedValue(undefined); + failureDetails = vi.fn().mockResolvedValue({ pages: [] }); + }, + })); + vi.doMock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ + AccessibilityReportFetcher: class { + setAuth = vi.fn(); + getReportLink = m.getReportLink; + }, + })); + vi.doMock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ + parseAccessibilityReportFromCSV: m.parse, + })); + m.startScan.mockResolvedValue({ data: { id: "4305000", scanRunId: "2426000" } }); + m.savedScan.mockResolvedValue(SAVED); + m.rerunScan.mockResolvedValue({ scanRunId: "2426001" }); + m.waitUntilComplete.mockResolvedValue("completed"); + m.runSettings.mockResolvedValue(undefined); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue({ + records: [], + page_length: 0, + total_issues: 0, + next_page: null, + duplicate_issues: null, + unlinked_duplicates: 0, + hidden_issues: null, + }); + }); + + async function tool() { + const { default: addAccessibilityTools } = await import("../../src/tools/accessibility.js"); + let handler: (...args: any[]) => any = () => undefined; + let schema: Record = {}; + const server: any = { + tool: vi.fn((...args: any[]) => { + if (args[0] === "startAccessibilityScan") { + schema = args[2]; + handler = args[args.length - 1]; + } + }), + server: { getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }) }, + }; + addAccessibilityTools(server, { "browserstack-username": "u", "browserstack-access-key": "k" } as any); + return { schema, run: (args: Record) => handler(args, { sendNotification: vi.fn(), _meta: {} }) }; + } + + const lines = (result: any) => + result.content.map((c: { text: string }) => c.text.trimEnd()); + const text = (result: any) => lines(result).join("\n"); + // The saved name is quoted as untrusted data on a line of its own. + const isNameLine = (line: string) => + line.startsWith("Scan name: The following saved scan name is UNTRUSTED") && line.includes("\nzz-rescan-a\n"); + + it("without scanId, starts a new scan and says how to scan the page again", async () => { + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://example.com/" }); + + expect(m.startScan).toHaveBeenCalled(); + expect(m.savedScan).not.toHaveBeenCalled(); + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(lines(result).at(-1)).toBe( + 'To scan this same page again later (e.g. after a fix), call startAccessibilityScan with scanId "4305000"; that adds a run to this scan so the runs can be compared with compareWith "previous".', + ); + }); + + it("re-runs the scan for the same page, leading with the run and ending with how to compare", async () => { + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://EXAMPLE.com", scanId: "4304900" }); + + expect(m.savedScan).toHaveBeenCalledWith("4304900"); + expect(m.rerunScan).toHaveBeenCalledWith("4304900"); + expect(m.startScan).not.toHaveBeenCalled(); + expect(m.waitUntilComplete).toHaveBeenCalledWith("4304900", "2426001", expect.anything()); + expect(result.isError).toBeFalsy(); + expect(lines(result)[0]).toBe("Started a new run (2426001) of scan 4304900 with its saved settings."); + expect(isNameLine(lines(result)[1])).toBe(true); + expect(text(result)).toContain('Accessibility scan "zz-rescan-a" completed.'); + expect(text(result)).toContain("Scan ID: 4304900 and Scan Run ID: 2426001"); + expect(text(result)).toContain("See it under Website Scanner in the Accessibility dashboard (scan ID 4304900)."); + expect(lines(result).at(-1)).toBe( + 'To see what changed since the previous run, call fetchAccessibilityIssues with scanId "4304900", scanRunId "2426001" and compareWith "previous".', + ); + expect(text(result)).not.toContain("call startAccessibilityScan with scanId"); + }); + + it("keeps the scan's name, and says so in one line when another was passed", async () => { + const { run } = await tool(); + + const result = await run({ name: "Home page after fix", pageURL: "https://example.com/", scanId: "4304900" }); + + expect(lines(result)[2]).toBe('The scan keeps its saved name; the name "Home page after fix" was not used.'); + expect(text(result)).toContain('Accessibility scan "zz-rescan-a" completed.'); + }); + + it("re-runs with settings passed that match the saved ones", async () => { + const { run } = await tool(); + + const result = await run({ + name: "zz-rescan-a", + pageURL: "https://example.com/", + scanId: "4304900", + wcagVersion: "wcag21aa", + advancedRules: true, + }); + + expect(m.rerunScan).toHaveBeenCalled(); + expect(text(result)).not.toContain("was not re-run"); + }); + + it("starts no run when a setting passed differs from the saved one, and says both ways forward", async () => { + const { run } = await tool(); + + const result = await run({ + name: "zz-rescan-a", + pageURL: "https://example.com/", + scanId: "4304900", + wcagVersion: "wcag22aa", + bestPractices: true, + needsReview: true, + }); + + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(m.startScan).not.toHaveBeenCalled(); + // Plain, so the server adds no "report it with submitFeedback" to it. + expect(result.isError).toBeFalsy(); + const [answer] = lines(result); + expect(lines(result)).toHaveLength(1); + expect(answer.startsWith( + [ + "Scan 4304900 was not re-run: a re-run uses the scan's saved settings, and these differ from what was passed:", + "- WCAG version: saved WCAG 2.1 AA, passed WCAG 2.2 AA", + "- best practices: saved off, passed on", + 'To re-run it with its saved settings, call startAccessibilityScan again with scanId "4304900" and without those settings. To scan with the settings passed, call it without scanId: that starts a new scan, whose runs can\'t be compared with this scan\'s.', + "", + ].join("\n"), + )).toBe(true); + // The quoted name comes after the list, so it is not read as one of the differences. + expect(isNameLine(answer.slice(answer.indexOf("Scan name: ")))).toBe(true); + }); + + it("refuses a login or tunnel the scan is not saved with", async () => { + const { run } = await tool(); + + const result = await run({ + name: "zz-rescan-a", + pageURL: "https://example.com/", + scanId: "4304900", + authConfigId: 12, + localIdentifier: "my-tunnel", + }); + + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(text(result)).toContain("- login: the scan has no auth config saved, passed authConfigId 12"); + expect(text(result)).toContain( + '- BrowserStack Local: the scan is saved without a tunnel, passed localIdentifier "my-tunnel"', + ); + }); + + it("re-runs a scan saved with a login, saying the login passed was not used", async () => { + m.savedScan.mockResolvedValue({ ...SAVED, authEnabled: true }); + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://example.com/", scanId: "4304900", authConfigId: 12 }); + + expect(m.rerunScan).toHaveBeenCalled(); + expect(lines(result)).toContain( + "authConfigId 12 was not used: a re-run logs in with the scan's saved login, which the API does not name. To scan with login 12, call startAccessibilityScan without scanId (that starts a new scan).", + ); + }); + + it("says a localIdentifier passed was not used when the overview does not say how the scan reaches its page", async () => { + m.savedScan.mockResolvedValue({ ...SAVED, localTestingEnabled: undefined }); + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://example.com/", scanId: "4304900", localIdentifier: "zz-tunnel" }); + + expect(m.rerunScan).toHaveBeenCalled(); + expect(lines(result)).toContain( + 'localIdentifier "zz-tunnel" was not used: a re-run uses the scan\'s saved BrowserStack Local settings, which the API does not give back. To scan through that tunnel, call startAccessibilityScan without scanId (that starts a new scan).', + ); + }); + + it("re-runs with a rule setting the overview does not give, saying it could not be checked", async () => { + m.savedScan.mockResolvedValue({ ...SAVED, settings: { advancedRules: true } }); + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://example.com/", scanId: "4304900", wcagVersion: "wcag22aa" }); + + expect(m.rerunScan).toHaveBeenCalled(); + expect(lines(result)).toContain( + "wcagVersion could not be checked against the scan's saved settings, which the API does not give back; the run uses the saved ones.", + ); + }); + + it("says a local scan's re-run needs the tunnel it was saved with", async () => { + m.savedScan.mockResolvedValue({ + ...SAVED, + urlList: ["http://bs-local.com:8799/"], + localTestingEnabled: true, + }); + const { run } = await tool(); + + const result = await run({ + name: "zz-rescan-a", + pageURL: "http://localhost:8799", + scanId: "4304900", + localIdentifier: "zz-tunnel", + }); + + expect(m.rerunScan).toHaveBeenCalled(); + expect(lines(result)).toContain( + 'This scan reaches its page through BrowserStack Local, and the run uses the scan\'s saved Local settings: a tunnel must be running with the same local identifier the scan was created with (the API does not give the saved identifier back, so it could not be checked against "zz-tunnel"). If the run cannot reach the page, start that tunnel and re-run.', + ); + }); + + it("starts a new scan for a scan saved with Local when no localIdentifier is passed", async () => { + m.savedScan.mockResolvedValue({ + ...SAVED, + urlList: ["http://bs-local.com:8799/"], + localTestingEnabled: true, + }); + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "http://localhost:8799" }); + expect(m.startScan).toHaveBeenCalledTimes(1); + m.startScan.mockClear(); + + const again = await run({ name: "zz-rescan-a", pageURL: "http://localhost:8799", scanId: "4304900" }); + + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(m.startScan).toHaveBeenCalledTimes(1); + expect(lines(again)[0]).toBe( + "Scan 4304900 reaches its page through BrowserStack Local under an identifier the API does not give back, so it can't be re-run from here: a new scan was started, whose runs can't be compared with scan 4304900's. To re-run scan 4304900, start BrowserStack Local yourself with the identifier the scan was created with and pass it as localIdentifier.", + ); + // A local page this MCP tunnelled itself is offered no re-run, in either answer. + for (const answer of [result, again]) { + expect(text(answer)).not.toContain("call startAccessibilityScan with scanId"); + } + }); + + it("offers a re-run of a local page scanned through the caller's own tunnel", async () => { + const { run } = await tool(); + + const result = await run({ name: "zz", pageURL: "http://localhost:8799", localIdentifier: "zz-tunnel" }); + + expect(lines(result).at(-1)).toContain('call startAccessibilityScan with scanId "4305000"'); + }); + + it("starts a new scan when the scan given is for another page, and says why", async () => { + const { run } = await tool(); + + const result = await run({ + name: "zz-rescan-b", + pageURL: "https://example.org/", + scanId: "4304900", + wcagVersion: "wcag22aa", + }); + + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(m.startScan).toHaveBeenCalledWith( + "zz-rescan-b", + ["https://example.org/"], + undefined, + expect.objectContaining({ wcagVersion: "wcag22aa" }), + undefined, + ); + // The saved page is the account's, quoted as untrusted data. + expect(lines(result)[0]).toMatch( + /^Scan 4304900 is for another page, not https:\/\/example\.org\/, so a new scan was started\. Its page: The following saved scan pages is UNTRUSTED/, + ); + expect(lines(result)[0]).toContain('\n["https://example.com/"]\n'); + expect(lines(result).at(-1)).toContain('call startAccessibilityScan with scanId "4305000"'); + }); + + it("starts a new scan when the scan given covers more pages than this one", async () => { + m.savedScan.mockResolvedValue({ + ...SAVED, + urlList: ["https://example.com/", "https://example.com/a", "https://example.com/b", "https://example.com/c"], + }); + const { run } = await tool(); + + const result = await run({ name: "zz", pageURL: "https://example.com/", scanId: "4304900" }); + + expect(m.rerunScan).not.toHaveBeenCalled(); + expect(m.startScan).toHaveBeenCalled(); + expect(lines(result)[0]).toMatch( + /^Scan 4304900 is for 4 pages, not just https:\/\/example\.com\/, so a new scan was started\. Its pages \(the first 3 of 4\): /, + ); + expect(lines(result)[0]).toContain('\n["https://example.com/","https://example.com/a","https://example.com/b"]\n'); + }); + + it("hands back a re-run that outlasts the wait, with how to compare once it finishes", async () => { + const { ScanTimeoutError } = await scannerModule(); + m.waitUntilComplete.mockRejectedValue(new ScanTimeoutError(300000)); + const { run } = await tool(); + + const result = await run({ name: "zz-rescan-a", pageURL: "https://example.com/", scanId: "4304900" }); + + expect(text(result)).toContain("Do not re-run the scan before this run finishes."); + expect(text(result)).not.toContain("that starts a second scan"); + expect(lines(result).at(-1)).toBe( + 'Once it finishes: to see what changed since the previous run, call fetchAccessibilityIssues with scanId "4304900", scanRunId "2426001" and compareWith "previous".', + ); + }); + + it("says the re-run endpoint's refusals as plain results, without the GitHub advice", async () => { + const { ScanRefusedError } = await scannerModule(); + m.rerunScan.mockRejectedValue(new ScanRefusedError("A run of scan 4304900 is still going (run 2426001, processing).")); + const { run } = await tool(); + + const going = await run({ name: "zz-rescan-a", pageURL: "https://example.com/", scanId: "4304900" }); + expect(going.isError).toBeFalsy(); + expect(text(going)).toBe("A run of scan 4304900 is still going (run 2426001, processing)."); + + m.savedScan.mockRejectedValue( + new ScanRefusedError("No scan with id 999999999 is visible to this account, so nothing was started. Leave out scanId to start a new scan."), + ); + const unknown = await run({ name: "zz", pageURL: "https://example.com/", scanId: "999999999" }); + expect(unknown.isError).toBeFalsy(); + expect(text(unknown)).toBe( + "No scan with id 999999999 is visible to this account, so nothing was started. Leave out scanId to start a new scan.", + ); + expect(text(unknown)).not.toMatch(/GitHub|Failed to start/); + }); + + it("gives the scan limit as the whole answer", async () => { + const { ScanRefusedError, scanLimitMessage } = await scannerModule(); + m.startScan.mockRejectedValue(new ScanRefusedError(scanLimitMessage("50"))); + const { run } = await tool(); + + const result = await run({ name: "zz", pageURL: "https://example.com/" }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + expect(result.content[0].text).toBe( + "Your plan allows 50 saved scans and this account has reached that limit. Delete scans you no longer need in the dashboard, or upgrade the plan to create more.", + ); + }); + + it("gives scanId and the settings no default, so a setting left out is not taken as passed", async () => { + const { schema } = await tool(); + + for (const field of [ + "scanId", + "wcagVersion", + "advancedRules", + "experimentalRules", + "bestPractices", + "needsReview", + "authConfigId", + "localIdentifier", + ]) { + expect(schema[field].parse(undefined)).toBeUndefined(); + } + expect(schema.scanId.description).toBe( + "Only to re-run an earlier scan of this same page (e.g. after a fix), with its saved settings, so runs can be compared.", + ); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index cfae3d74..416e9b48 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -17,6 +17,7 @@ const m = vi.hoisted(() => ({ parse: vi.fn(), dashboardLink: vi.fn(), failureDetails: vi.fn(async () => ({ pages: [] })), + unscannedPages: vi.fn(async () => []), })); vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ @@ -27,6 +28,7 @@ vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ runSettings = vi.fn().mockResolvedValue(undefined); dashboardLink = m.dashboardLink; failureDetails = m.failureDetails; + unscannedPages = m.unscannedPages; }, ScanTimeoutError: class extends Error { constructor(readonly waitedMs: number) { @@ -74,12 +76,10 @@ vi.mock("../../src/index.js", () => ({ PackageJsonVersion: "0.0.0", })); -import addAccessibilityTools, { - REPORT_STATUS_POLL_MS, - REPORT_STATUS_WAIT_MS, -} from "../../src/tools/accessibility.js"; +import addAccessibilityTools from "../../src/tools/accessibility.js"; import { ExportPendingError, + ScanNotInListError, ScanRunNotFoundError, } from "../../src/tools/accessiblity-utils/report-fetcher.js"; @@ -106,6 +106,14 @@ function register() { const text = (result: any) => result.content.map((c: { text: string }) => c.text).join("\n"); +/** The content of the first untrusted-data block with this label. */ +function quoted(out: string, label: string): string { + const block = out.match( + new RegExp(`«UNTRUSTED ${label} ([0-9a-f]+)»\\n([\\s\\S]*?)\\n«END UNTRUSTED \\1»`), + ); + return block?.[2] ?? ""; +} + const CONTEXT = { sendNotification: vi.fn(), _meta: {} }; /** A grouped report: 79 main issues, 43 duplicates (all matched), nothing hidden. */ @@ -392,55 +400,31 @@ describe("fetchAccessibilityIssues and a report still processing", () => { expect(body).not.toContain("still processing"); }); - it("waits while the report is finalizing, then reads the complete issues", async () => { - m.reportStatus - .mockResolvedValueOnce("finalizing") - .mockResolvedValueOnce("finalizing") - .mockResolvedValueOnce("completed"); - const pending = handlers.fetchAccessibilityIssues( + it("does not wait while the report is finalizing: says to call again, without reading any issues", async () => { + m.reportStatus.mockResolvedValue("finalizing"); + const result = await handlers.fetchAccessibilityIssues( { buildId: BUILD_ID }, context, ); - await vi.advanceTimersByTimeAsync(REPORT_STATUS_POLL_MS * 2); - const result = await pending; - expect(m.reportStatus).toHaveBeenCalledTimes(3); - expect(m.getReportLink).toHaveBeenCalledTimes(1); - expect(sendNotification).toHaveBeenCalledWith( - expect.objectContaining({ - method: "notifications/progress", - params: expect.objectContaining({ - message: expect.stringContaining("still finalizing"), - }), - }), - ); + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(m.getReportLink).not.toHaveBeenCalled(); + expect(m.parse).not.toHaveBeenCalled(); + expect(sendNotification).not.toHaveBeenCalled(); + expect(result.isError).toBeFalsy(); const body = text(result); expect(body).toContain( - "The report was still processing (status: finalizing); waited 20s for it to complete", - ); - expect(body).toContain("✅ All issues retrieved."); - }); - - it("says the issues are only those found so far when processing outlasts the wait", async () => { - m.reportStatus.mockResolvedValue("finalizing"); - const pending = handlers.fetchAccessibilityIssues( - { buildId: BUILD_ID }, - context, + `The report for Automated Tests build ${BUILD_ID} is still finalizing (status: finalizing)`, ); - await vi.advanceTimersByTimeAsync( - REPORT_STATUS_WAIT_MS + REPORT_STATUS_POLL_MS, - ); - const result = await pending; - - const body = text(result); + expect(body).toContain("No issues have been read yet"); expect(body).toContain( - "This report is still processing (status: finalizing", + `Call fetchAccessibilityIssues again with buildId: "${BUILD_ID}" in a minute or two.`, ); - expect(body).toContain("only those found so far"); - expect(body).not.toContain("✅ All issues retrieved."); + expect(body).not.toContain("Retrieved"); + expect(body).not.toContain("All issues retrieved"); }); - it("does not wait again for a later page, but still says the report is processing", async () => { + it("reads a later page as it is, saying the report is still processing", async () => { m.reportStatus.mockResolvedValue("finalizing"); const result = await handlers.fetchAccessibilityIssues( { buildId: BUILD_ID, cursor: 4400 }, @@ -452,8 +436,7 @@ describe("fetchAccessibilityIssues and a report still processing", () => { }); it("does not wait while the tests are still running, and says the issues are those found so far", async () => { - // A long suite can run for an hour; blocking the first read for five minutes and then - // returning partial results anyway serves nobody. Only post-run processing is waited on. + // A long suite can run for an hour, so a running report is read as it is, with a note. m.reportStatus.mockResolvedValue("running"); const result = await handlers.fetchAccessibilityIssues( @@ -608,22 +591,24 @@ describe("fetchAccessibilityIssues and App Accessibility builds", () => { expect(body.indexOf(LINK)).toBeGreaterThan(block); }); - it("waits while an app build's rules are still processing", async () => { - m.reportStatus - .mockResolvedValueOnce("rule_processing") - .mockResolvedValueOnce("failed"); + it("does not wait while an app build's rules are still processing: says to call again", async () => { + m.reportStatus.mockResolvedValue("rule_processing"); - const pending = handlers.fetchAccessibilityIssues( + const result = await handlers.fetchAccessibilityIssues( { buildId: BUILD_ID }, context, ); - await vi.advanceTimersByTimeAsync(REPORT_STATUS_POLL_MS); - const result = await pending; - expect(m.reportStatus).toHaveBeenCalledTimes(2); - expect(m.appBuildReport).toHaveBeenCalledTimes(1); - expect(text(result)).toContain( - "The report was still processing (status: rule_processing); waited", + expect(m.reportStatus).toHaveBeenCalledTimes(1); + expect(m.appBuildReport).not.toHaveBeenCalled(); + expect(result.isError).toBeFalsy(); + const body = text(result); + expect(body).toContain( + "is still running its accessibility rules (status: rule_processing)", + ); + expect(body).toContain("No issues have been read yet"); + expect(body).toContain( + `Call fetchAccessibilityIssues again with buildId: "${BUILD_ID}" in a minute or two.`, ); }); @@ -946,14 +931,17 @@ describe("fetchAccessibilityIssues and a report link read by its build id", () = expect(body).toContain(`Use fetchAccessibilityIssues with buildId: "${TH}" and cursor: 140`); }); - it("points at the report link when the API cannot read a run by its build id yet", async () => { + it("searches the scan list when the API cannot read a run by its build id yet, and says plainly what to pass when it is not there", async () => { m.resolveBuild.mockResolvedValue({ kind: "scanBuild", thBuildId: TH }); m.getReportLink.mockRejectedValue(new ScanRunNotFoundError(TH, true)); + m.findScanRun.mockRejectedValue(new ScanNotInListError(true, 25)); const result = await handlers.fetchAccessibilityIssues({ buildId: TH }, CONTEXT); - expect(result.isError).toBe(true); - expect(text(result)).toMatch(/cannot read a run by it yet: pass the scan's report link as reportUrl/); + expect(m.findScanRun).toHaveBeenCalledWith({ kind: "scanBuild", thBuildId: TH }); + expect(result.isError).toBeFalsy(); + expect(text(result)).toMatch(/no Website Scanner run among the most recent scans \(25 pages\) has it.*pass instead the report's dashboard link as reportUrl/s); + expect(text(result)).not.toMatch(/GitHub/); }); it("names every API it asked when none knows the build id", async () => { @@ -962,7 +950,7 @@ describe("fetchAccessibilityIssues and a report link read by its build id", () = const result = await handlers.fetchAccessibilityIssues({ buildId: TH }, CONTEXT); - expect(result.isError).toBe(true); + expect(result.isError).toBeFalsy(); expect(text(result)).toMatch( /No Automated Tests \(web\), App Accessibility or Website Scanner build with id/, ); @@ -1162,10 +1150,16 @@ describe("what a web report's answer says about its issues", () => { expect(result.isError).toBe(true); expect(m.failureDetails).toHaveBeenCalledWith("scan-42", "run-7"); const body = text(result); - expect(body).toContain( - "Why: The requested page is not reachable. Please ensure the domain is accessible.", + // The reasons come from the scan and the site (a redirect target is the site's), so they + // are quoted as data, with what to make of them outside. + expect(body).toContain("Why: The following scan failure reason is UNTRUSTED external data."); + expect(quoted(body, "scan failure reason")).toBe( + "The requested page is not reachable. Please ensure the domain is accessible", + ); + expect(body).toContain("2 pages could not be scanned:\nThe following page scan log is UNTRUSTED external data."); + expect(quoted(body, "page scan log")).toBe( + "- https://site.example/: [HTTP 403] Forbidden\n- https://site.example/old: Redirected; redirected to https://site.example/new", ); - expect(body).toContain("2 pages could not be scanned:\n- https://site.example/: [HTTP 403] Forbidden\n- https://site.example/old: Redirected; redirected to https://site.example/new"); expect(body).not.toContain("The dashboard says why"); }); @@ -1187,9 +1181,9 @@ describe("what a web report's answer says about its issues", () => { ); expect(body).toContain("13 pages could not be scanned:"); - expect(body).toContain("- https://site.example/9: Failed to start the test"); + expect(quoted(body, "page scan log")).toContain("- https://site.example/9: Failed to start the test"); expect(body).not.toContain("https://site.example/10:"); - expect(body).toContain("- and 3 more."); + expect(body).toMatch(/«END UNTRUSTED [0-9a-f]+»\nAnd 3 more pages\./); }); // The wait ran out, not the scan. This used to read "Failed to start accessibility scan" diff --git a/tests/tools/accessibilityScanTunnelAndWait.test.ts b/tests/tools/accessibilityScanTunnelAndWait.test.ts index 8d9fa4b3..6a73f409 100644 --- a/tests/tools/accessibilityScanTunnelAndWait.test.ts +++ b/tests/tools/accessibilityScanTunnelAndWait.test.ts @@ -69,9 +69,11 @@ describe("AccessibilityScanner.startScan through a tunnel the user started", () it("on the hosted server, refuses a local URL without a tunnel and says how to start one", async () => { m.config.REMOTE_MCP = true; + const { ScanRefusedError } = await scannerModule(); const start = (await scanner()).startScan("dev", ["http://localhost:3000/"]); + await expect(start).rejects.toBeInstanceOf(ScanRefusedError); await expect(start).rejects.toThrow(/cannot open a tunnel to a local URL on your machine/); await expect(start).rejects.toThrow(/--local-identifier \) and pass that name as localIdentifier/); expect(m.post).not.toHaveBeenCalled(); diff --git a/tests/tools/accessibilityWebBuildCompare.test.ts b/tests/tools/accessibilityWebBuildCompare.test.ts new file mode 100644 index 00000000..b35da071 --- /dev/null +++ b/tests/tools/accessibilityWebBuildCompare.test.ts @@ -0,0 +1,499 @@ +import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; + +// compareWith on an Automated Tests (web) build: two builds of one name and project are +// compared by the Automated Tests compare API, any other pair (or one the API refuses or does +// not answer) by matching the builds' issue exports by rule, element and page. The older +// build is "before" whichever order they come in, and a build with no results is not compared. +// The HTTP layer is mocked; the report fetcher and the CSV parser run as they do live. + +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: vi.fn() }, +})); +vi.mock("../../src/lib/get-auth.js", () => ({ + getBrowserStackAuth: vi.fn().mockReturnValue("fake-user:fake-key"), +})); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); +vi.mock("../../src/index.js", () => ({ trackMCP: vi.fn(), PackageJsonVersion: "0.0.0" })); + +import { apiClient } from "../../src/lib/apiClient.js"; +import addAccessibilityTools from "../../src/tools/accessibility.js"; +import { reportExports } from "../../src/tools/accessiblity-utils/report-fetcher.js"; + +const get = apiClient.get as Mock; +const API = "https://api-accessibility.browserstack.com/api"; +const COMPARE = `${API}/automated-tests/v1/builds/compare/issues`; + +type Build = { id: string; name: string; number: number; time: string; status: string }; + +const OLD: Build = { id: "c1oldbuildaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", name: "checkout", number: 1, time: "2026-11-03T11:28:55.781Z", status: "completed" }; +const NEW: Build = { id: "c2newbuildbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", name: "checkout", number: 2, time: "2026-11-03T11:32:05.806Z", status: "completed" }; +const OTHER: Build = { id: "s1smokebuildcccccccccccccccccccccccccccc", name: "smoke", number: 1, time: "2026-11-03T13:35:49.619Z", status: "completed" }; +const EMPTY: Build = { id: "c3emptybuilddddddddddddddddddddddddddddd", name: "checkout", number: 3, time: "2026-11-04T09:00:00.000Z", status: "failed" }; +const LATEST: Build = { id: "c4latestbuildeeeeeeeeeeeeeeeeeeeeeeeeeee", name: "checkout", number: 4, time: "2026-11-04T11:52:19.537Z", status: "completed" }; +const BUILDS = [LATEST, EMPTY, OTHER, NEW, OLD]; // the project's build list: newest first +const PROJECT = "Shop"; +const APP_BUILD = "a9appbuildgggggggggggggggggggggggggggggg"; +const APP_API = "https://api-app-accessibility.browserstack.com/automate/api/v1"; + +const PAGE_A = "http://localhost:8765/home.html"; +const PAGE_B = "http://localhost:8765/usage.html"; +const PRIVOXY = 'Local'; + +// ---- the API's answers -------------------------------------------------------------- + +type Reply = { ok: boolean; status: number; data: unknown; statusText?: string }; +const ok = (data: unknown): Reply => ({ ok: true, status: 200, data: { success: true, data, error: [] } }); + +// The API's summary leaves the project name out (null), as it does live. +const summaryOf = (b: Build, autoResolved?: boolean) => ({ + id: b.id, + createdOn: b.time, + projectName: null, + buildName: b.name, + buildNumber: b.number, + isGrouped: true, + ...(autoResolved === undefined ? {} : { autoResolved }), +}); + +function apiRow(status: string, rule: string, selector: string, page = PAGE_A, html = "") { + return { issueId: `${status}-${rule}-${selector}`, status, impact: "serious", url: page, componentId: selector, target: [selector], html, ruleData: { id: rule }, behaviour: "visible" }; +} + +function comparison( + rows: ReturnType[], + { base = OLD, after = NEW, counts, autoResolved = false }: { base?: Build; after?: Build; counts?: [number, number, number]; autoResolved?: boolean } = {}, +): Reply { + const count = (s: string) => rows.filter((r) => r.status === s).length; + const [n, retained, resolved] = counts ?? [count("new"), count("retained"), count("resolved")]; + return ok({ + summary: { + newIssues: n, + retainedIssues: retained, + resolvedIssues: resolved, + isMixedComparison: false, + isGroupedComparison: true, + baseReport: summaryOf(base, autoResolved), + newReport: summaryOf(after), + }, + issues: rows, + pagination: { hasNext: false, nextPage: null }, + }); +} + +// ---- the builds' CSV exports --------------------------------------------------------- + +const COLUMNS = ["Issue type", "Severity", "Component", "Affected page", "HTML snippet", "CSS selector", "Issue link", "Duplicate Issue of", "Issue status"]; +const csvRow = (type: string, selector: string, page = PAGE_A, snippet = "") => ({ + "Issue type": type, + Severity: "serious", + Component: selector, + "Affected page": page, + "HTML snippet": snippet, + "CSS selector": selector, + "Issue link": `https://dash.example/issue?activeIssueId=${type}-${selector}-${page}`, + "Duplicate Issue of": "NA", + "Issue status": "visible", +}); +const toCsv = (rows: Record[]) => + [COLUMNS, ...rows.map((r) => COLUMNS.map((c) => r[c] ?? ""))] + .map((cells) => cells.map((v) => `"${v.replace(/"/g, '""')}"`).join(",")) + .join("\n"); + +let exportsOf: Record[]>; +/** Builds whose overview gives no start time, and build-list statuses that differ from the fixtures'. */ +let undated: Set; +let statusOf: Record; +let compareReply: (params: URLSearchParams) => Reply | Promise; + +function route({ url }: { url: string }): Reply | Promise { + const u = new URL(url); + if (url.startsWith(COMPARE)) return compareReply(u.searchParams); + const overview = u.pathname.match(/\/automated-tests\/v1\/builds\/([a-z0-9]{40})$/)?.[1]; + const build = BUILDS.find((b) => b.id === overview); + if (build) { + return ok({ id: build.id, name: build.name, projectName: PROJECT, buildNumber: build.number, time: undated.has(build.id) ? null : build.time }); + } + if (u.pathname.endsWith("/automated-tests/v1/builds") && u.searchParams.get("project_name") === PROJECT) { + return ok({ + builds: BUILDS.map((b) => ({ id: b.id, name: b.name, buildNumber: b.number, time: b.time, status: statusOf[b.id] ?? b.status })), + pagination: { has_next: false, next_page: null }, + }); + } + if (u.pathname.endsWith("/automated-tests/v1/builds/issues")) { + const start = u.searchParams.get("buildUuid"); + return { + ok: true, + status: 200, + data: start + ? { success: true, data: { task_id: `t-${start}` } } + : { success: true, data: { reportLink: `https://csv.example/${u.searchParams.get("taskId")!.slice(2)}` } }, + }; + } + if (url === `${APP_API}/build/${APP_BUILD}`) { + return { ok: true, status: 200, data: { meta_data: { build_id: 7, th_build_id: APP_BUILD, name: "app", status: "done" } } }; + } + if (u.host === "csv.example") { + return { ok: true, status: 200, statusText: "OK", data: toCsv(exportsOf[u.pathname.slice(1)] ?? []) }; + } + return { ok: false, status: 404, data: {} }; +} + +const compareCalls = () => + get.mock.calls.map(([o]) => o.url as string).filter((u) => u.startsWith(COMPARE)).map((u) => new URL(u).searchParams); +const exportCalls = () => get.mock.calls.map(([o]) => o.url as string).filter((u) => u.includes("buildUuid=")); + +function fetchIssues() { + let handler: (...args: any[]) => any = () => undefined; + const server: any = { + tool: vi.fn((...args: any[]) => { + if (args[0] === "fetchAccessibilityIssues") handler = args[args.length - 1]; + }), + server: { getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }) }, + }; + addAccessibilityTools(server, { "browserstack-username": "fake-user", "browserstack-access-key": "fake-key" } as any); + return (args: Record) => handler(args, { sendNotification: vi.fn(), _meta: {} }); +} + +const text = (result: any) => result.content.map((c: { text: string }) => c.text).join("\n"); +/** The JSON array a list's untrusted block holds. */ +function listed(out: string, label: string): any[] { + const at = out.indexOf(label); + expect(at).toBeGreaterThan(-1); + const open = out.indexOf("»\n", at) + 2; + const close = out.indexOf("\n«END UNTRUSTED", open); + return JSON.parse(out.slice(open, close)); +} + +beforeEach(() => { + get.mockReset(); + get.mockImplementation(route); + reportExports.clear(); + undated = new Set(); + statusOf = {}; + exportsOf = { + // Before: a and b on page A. After (NEW): a and c. After (OTHER): a and d. + [OLD.id]: [csvRow("image-alt", "img"), csvRow("label", "input#q")], + [NEW.id]: [csvRow("image-alt", "img"), csvRow("region", "div.main")], + [OTHER.id]: [csvRow("image-alt", "img"), csvRow("link-name", "a.more")], + [LATEST.id]: [csvRow("image-alt", "img")], + }; + compareReply = () => comparison([apiRow("new", "region", "div.main"), apiRow("retained", "image-alt", "img"), apiRow("resolved", "label", "input#q")]); +}); + +describe("compareWith on an Automated Tests build", () => { + it("compares two builds of one name and project through the compare API, newer as newReport", async () => { + compareReply = () => + comparison( + [apiRow("new", "region", "div.main"), apiRow("retained", "image-alt", "img"), apiRow("resolved", "label", "input#q")], + { counts: [6, 1, 13] }, + ); + + const result = await fetchIssues()({ buildId: NEW.id, compareWith: OLD.id }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + const [call] = compareCalls(); + expect(call.get("newReport")).toBe(NEW.id); + expect(call.get("baseReport")).toBe(OLD.id); + expect(exportCalls()).toEqual([]); + expect(out).toContain( + "Same build name and project, so this is the Automated Tests compare API's comparison, as the dashboard shows it: issues are matched by the dashboard's own issue key (key below), not by rule, element and page alone.", + ); + expect(out).toContain("6 new, 13 fixed, 1 still open."); + expect(out).toContain(`before build #1 (${OLD.id}, started ${OLD.time}), after build #2 (${NEW.id}, started ${NEW.time})`); + expect(out).not.toContain("compareWith was the newer build"); + expect(listed(out, "New (only in the later build) (6)").map((r) => r.issue_type)).toEqual(["region"]); + expect(listed(out, "Fixed (only in the earlier build) (13)").map((r) => r.issue_type)).toEqual(["label"]); + expect(listed(out, "Still open (in both) (1)").map((r) => r.issue_type)).toEqual(["image-alt"]); + // The builds' names are the user's own, so they are quoted as data. + expect(out.indexOf('"name": "checkout"')).toBeGreaterThan(out.indexOf("«UNTRUSTED Automated Tests builds")); + expect(out).not.toMatch(/GitHub/); + }); + + it("keys each issue as the API matched it, so one it lists as both fixed and new reads as two", async () => { + // A fix added lang to : the API tells the element's issue before and after apart. + compareReply = () => + comparison([ + { ...apiRow("new", "css-orientation-lock", "html", PAGE_B, ''), signatureKey: "41ce86653b15" }, + { ...apiRow("resolved", "css-orientation-lock", "html", PAGE_B, ""), signatureKey: "d75d13a148fd" }, + ]); + + const out = text(await fetchIssues()({ buildId: NEW.id, compareWith: OLD.id })); + + expect(listed(out, "New (only in the later build) (1)")).toMatchObject([{ key: "41ce86653b15", selector: "html", affected_page: PAGE_B }]); + expect(listed(out, "Fixed (only in the earlier build) (1)")).toMatchObject([{ key: "d75d13a148fd", selector: "html", affected_page: PAGE_B }]); + }); + + it("takes the other build as an Automated Tests report link", async () => { + const link = `https://accessibility.browserstack.com/automated-tests/projects/Shop/builds/checkout/1?thBuildId=${OLD.id}`; + + await fetchIssues()({ buildId: NEW.id, compareWith: link }); + + expect(compareCalls()[0].get("baseReport")).toBe(OLD.id); + }); + + it("matches two builds of different names by rule, element and page, and says so", async () => { + const result = await fetchIssues()({ buildId: OTHER.id, compareWith: OLD.id }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + expect(compareCalls()).toEqual([]); + expect(out).toContain("Different build names, so the issue lists were matched by rule, element and page."); + expect(out).toContain("1 new, 1 fixed, 1 still open."); + expect(listed(out, "New (only in the later build) (1)").map((r) => r.issue_type)).toEqual(["link-name"]); + expect(listed(out, "Fixed (only in the earlier build) (1)").map((r) => r.issue_type)).toEqual(["label"]); + expect(listed(out, "Still open (in both) (1)").map((r) => r.issue_type)).toEqual(["image-alt"]); + expect(out).toContain("CSV of each build, valid for about 12 hours: later https://csv.example/"); + }); + + it.each<[string, () => Reply | Promise, string]>([ + [ + "refuses a pair over its issue-count threshold", + () => ({ ok: true, status: 200, data: { success: false, message: "Comparison of larger reports are currently not supported." } }), + "The compare API gave no comparison (Comparison of larger reports are currently not supported.), so the issue lists were matched by rule, element and page.", + ], + [ + "runs out of its 30 s", + () => ({ ok: false, status: 504, data: { success: false, data: null, error: ["Request timed out"] } }), + "The compare API gave no comparison (Request timed out), so the issue lists were matched by rule, element and page.", + ], + [ + "does not answer at all", + () => Promise.reject(Object.assign(new Error("timeout of 40000ms exceeded"), { code: "ECONNABORTED" })), + "The compare API gave no comparison (no answer within 40 s), so the issue lists were matched by rule, element and page.", + ], + ])("matches the issue lists itself when the compare API %s", async (_case, reply, method) => { + compareReply = reply; + + const result = await fetchIssues()({ buildId: NEW.id, compareWith: OLD.id }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + expect(out).toContain(method); + expect(out).toContain("1 new, 1 fixed, 1 still open."); + expect(listed(out, "New (only in the later build) (1)").map((r) => r.issue_type)).toEqual(["region"]); + expect(out).not.toMatch(/GitHub/); + }); + + it("uses the older build as 'before' when compareWith is the newer one, and says so", async () => { + const result = await fetchIssues()({ buildId: OLD.id, compareWith: NEW.id }); + const out = text(result); + + const [call] = compareCalls(); + expect(call.get("newReport")).toBe(NEW.id); + expect(call.get("baseReport")).toBe(OLD.id); + expect(out).toContain("compareWith was the newer build, so it is used as 'after'."); + expect(out).toContain(`before build #1 (${OLD.id}`); + }); + + it("swaps a different-name pair the same way before matching it", async () => { + const out = text(await fetchIssues()({ buildId: OLD.id, compareWith: OTHER.id })); + + expect(out).toContain("compareWith was the newer build, so it is used as 'after'."); + expect(listed(out, "New (only in the later build) (1)").map((r) => r.issue_type)).toEqual(["link-name"]); + expect(listed(out, "Fixed (only in the earlier build) (1)").map((r) => r.issue_type)).toEqual(["label"]); + }); + + it('compares with "previous" through the compare API, and names the build it used', async () => { + compareReply = () => comparison([apiRow("new", "region", "div.main")], { autoResolved: true }); + + const out = text(await fetchIssues()({ buildId: NEW.id, compareWith: "previous" })); + + const [call] = compareCalls(); + expect(call.get("newReport")).toBe(NEW.id); + expect(call.has("baseReport")).toBe(false); + expect(out).toContain( + `compareWith "previous": the compare API used the run before it with the same build name and project, build #1 (${OLD.id}, started ${OLD.time}).`, + ); + expect(out).toContain("1 new, 0 fixed, 0 still open."); + expect(out).toContain('"project": "Shop"'); + expect(out).not.toContain('"project": null'); + }); + + it('says when "previous" has nothing earlier, as an answer rather than a failure to report', async () => { + compareReply = () => ({ ok: false, status: 404, data: { success: false, message: "No previous completed run of this build to compare against." } }); + + const result = await fetchIssues()({ buildId: OLD.id, compareWith: "previous" }); + + expect(result.isError).toBe(true); + expect(text(result)).toContain("is the first completed run of its build name in its project"); + expect(text(result)).not.toMatch(/GitHub/); + }); + + it('finds the run before it itself when the compare API gives no comparison for "previous"', async () => { + compareReply = () => ({ ok: false, status: 504, data: { success: false, error: ["Request timed out"] } }); + + const out = text(await fetchIssues()({ buildId: NEW.id, compareWith: "previous" })); + + expect(out).toContain(`compareWith "previous": the run before it with the same build name and project, build #1 (${OLD.id}`); + expect(out).toContain("1 new, 1 fixed, 1 still open."); + }); + + it("does not compare a build with no accessibility results, and says so as guidance", async () => { + const result = await fetchIssues()({ buildId: EMPTY.id, compareWith: NEW.id }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + expect(out).toContain(`Build #3 (${EMPTY.id}, started ${EMPTY.time}) has no accessibility results: its status is failed`); + expect(compareCalls()).toEqual([]); + expect(exportCalls()).toEqual([]); + expect(out).not.toMatch(/GitHub/); + }); + + it("does not compare while either build is still processing, and says to ask again later", async () => { + statusOf = { [NEW.id]: "processing" }; + + for (const [buildId, compareWith] of [[NEW.id, OLD.id], [OLD.id, NEW.id]]) { + get.mockClear(); + const result = await fetchIssues()({ buildId, compareWith }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + expect(out).toContain(`⏳ Automated Tests build #2 (${NEW.id}, started ${NEW.time}) has not finished yet (status: processing), so nothing was compared`); + expect(compareCalls()).toEqual([]); + expect(exportCalls()).toEqual([]); + } + }); + + it("orders builds of one name and project by build number when a start time is missing", async () => { + undated = new Set([NEW.id]); + + const out = text(await fetchIssues()({ buildId: OLD.id, compareWith: NEW.id })); + + const [call] = compareCalls(); + expect(call.get("newReport")).toBe(NEW.id); + expect(call.get("baseReport")).toBe(OLD.id); + expect(out).toContain("compareWith was the newer build, so it is used as 'after'."); + expect(out).not.toContain("start times could not be read"); + }); + + it("keeps the order passed, and says so, when start times are missing and build numbers can't be compared", async () => { + undated = new Set([OTHER.id]); + + const out = text(await fetchIssues()({ buildId: OTHER.id, compareWith: OLD.id })); + + expect(out).toContain( + "The builds' start times could not be read, so compareWith was taken as the earlier build. If it is the newer one, swap them: new and fixed issues are then the other way around.", + ); + expect(out).not.toContain("compareWith was the newer build"); + // OLD, passed as compareWith, is "before": its label issue is the one fixed. + expect(listed(out, "Fixed (only in the earlier build) (1)").map((r) => r.issue_type)).toEqual(["label"]); + }); + + it('does not compare with a "previous" build that has no accessibility results', async () => { + compareReply = () => comparison([], { base: EMPTY, after: LATEST, autoResolved: true }); + + const result = await fetchIssues()({ buildId: LATEST.id, compareWith: "previous" }); + const out = text(result); + + expect(result.isError).toBeFalsy(); + expect(out).toContain(`The run before this build with the same build name and project ("previous") is build #3 (${EMPTY.id}`); + expect(out).toContain("has no accessibility results"); + expect(out).not.toContain("still open."); + }); + + it("lists at most 50 issues of each list, reading a list the first page cuts short on its own, and says how many more", async () => { + const news = Array.from({ length: 100 }, (_, i) => apiRow("new", "region", `div.n${i}`)); + const resolved = Array.from({ length: 50 }, (_, i) => apiRow("resolved", "label", `input.r${i}`)); + compareReply = (params) => + params.get("status") === "resolved" + ? comparison(resolved, { counts: [0, 0, 60] }) + : comparison(news, { counts: [120, 0, 60] }); + + const out = text(await fetchIssues()({ buildId: NEW.id, compareWith: OLD.id })); + + const calls = compareCalls(); + expect(calls).toHaveLength(2); + expect(calls[0].get("limit")).toBe("100"); + expect(calls[1].get("status")).toBe("resolved"); + expect(calls[1].get("limit")).toBe("50"); + expect(calls[1].get("baseReport")).toBe(OLD.id); + expect(out).toContain("120 new, 60 fixed, 0 still open."); + expect(listed(out, "New (only in the later build) (120)")).toHaveLength(50); + expect(out).toContain("(70 more not listed.)"); + expect(listed(out, "Fixed (only in the earlier build) (60)")).toHaveLength(50); + expect(out).toContain("(10 more not listed.)"); + }); + + it("caps the lists it matches itself the same way", async () => { + exportsOf[OTHER.id] = Array.from({ length: 53 }, (_, i) => csvRow("region", `div.n${i}`)); + + const out = text(await fetchIssues()({ buildId: OTHER.id, compareWith: OLD.id })); + + expect(listed(out, "New (only in the later build) (53)")).toHaveLength(50); + expect(out).toContain("(3 more not listed; see the CSV links.)"); + }); + + it("sets apart issues the compare API found on an error page, as a page that did not load", async () => { + compareReply = () => + comparison([apiRow("new", "region", "div.main"), apiRow("new", "link-name", "a", PAGE_B, PRIVOXY), apiRow("retained", "image-alt", "img")]); + + const out = text(await fetchIssues()({ buildId: NEW.id, compareWith: OLD.id })); + + expect(out).toContain("⚠️ 1 issue listed was found on an error page"); + expect(listed(out, "New (only in the later build) (2)").map((r) => r.issue_type)).toEqual(["region"]); + expect(listed(out, "Page did not load (1)")).toMatchObject([{ issue_type: "link-name", list: "new", on_error_page: true }]); + }); + + it("sets apart a page that did not load in the later build when it matches the lists itself", async () => { + exportsOf[OLD.id] = [csvRow("image-alt", "img"), csvRow("label", "input#q", PAGE_B)]; + exportsOf[OTHER.id] = [csvRow("image-alt", "img"), csvRow("link-name", "a", PAGE_B, PRIVOXY)]; + + const out = text(await fetchIssues()({ buildId: OTHER.id, compareWith: OLD.id })); + + expect(out).toContain(`⚠️ 1 page did not load in build #1 (${OTHER.id}): an error page was scanned in its place.`); + expect(out).toContain("Run the tests again once it loads."); + expect(out).toContain("0 new, 0 fixed, 1 still open."); + expect(listed(out, `Not checked (page did not load in build #1 (${OTHER.id}))`).map((r) => r.issue_type)).toEqual(["label"]); + }); + + it("sets apart a page whose error-page issues are all duplicates of another page's", async () => { + // A grouped report: the error page on page A is the main issue, the same error page on + // page B a duplicate of it. Page B did not load either. + const main = csvRow("link-name", "a", PAGE_A, PRIVOXY); + exportsOf[OLD.id] = [csvRow("image-alt", "img"), csvRow("label", "input#q", PAGE_B)]; + exportsOf[OTHER.id] = [main, { ...csvRow("link-name", "a", PAGE_B, PRIVOXY), "Duplicate Issue of": main["Issue link"] }]; + + const out = text(await fetchIssues()({ buildId: OTHER.id, compareWith: OLD.id })); + + expect(out).toContain(`⚠️ 2 pages did not load in build #1 (${OTHER.id})`); + expect(out).toContain("0 new, 0 fixed, 0 still open."); + expect(listed(out, `Not checked (page did not load in build #1 (${OTHER.id}))`).map((r) => r.issue_type).sort()).toEqual(["image-alt", "label"]); + }); + + it("refuses to compare a build with itself", async () => { + const result = await fetchIssues()({ buildId: NEW.id, compareWith: NEW.id }); + + expect(result.isError).toBe(true); + expect(text(result)).toContain("compareWith names this build itself"); + }); + + it("says to read both, as guidance, when compareWith is an app build", async () => { + const result = await fetchIssues()({ buildId: NEW.id, compareWith: APP_BUILD }); + + expect(result.isError).toBeFalsy(); + expect(text(result)).toContain(`compareWith names App Accessibility build ${APP_BUILD}, which is not an Automated Tests build`); + expect(text(result)).toContain("Read both reports with fetchAccessibilityIssues"); + expect(compareCalls()).toEqual([]); + }); + + it("says plainly when compareWith names no build, without a request to report it", async () => { + const result = await fetchIssues()({ buildId: NEW.id, compareWith: "zz9unknownbuildfffffffffffffffffffffffff" }); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch(/^compareWith: No Automated Tests \(web\) or App Accessibility build/); + expect(text(result)).not.toMatch(/GitHub/); + }); + + it("starts from an Automated Tests report link given as reportUrl", async () => { + const link = `https://accessibility.browserstack.com/automated-tests/projects/Shop/builds/checkout/2?thBuildId=${NEW.id}`; + + const out = text(await fetchIssues()({ reportUrl: link, compareWith: OLD.id })); + + expect(out).toContain(`Resolved the report link to Automated Tests build ${NEW.id}.`); + expect(compareCalls()[0].get("newReport")).toBe(NEW.id); + }); +});