diff --git a/docs/TOOLS.md b/docs/TOOLS.md index e8233c92..ac6e0e89 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -344,7 +344,7 @@ searchCapability → describeCapability → invokeCapability | `accessibilityExpert` | Ask the A11y Expert (WCAG 2.0/2.1/2.2, mobile/web usability, best practices). | | `createAccessibilityAuthConfig` | Create an authentication configuration (form-based or basic) for accessibility scans behind a login. | | `getAccessibilityAuthConfig` | Retrieve an existing accessibility authentication configuration by ID. | -| `fetchAccessibilityIssues` | Fetch accessibility issues, with pagination, from a Website Scanner run, an Automated Tests build (web) or App Accessibility build (mobile app, by the build id the SDK prints), a Workflow Analyzer report, an App Accessibility manual report (`appReportId`), or a report's dashboard link. Once the tests or scan have finished, waits while the report is still processing; a run still in progress is read as it is, with a note. For an app build with no scans, lists the likely causes. | +| `fetchAccessibilityIssues` | Fetch accessibility issues, with pagination, from a Website Scanner run, an Automated Tests build (web) or App Accessibility build (mobile app, by the build id the SDK prints), a Workflow Analyzer report, an App Accessibility manual report (`appReportId`), or a report's dashboard link (a Website Scanner link is read by the run's build id, however old the scan; a Workflow Analyzer link by the report id it carries). Once the tests or scan have finished, waits while the report is still processing; a run still in progress is read as it is, with a note. For an app build with no scans, lists the likely causes. | | `list_app_accessibility_builds` | List an App Accessibility project's builds newest first, with each build's id, status, app and issue counts - use this to find the latest run, a build by name, or one person's builds. | | `list_app_accessibility_projects` | List the account's App Accessibility projects newest first, or find one by name - use this to get the project id that lists its builds. | | `list_manual_app_accessibility_reports` | List manual App Accessibility reports (App Live sessions scanned with Workflow Analyzer) newest first, with each report's id, app, status and issue counts - use this to find a report QA shared or the latest manual scan of an app. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 2936bc04..82dc1449 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -9,6 +9,8 @@ import { AccessibilityReportFetcher, EXPORT_WAIT_MS, ExportPendingError, + ScanRunNotFoundError, + scanRunLink, type IssueSource, } from "./accessiblity-utils/report-fetcher.js"; import { @@ -213,6 +215,11 @@ function sourceArgs(source: IssueSource): string { switch (source.kind) { case "scan": return `scanId: "${source.scanId}", scanRunId: "${source.scanRunId}"`; + // A run read by its build id is named by its link again, or, given as a build id, by that. + case "scanBuild": + return source.scanName !== undefined + ? `reportUrl: "${scanRunLink(source)}"` + : `buildId: "${source.thBuildId}"`; case "build": case "appBuild": return `buildId: "${source.buildId}"`; @@ -228,6 +235,8 @@ function describeSource(source: IssueSource): string { switch (source.kind) { case "scan": return `Website Scanner scan ${source.scanId}, run ${source.scanRunId}`; + case "scanBuild": + return `Website Scanner run with build id ${source.thBuildId}`; case "build": return `Automated Tests build ${source.buildId}`; case "workflow": @@ -955,12 +964,69 @@ async function fetchAccessibilityIssues( requested.kind === "link" ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; - // A build id names a web build or a mobile app build; the APIs tell which. - const source = await reportFetcher.resolveBuild(named); + // A build id names a web build, a mobile app build, or a Website Scanner run (a scanner + // link's thBuildId); the APIs tell which. + let source = await reportFetcher.resolveBuild(named, { orScanRun: true }); + // Comparing runs needs the scan's ids, which only the scan list maps a build id to. + if (compareWith !== undefined && source.kind === "scanBuild") { + source = await reportFetcher.findScanRun(source); + } // Two scan runs are compared through their CSV exports; two app builds, here below. if (compareWith !== undefined && source.kind === "scan") { return compareScanRuns(source, compareWith, config, context); } + const pageArg = pageUrl ? `, pageUrl: "${pageUrl}"` : ""; + const notes: string[] = []; + // A run named by its build id is read by it, which works however old the scan is. If the + // API does not know it by that id, the scan list is searched as before. + let reportLink: string | undefined; + if (source.kind === "scanBuild") { + try { + reportLink = await reportFetcher.getReportLink(source, { + exportTaskId, + onWait: exportProgress(context), + }); + if (named.kind === "build") { + notes.push( + `Build id ${source.thBuildId} is a Website Scanner run's build id; read it as one.`, + ); + } + if (cursor === 0) { + notes.push( + "Read by the run's build id, so its processing status was not checked: a scan that finished in the last few minutes may still be adding issues.", + ); + } + } catch (error) { + if (error instanceof ExportPendingError) { + return createSuccessResponse( + exportPendingMessages(source, error, pageArg), + ); + } + if (!(error instanceof ScanRunNotFoundError)) throw error; + if (source.scanName === undefined) { + throw new Error( + error.routeMissing + ? `No Automated Tests (web) or App Accessibility build with id "${source.thBuildId}". If it is a Website Scanner run's build id, this API cannot read a run by it yet: pass the scan's report link as reportUrl instead` + : `No Automated Tests (web), App Accessibility or Website Scanner build with id "${source.thBuildId}". Pass the id the BrowserStack SDK printed as "testhub id" at the end of the run (40 lowercase letters and digits), or the report's dashboard link as reportUrl`, + ); + } + const why = error.routeMissing + ? "This API does not read a run by its build id yet" + : "The API could not read this run by the link's build id (not found, or not visible to this account)"; + try { + source = await reportFetcher.findScanRun(source); + } catch (searchError) { + throw new Error( + `${why}, and searching the scan list found nothing either: ${ + searchError instanceof Error + ? searchError.message + : String(searchError) + }`, + ); + } + notes.push(`${why}, so it was found in the scan list instead.`); + } + } if (compareWith !== undefined && source.kind !== "appBuild") { return createErrorResponse( `compareWith compares Website Scanner runs and App Accessibility builds; ${describeSource(source)} is neither.`, @@ -975,10 +1041,12 @@ async function fetchAccessibilityIssues( const stillProcessing = processing.status !== undefined && PROCESSING_STATUSES.has(processing.status); - const preface = - requested.kind === "link" + const preface = [ + ...(requested.kind === "link" ? [`Resolved the report link to ${describeSource(source)}.`] - : []; + : []), + ...notes, + ]; if (source.kind === "appBuild" && compareWith !== undefined) { return compareAppBuilds( @@ -1015,10 +1083,8 @@ async function fetchAccessibilityIssues( ); } - const pageArg = pageUrl ? `, pageUrl: "${pageUrl}"` : ""; - let reportLink: string; try { - reportLink = await reportFetcher.getReportLink(source, { + reportLink ??= await reportFetcher.getReportLink(source, { exportTaskId, onWait: exportProgress(context), stillProcessing, @@ -1156,10 +1222,13 @@ async function compareScanRuns( context?: ScanProgressContext, ): Promise { const reportFetcher = await initializeReportFetcher(config); - const source = + const named = requested.kind === "link" ? await reportFetcher.sourceFromReportUrl(requested.reportUrl) : requested; + // Runs are compared by their scan's ids, which only the scan list maps a build id to. + const source = + named.kind === "scanBuild" ? await reportFetcher.findScanRun(named) : named; if (source.kind !== "scan") { return createErrorResponse( 'compareWith compares Website Scanner runs: pass the newer run (scanId and scanRunId, or its report link) and compareWith: "previous" or the older run\'s scanRunId.', diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index 1f99bb13..9b3f91a0 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -30,15 +30,50 @@ interface ReportResponse { /** * What to read issues from: a Website Scanner run, an Automated Tests build, a Workflow * Analyzer report, or an App Accessibility build (a mobile app run through App Automate). + * A Website Scanner run is named by its ids, or (`scanBuild`) by the build id its dashboard + * link carries, which the API reads without the ids. */ export type IssueSource = | { kind: "scan"; scanId: string; scanRunId: string } + | ScanBuildSource | { kind: "build"; buildId: string } | { kind: "workflow"; reportId: string } | { kind: "appBuild"; buildId: string } | { kind: "appReport"; reportId: string }; +/** A Website Scanner run named by its build id (a dashboard link's thBuildId), and the scan's name when a link gave it. */ +export type ScanBuildSource = { + kind: "scanBuild"; + thBuildId: string; + scanName?: string; +}; + const apiBase = () => `${getRegionalHost("accessibilityApi")}/api`; +const dashboard = () => getRegionalHost("accessibilityDashboard"); + +/** + * The API does not know a Website Scanner run by this build id, or not for this account: + * a 404, which the API gives alike for an unknown id and one the caller may not see. An + * API that does not have the route yet answers a 404 page instead (`routeMissing`). + */ +export class ScanRunNotFoundError extends Error { + constructor( + readonly thBuildId: string, + readonly routeMissing = false, + ) { + super( + routeMissing + ? "This API does not read a Website Scanner run by its build id yet" + : `No Website Scanner run with build id ${thBuildId} that this account can read`, + ); + this.name = "ScanRunNotFoundError"; + } +} + +/** The dashboard link of a run named by its build id, without the extra query a copied link carries. */ +export function scanRunLink(source: ScanBuildSource): string { + return `${dashboard()}/site-scanner/projects/${encodeURIComponent(source.scanName ?? "")}/builds?thBuildId=${encodeURIComponent(source.thBuildId)}`; +} /** * How far a report link's scan is searched for. The scan list has no name or link filter and @@ -75,6 +110,12 @@ const EXPORT_POLL_MS = [1000, 2000, 3000, 5000]; /** A finished export's link is reused until shortly before it expires (about 12 hours). */ const LINK_EXPIRY_MARGIN_MS = 10 * 60 * 1000; const LINK_FALLBACK_TTL_MS = 60 * 60 * 1000; +/** + * A run read by its build id (`scanBuild`) has no status check: its export may be a snapshot + * of a run still adding issues, so its link is reused only briefly. Long enough that paging + * through the report reuses it; short enough that a later read sees the finished run. + */ +const SCAN_BUILD_LINK_TTL_MS = 5 * 60 * 1000; /** * Exports per report: the finished link to reuse for later pages, or the task of an export @@ -164,6 +205,14 @@ function exportUrls(source: IssueSource): { poll: (taskId) => `${route}?task_id=${q(taskId)}`, }; } + case "scanBuild": { + // The run's issues by its build id; the poll finds the run again from the task id. + const route = `${apiBase()}/website-scanner/v1/scan_runs/issues`; + return { + start: `${route}?thBuildId=${q(source.thBuildId)}`, + poll: (taskId) => `${route}?task_id=${q(taskId)}`, + }; + } case "build": { const route = `${apiBase()}/automated-tests/v1/builds/issues`; return { @@ -232,11 +281,11 @@ export class AccessibilityReportFetcher { /** * The source a dashboard report link points at. An Automated Tests link carries the build - * id itself. An Automate build or session link is mapped to that build through a session - * (see buildOfAutomateSession). A Website Scanner link carries the thBuildId of one run, - * which only the scan list can map to ids: each scan there carries its latest run's link. - * A link to an older run therefore matches no scan, and is reported as such rather than - * guessed at. An App Accessibility link names its build by project, name and number, + * id itself, and a Workflow Analyzer report link its report id (`ids`). An Automate build + * or session link is mapped to that build through a session (see buildOfAutomateSession). + * A Website Scanner link carries the build id (thBuildId) of one run, which the API reads + * directly (`scanBuild`); only what needs the scan's ids searches the scan list for it + * (findScanRun). An App Accessibility link names its build by project, name and number, * which the project's build list maps to the id. An App Automate build or session link * maps to the accessibility build through a session's test code. */ @@ -251,6 +300,32 @@ export class AccessibilityReportFetcher { // link it shows as its reportUrl parameter. const shared = url?.searchParams.get("reportUrl"); if (shared && shared !== reportUrl) return this.sourceFromReportUrl(shared); + const reportPage = url?.pathname.match( + /\/(workflow-analyzer|reports|assisted-test)\/report\/?$/, + )?.[1]; + if (url && reportPage === "assisted-test") { + throw new Error( + "This is an Assisted Test report link: fetchAccessibilityIssues does not read Assisted Test reports. It reads Website Scanner runs, Automated Tests and App Accessibility builds, and Workflow Analyzer reports", + ); + } + const reportIds = reportPage ? url?.searchParams.get("ids") : null; + if (reportIds) { + const ids = reportIds + .split(",") + .map((id) => id.trim()) + .filter(Boolean); + if (!ids.length) { + throw new Error( + "This Workflow Analyzer link's ids parameter names no report. Pass the report's id as workflowReportId, or copy the link again", + ); + } + if (ids.length > 1) { + throw new Error( + `This link opens several Workflow Analyzer reports (${ids.length}). Pass the one to read as workflowReportId`, + ); + } + return { kind: "workflow", reportId: ids[0] }; + } const manualReportId = url && manualReportIdFromDashboardLink(url); if (manualReportId) return { kind: "appReport", reportId: manualReportId }; const appBuildId = @@ -275,10 +350,14 @@ export class AccessibilityReportFetcher { /\/site-scanner\/projects\/([^/]+)\/builds/, )?.[1]; if (thBuildId && scanName) { - return this.findScanRun(thBuildId, decodeURIComponent(scanName)); + return { + kind: "scanBuild", + thBuildId, + scanName: decodeURIComponent(scanName), + }; } throw new Error( - "This link does not name a report the API can look up. Pass an Automated Tests or Website Scanner report link from the Accessibility dashboard (both carry thBuildId), an App Accessibility build link (automated-tests/projects/…/builds/…), an Automate or App Automate build or session link, a scanner share link, an App Accessibility manual report link (report/…), or the ids: scanId with scanRunId, buildId, workflowReportId, or appReportId", + "This link does not name a report the API can look up. Pass an Automated Tests or Website Scanner report link from the Accessibility dashboard (both carry thBuildId), a Workflow Analyzer report link (workflow-analyzer/report?ids=…), an App Accessibility build link (automated-tests/projects/…/builds/…), an Automate or App Automate build or session link, a scanner share link, an App Accessibility manual report link (report/…), or the ids: scanId with scanRunId, buildId, workflowReportId, or appReportId", ); } @@ -286,9 +365,12 @@ export class AccessibilityReportFetcher { * A build id names a web build or a mobile app build: both are TestHub ids, and only the * API that ran the build knows it. The web API is asked first; a build it does not know * that the App Accessibility API does is an app build. One neither knows is refused - * here, naming both. + * here, naming both, or with `orScanRun` taken as a Website Scanner run's build id. */ - async resolveBuild(source: IssueSource): Promise { + async resolveBuild( + source: IssueSource, + { orScanRun = false }: { orScanRun?: boolean } = {}, + ): Promise { if (source.kind !== "build") return source; // Only a 404 means the web API does not know the id: any other failure is not an // answer about the build, and asking the app API next would hide it. @@ -311,6 +393,9 @@ export class AccessibilityReportFetcher { } const meta = await this.app.buildMeta(source.buildId); if (meta) return { kind: "appBuild", buildId: source.buildId }; + // A Website Scanner run's build id (a scanner link's thBuildId) is a TestHub id too, which + // only the scanner's issues route knows: the caller reads it there, and says so if not. + if (orScanRun) return { kind: "scanBuild", thBuildId: source.buildId }; throw new Error( `No Automated Tests (web) or App Accessibility build with id "${source.buildId}". Pass the id the BrowserStack SDK printed as "testhub id" at the end of the run (40 lowercase letters and digits), or the report's dashboard link as reportUrl`, ); @@ -412,11 +497,16 @@ export class AccessibilityReportFetcher { return { kind: "build", buildId }; } - /** Walk the newest scans for the one whose own link carries this thBuildId. */ - private async findScanRun( - thBuildId: string, - scanName: string, - ): Promise { + /** + * A run named by its build id, as its scan's ids: the newest scans are walked for the one + * whose own link carries this thBuildId. What needs the ids (comparing runs) uses this, as + * does a link the API does not know by its build id. + */ + async findScanRun({ thBuildId, scanName }: ScanBuildSource): Promise<{ + kind: "scan"; + scanId: string; + scanRunId: string; + }> { let nextPage: string | undefined; let sameName: number | undefined; for (let page = 0; page < MAX_SCAN_LIST_PAGES; page++) { @@ -441,7 +531,7 @@ export class AccessibilityReportFetcher { scanRunId: String(scanRunId), }; } - if (sameName === undefined && scan.name === scanName) { + if (sameName === undefined && scanName && scan.name === scanName) { sameName = scan.id; } } @@ -496,6 +586,12 @@ export class AccessibilityReportFetcher { raise_error: false, }); const initData: ReportInitResponse = initResp.data; + if (source.kind === "scanBuild" && initResp.status === 404) { + reportExports.delete(key); + // The API's own 404 is JSON; a 404 page means the route is not there. + const routeMissing = typeof initData !== "object" || initData === null; + throw new ScanRunNotFoundError(source.thBuildId, routeMissing); + } if (!initResp.ok || !initData?.success) { throw new Error(startFailure(source, initResp.status, initData)); } @@ -518,7 +614,13 @@ export class AccessibilityReportFetcher { const link = reportData.data?.reportLink; if (link) { if (options.stillProcessing) reportExports.delete(key); - else rememberExport(key, { link, expiresAt: linkExpiry(link) }); + else { + const expiresAt = + source.kind === "scanBuild" + ? Math.min(linkExpiry(link), Date.now() + SCAN_BUILD_LINK_TTL_MS) + : linkExpiry(link); + rememberExport(key, { link, expiresAt }); + } return link; } const message = reportData.data?.message ?? ""; @@ -591,6 +693,9 @@ export class AccessibilityReportFetcher { `${apiBase()}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}/status`, ) )?.status; + case "scanBuild": + // A run's status is read by its scan's ids, which a build id does not give. + return undefined; case "workflow": return ( await this.getData( diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index 4ebb84ee..bffbf768 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -12,6 +12,7 @@ const m = vi.hoisted(() => ({ parse: vi.fn(), loadMainIssues: vi.fn(), compareIssues: vi.fn(), + findScanRun: vi.fn(), })); vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ @@ -23,6 +24,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ resolveBuild = vi.fn(async (source: unknown) => source); previousScanRun = m.previousScanRun; changesSinceLastRun = m.changesSinceLastRun; + findScanRun = m.findScanRun; }, EXPORT_WAIT_MS: 45_000, ExportPendingError: class extends Error { @@ -177,6 +179,21 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { expect(out).toMatch(/image-alt/); }); + it("compares a scanner link's run through its scan's ids, which only the scan list has", async () => { + const link = "https://accessibility.browserstack.com/site-scanner/projects/shop/builds?thBuildId=k3v9q2m7x1c8b4n6z0w5r2t9y4p1s8d3f6g0h2ja"; + const byBuild = { kind: "scanBuild", thBuildId: "k3v9q2m7x1c8b4n6z0w5r2t9y4p1s8d3f6g0h2ja", scanName: "shop" }; + m.sourceFromReportUrl.mockResolvedValue(byBuild); + m.findScanRun.mockResolvedValue({ kind: "scan", scanId: "7310042", scanRunId: "5520318" }); + m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.changesSinceLastRun.mockResolvedValue({ newIssues: 1, retainedIssues: 146, resolvedIssues: 1 }); + + const result = await fetchIssues()({ reportUrl: link, compareWith: "previous" }); + + expect(m.findScanRun).toHaveBeenCalledWith(byBuild); + expect(m.previousScanRun).toHaveBeenCalledWith("7310042", "5520318"); + expect(text(result)).toMatch(/1 new, 1 resolved, 146 still open/); + }); + it("compares with a run it is given, counting by matching the issues", async () => { m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index fd093342..a560875f 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -131,6 +131,11 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { expect(get).not.toHaveBeenCalled(); }); + // A Website Scanner link is read by its run's build id; the scan list is searched only for + // what needs the scan's ids (comparing runs), or when the API does not know the build id. + const linkRun = (scanName: string, thBuildId: string) => + ({ kind: "scanBuild", thBuildId, scanName }) as const; + it("finds a Website Scanner link's scan and latest run by paging the scan list", async () => { get .mockResolvedValueOnce(scansPage([scan(1, "other", "aaa", 11)], "tok2")) @@ -139,7 +144,7 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { ); await expect( - fetcher().sourceFromReportUrl(scannerLink("checkout fix", "bbb")), + fetcher().findScanRun(linkRun("checkout fix", "bbb")), ).resolves.toEqual({ kind: "scan", scanId: "1234", scanRunId: "5678" }); // No limit: larger pages make the list skip scans, so it is walked at its default size. expect(get.mock.calls.map(([options]) => options.url)).toEqual([ @@ -154,7 +159,7 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { ); await expect( - fetcher().sourceFromReportUrl(scannerLink("checkout fix", "older")), + fetcher().findScanRun(linkRun("checkout fix", "older")), ).rejects.toThrow(/earlier run of scan "checkout fix" \(scanId 1234\)/); }); @@ -162,7 +167,7 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl", () => { get.mockResolvedValue(scansPage([scan(1, "other", "aaa", 11)], "more")); await expect( - fetcher().sourceFromReportUrl(scannerLink("checkout fix", "bbb")), + fetcher().findScanRun(linkRun("checkout fix", "bbb")), ).rejects.toThrow(/most recent scans.*scanId and scanRunId/s); expect(get).toHaveBeenCalledTimes(MAX_SCAN_LIST_PAGES); }); @@ -188,9 +193,9 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl and a scan without a ru }), ); - await expect(fetcher().sourceFromReportUrl(link)).rejects.toThrow( - /scan 1234 has no completed run/, - ); + await expect( + fetcher().findScanRun({ kind: "scanBuild", thBuildId: "bbb", scanName: "checkout" }), + ).rejects.toThrow(/scan 1234 has no completed run/); }); }); @@ -362,18 +367,13 @@ describe("AccessibilityReportFetcher.sourceFromReportUrl and a shared report lin it("reads the dashboard link a scanner share link carries", async () => { const inner = `https://accessibility.browserstack.com/site-scanner/projects/checkout/builds?thBuildId=bbb&recurring=false`; const share = `https://scanner.browserstack.com/site-scanner/checkout/a11y-report?reportUrl=${encodeURIComponent(inner)}&minimal=true&report_source=private_share`; - get.mockResolvedValueOnce( - ok({ - scans: [{ id: 1234, name: "checkout", scanUrl: inner, lastScanDetails: { scanRunId: 5678 } }], - pagination: { hasNext: false }, - }), - ); await expect(fetcher().sourceFromReportUrl(share)).resolves.toEqual({ - kind: "scan", - scanId: "1234", - scanRunId: "5678", + kind: "scanBuild", + thBuildId: "bbb", + scanName: "checkout", }); + expect(get).not.toHaveBeenCalled(); }); }); diff --git a/tests/tools/accessibilityReportLinks.test.ts b/tests/tools/accessibilityReportLinks.test.ts new file mode 100644 index 00000000..03f1a6b5 --- /dev/null +++ b/tests/tools/accessibilityReportLinks.test.ts @@ -0,0 +1,178 @@ +import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; + +// A report's dashboard link is all a user usually has. A Website Scanner link carries the +// run's build id (thBuildId), which the API now reads directly, so the link works however old +// the scan is; a Workflow Analyzer link carries the report's id in `ids`. + +vi.mock("../../src/lib/apiClient.js", () => ({ + apiClient: { get: vi.fn() }, +})); + +import { apiClient } from "../../src/lib/apiClient.js"; +import { + AccessibilityReportFetcher, + reportExports, + ScanRunNotFoundError, + scanRunLink, +} from "../../src/tools/accessiblity-utils/report-fetcher.js"; + +const get = apiClient.get as Mock; +const API = "https://api-accessibility.browserstack.com/api"; +const DASHBOARD = "https://accessibility.browserstack.com"; +const TH_BUILD = "k3v9q2m7x1c8b4n6z0w5r2t9y4p1s8d3f6g0h2ja"; + +const ok = (data: unknown) => ({ ok: true, status: 200, data: { success: true, data } }); +const failed = (status: number, error: unknown) => ({ + ok: false, + status, + data: { success: false, data: null, error }, +}); + +function fetcher() { + const f = new AccessibilityReportFetcher(); + f.setAuth({ username: "u", password: "p" }); + return f; +} + +const scannerLink = (name: string, thBuildId: string) => + `${DASHBOARD}/site-scanner/projects/${encodeURIComponent(name)}/builds?thBuildId=${thBuildId}&%3Brecurring=false`; + +describe("a Website Scanner report link", () => { + beforeEach(() => { + vi.clearAllMocks(); + reportExports.clear(); + }); + + it("names the run by its build id, without searching the scan list", async () => { + await expect( + fetcher().sourceFromReportUrl(scannerLink("checkout fix", TH_BUILD)), + ).resolves.toEqual({ kind: "scanBuild", thBuildId: TH_BUILD, scanName: "checkout fix" }); + expect(get).not.toHaveBeenCalled(); + }); + + it("is read through the issues route that takes the build id", async () => { + get + .mockResolvedValueOnce(ok({ task_id: "ext_api_issues:WebsiteScannerTest:x:1:2", message: "started" })) + .mockResolvedValueOnce(ok({ thBuildId: TH_BUILD, reportLink: "https://csv" })); + + await expect( + fetcher().getReportLink({ kind: "scanBuild", thBuildId: TH_BUILD, scanName: "checkout fix" }), + ).resolves.toBe("https://csv"); + expect(get.mock.calls.map(([options]) => options.url)).toEqual([ + `${API}/website-scanner/v1/scan_runs/issues?thBuildId=${TH_BUILD}`, + `${API}/website-scanner/v1/scan_runs/issues?task_id=ext_api_issues%3AWebsiteScannerTest%3Ax%3A1%3A2`, + ]); + }); + + it("says when the API does not know the run by its build id, so the caller can search instead", async () => { + get.mockResolvedValueOnce(failed(404, "Scan run not found. Invalid thBuildId or task id.")); + + await expect( + fetcher().getReportLink({ kind: "scanBuild", thBuildId: TH_BUILD }), + ).rejects.toBeInstanceOf(ScanRunNotFoundError); + }); + + it("tells an API without the route from one that does not know the run", async () => { + get.mockResolvedValueOnce({ ok: false, status: 404, data: "Not Found" }); + + const error = await fetcher() + .getReportLink({ kind: "scanBuild", thBuildId: TH_BUILD }) + .catch((e) => e); + + expect(error).toBeInstanceOf(ScanRunNotFoundError); + expect(error.routeMissing).toBe(true); + }); + + it("passes on the API's reason for any other refusal", async () => { + get.mockResolvedValueOnce(failed(422, ["thBuildId must be the build id from the report link"])); + + await expect( + fetcher().getReportLink({ kind: "scanBuild", thBuildId: "bad" }), + ).rejects.toThrow("thBuildId must be the build id from the report link"); + }); + + it("is named again by a clean link, for the next page", () => { + expect(scanRunLink({ kind: "scanBuild", thBuildId: TH_BUILD, scanName: 'checkout "fix"' })).toBe( + `${DASHBOARD}/site-scanner/projects/checkout%20%22fix%22/builds?thBuildId=${TH_BUILD}`, + ); + }); + + it("can still be found in the scan list, for what needs the scan's ids", async () => { + get.mockResolvedValueOnce( + ok({ + scans: [ + { + id: 1234, + name: "checkout fix", + scanUrl: scannerLink("checkout fix", TH_BUILD), + lastScanDetails: { scanRunId: 5678 }, + }, + ], + pagination: { hasNext: false }, + }), + ); + + await expect( + fetcher().findScanRun({ kind: "scanBuild", thBuildId: TH_BUILD, scanName: "checkout fix" }), + ).resolves.toEqual({ kind: "scan", scanId: "1234", scanRunId: "5678" }); + }); + + it("has no processing status to read without the scan's ids", async () => { + await expect( + fetcher().reportStatus({ kind: "scanBuild", thBuildId: TH_BUILD }), + ).resolves.toBeUndefined(); + expect(get).not.toHaveBeenCalled(); + }); +}); + +describe("a Workflow Analyzer report link", () => { + beforeEach(() => vi.clearAllMocks()); + + it.each([ + [`${DASHBOARD}/workflow-analyzer/report?ids=123456`], + [`${DASHBOARD}/reports/report?ids=123456`], + [`${DASHBOARD}/workflow-analyzer/report/?ids=123456&tab=issues`], + ])("names the report by the id it carries: %s", async (link) => { + await expect(fetcher().sourceFromReportUrl(link)).resolves.toEqual({ + kind: "workflow", + reportId: "123456", + }); + expect(get).not.toHaveBeenCalled(); + }); + + it("asks for one report when the link opens several", async () => { + await expect( + fetcher().sourceFromReportUrl(`${DASHBOARD}/workflow-analyzer/report?ids=123,456`), + ).rejects.toThrow(/several Workflow Analyzer reports.*workflowReportId/s); + }); +}); + +describe("an Assisted Test report link", () => { + it("says this tool does not read Assisted Test reports, rather than that the link is unknown", async () => { + await expect( + fetcher().sourceFromReportUrl(`${DASHBOARD}/assisted-test/report?ids=98765`), + ).rejects.toThrow(/Assisted Test reports/); + }); +}); + +describe("a build id no build API knows", () => { + const missing = () => ({ ok: false, status: 404, data: { success: false, error: ["Build not found"] } }); + + beforeEach(() => vi.clearAllMocks()); + + it("is tried as a Website Scanner run's build id when asked to", async () => { + get.mockResolvedValueOnce(missing()).mockResolvedValueOnce(missing()); + + await expect( + fetcher().resolveBuild({ kind: "build", buildId: TH_BUILD }, { orScanRun: true }), + ).resolves.toEqual({ kind: "scanBuild", thBuildId: TH_BUILD }); + }); + + it("is still refused when only build APIs are asked", async () => { + get.mockResolvedValueOnce(missing()).mockResolvedValueOnce(missing()); + + await expect( + fetcher().resolveBuild({ kind: "build", buildId: TH_BUILD }), + ).rejects.toThrow(/No Automated Tests \(web\) or App Accessibility build/); + }); +}); diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 0a8084c1..ffacf9f4 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -13,6 +13,7 @@ const m = vi.hoisted(() => ({ resolveBuild: vi.fn(async (source: unknown) => source), appBuildReport: vi.fn(), appReportIssues: vi.fn(), + findScanRun: vi.fn(), parse: vi.fn(), dashboardLink: vi.fn(), })); @@ -26,7 +27,10 @@ vi.mock("../../src/tools/accessiblity-utils/scanner.js", () => ({ dashboardLink = m.dashboardLink; }, })); -vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ +vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", async () => ({ + ...(await vi.importActual< + typeof import("../../src/tools/accessiblity-utils/report-fetcher.js") + >("../../src/tools/accessiblity-utils/report-fetcher.js")), AccessibilityReportFetcher: class { setAuth = vi.fn(); getReportLink = m.getReportLink; @@ -35,6 +39,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ resolveBuild = m.resolveBuild; appBuildReport = m.appBuildReport; appReportIssues = m.appReportIssues; + findScanRun = m.findScanRun; }, EXPORT_WAIT_MS: 45_000, ExportPendingError: class extends Error { @@ -66,7 +71,10 @@ import addAccessibilityTools, { REPORT_STATUS_POLL_MS, REPORT_STATUS_WAIT_MS, } from "../../src/tools/accessibility.js"; -import { ExportPendingError } from "../../src/tools/accessiblity-utils/report-fetcher.js"; +import { + ExportPendingError, + ScanRunNotFoundError, +} from "../../src/tools/accessiblity-utils/report-fetcher.js"; const CONFIG = { "browserstack-username": "fake-user", @@ -823,6 +831,138 @@ describe("fetchAccessibilityIssues and App Accessibility manual reports", () => }); }); +// A Website Scanner link is read by its run's build id (thBuildId), so it works however old the +// scan is. The scan list is searched only when the API does not know the build id, so a link +// that read before still reads; and a build id the build APIs do not know is tried as a run's. +describe("fetchAccessibilityIssues and a report link read by its build id", () => { + let handlers: Record any>; + const TH = "k3v9q2m7x1c8b4n6z0w5r2t9y4p1s8d3f6g0h2ja"; + const LINK = `https://accessibility.browserstack.com/site-scanner/projects/checkout%20fix/builds?thBuildId=${TH}&%3Brecurring=false`; + const CLEAN_LINK = `https://accessibility.browserstack.com/site-scanner/projects/checkout%20fix/builds?thBuildId=${TH}`; + const BY_BUILD = { kind: "scanBuild", thBuildId: TH, scanName: "checkout fix" }; + const BY_IDS = { kind: "scan", scanId: "scan-42", scanRunId: "run-7" }; + + beforeEach(() => { + vi.clearAllMocks(); + m.getReportLink.mockReset().mockResolvedValue("https://csv"); + m.parse.mockResolvedValue(GROUPED); + m.reportStatus.mockResolvedValue(undefined); + m.resolveBuild.mockImplementation(async (source: unknown) => source); + m.sourceFromReportUrl.mockResolvedValue(BY_BUILD); + handlers = register(); + }); + + it("reads a scanner link's run by its build id, and names it by its link for the next page", async () => { + const result = await handlers.fetchAccessibilityIssues({ reportUrl: LINK }, CONTEXT); + + expect(result.isError).toBeFalsy(); + expect(m.getReportLink).toHaveBeenCalledWith(BY_BUILD, expect.anything()); + expect(m.findScanRun).not.toHaveBeenCalled(); + const body = text(result); + expect(body).toContain(`Resolved the report link to Website Scanner run with build id ${TH}.`); + expect(body).toContain( + `Use fetchAccessibilityIssues with reportUrl: "${CLEAN_LINK}" and cursor: 140`, + ); + expect(body).toMatch(/processing status was not checked/); + }); + + it("does not repeat the processing caveat on a later page", async () => { + const body = text( + await handlers.fetchAccessibilityIssues({ reportUrl: LINK, cursor: 140 }, CONTEXT), + ); + + expect(body).not.toMatch(/processing status was not checked/); + }); + + it("continues a still-generating export through the same link", async () => { + m.getReportLink.mockRejectedValueOnce(new (ExportPendingError as any)("t:9", 45_000)); + + const body = text(await handlers.fetchAccessibilityIssues({ reportUrl: LINK }, CONTEXT)); + + expect(body).toContain(`reportUrl: "${CLEAN_LINK}" and exportTaskId: "t:9"`); + }); + + it("falls back to the scan list when the API does not know the run by its build id", async () => { + m.getReportLink + .mockRejectedValueOnce(new ScanRunNotFoundError(TH)) + .mockResolvedValueOnce("https://csv"); + m.findScanRun.mockResolvedValue(BY_IDS); + + const result = await handlers.fetchAccessibilityIssues({ reportUrl: LINK }, CONTEXT); + + expect(result.isError).toBeFalsy(); + expect(m.findScanRun).toHaveBeenCalledWith(BY_BUILD); + expect(m.getReportLink).toHaveBeenLastCalledWith(BY_IDS, expect.anything()); + const body = text(result); + expect(body).toMatch(/found in the scan list instead/); + expect(body).toContain( + 'Use fetchAccessibilityIssues with scanId: "scan-42", scanRunId: "run-7" and cursor: 140', + ); + }); + + it("says the API cannot read a run by its build id yet, where it has no such route", async () => { + m.getReportLink + .mockRejectedValueOnce(new ScanRunNotFoundError(TH, true)) + .mockResolvedValueOnce("https://csv"); + m.findScanRun.mockResolvedValue(BY_IDS); + + const body = text(await handlers.fetchAccessibilityIssues({ reportUrl: LINK }, CONTEXT)); + + expect(body).toContain( + "This API does not read a run by its build id yet, so it was found in the scan list instead.", + ); + expect(body).not.toMatch(/not visible to this account/); + }); + + it("says both ways failed when the scan list does not have the run either", async () => { + m.getReportLink.mockRejectedValue(new ScanRunNotFoundError(TH)); + m.findScanRun.mockRejectedValue( + new Error("No Website Scanner scan among the most recent scans (15 pages) matches this link. Pass scanId and scanRunId instead"), + ); + + const result = await handlers.fetchAccessibilityIssues({ reportUrl: LINK }, CONTEXT); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch(/by the link's build id.*scan list found nothing either.*15 pages/s); + }); + + it("reads a build id the build APIs do not know as a Website Scanner run's", async () => { + m.resolveBuild.mockResolvedValue({ kind: "scanBuild", thBuildId: TH }); + + const result = await handlers.fetchAccessibilityIssues({ buildId: TH }, CONTEXT); + + expect(result.isError).toBeFalsy(); + expect(m.resolveBuild).toHaveBeenCalledWith({ kind: "build", buildId: TH }, { orScanRun: true }); + expect(m.getReportLink).toHaveBeenCalledWith({ kind: "scanBuild", thBuildId: TH }, expect.anything()); + const body = text(result); + expect(body).toContain(`Build id ${TH} is a Website Scanner run's build id; read it as one.`); + expect(body).toContain(`Use fetchAccessibilityIssues with buildId: "${TH}" and cursor: 140`); + }); + + it("points at the report link when the API cannot read a run by its build id yet", async () => { + m.resolveBuild.mockResolvedValue({ kind: "scanBuild", thBuildId: TH }); + m.getReportLink.mockRejectedValue(new ScanRunNotFoundError(TH, true)); + + const result = await handlers.fetchAccessibilityIssues({ buildId: TH }, CONTEXT); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch(/cannot read a run by it yet: pass the scan's report link as reportUrl/); + }); + + it("names every API it asked when none knows the build id", async () => { + m.resolveBuild.mockResolvedValue({ kind: "scanBuild", thBuildId: TH }); + m.getReportLink.mockRejectedValue(new ScanRunNotFoundError(TH)); + + const result = await handlers.fetchAccessibilityIssues({ buildId: TH }, CONTEXT); + + expect(result.isError).toBe(true); + expect(text(result)).toMatch( + /No Automated Tests \(web\), App Accessibility or Website Scanner build with id/, + ); + expect(m.findScanRun).not.toHaveBeenCalled(); + }); +}); + // A web report's answer counts the whole report, says which issues a person still has to // confirm, and names each rule's text once; and a scan's answer links the scan's own page. describe("what a web report's answer says about its issues", () => {