diff --git a/capability/web_accessibility.capability-index.json b/capability/web_accessibility.capability-index.json index 3b3aa5cf..4f3067fd 100644 --- a/capability/web_accessibility.capability-index.json +++ b/capability/web_accessibility.capability-index.json @@ -3,7 +3,7 @@ "version": "1.0", "build_id": "web_accessibility-v1.0-lists", "web_accessibility": { - "summary": "BrowserStack Web Accessibility: WCAG accessibility reports for websites and web apps. Find the account's Workflow Analyzer reports from the browser extension, Website Scanner scans with their scan history, Automated Tests projects and builds from web tests executed with accessibility checks on, and Assisted Test reports, each with the id, creator, status and issue counts needed to pick one. Each list names the id the accessibility issue tool reads. These capabilities list reports; they do not read a report's individual issues, start scans, or change or delete anything. Mobile apps are covered by a separate product.", + "summary": "BrowserStack Web Accessibility: WCAG accessibility reports for websites and web apps. Find the account's Workflow Analyzer reports from the browser extension, Website Scanner scans with their scan history, Automated Tests projects and builds from web tests executed with accessibility checks on, and Assisted Test reports, each with the id, creator, status and issue counts needed to pick one. Each list names the id the accessibility issue tool reads. These capabilities list reports; they do not read a report's individual issues or start scans. They also keep the account's false-positive memory: issues users confirmed are not real, saved per rule so later reports filter them without asking again. Mobile apps are covered by a separate product.", "routing_terms": [ "web accessibility", "website accessibility", @@ -12,7 +12,10 @@ "site scanner", "workflow analyzer", "assisted test", - "automated tests accessibility" + "automated tests accessibility", + "false positive", + "not an issue", + "false positive memory" ], "auth": { "type": "http", @@ -411,6 +414,412 @@ "$response": "UpstreamFailure" } } + }, + { + "name": "submit_false_positive_feedback", + "method": "POST", + "path": "/api/fp-memory/v1/feedback", + "mode": "write", + "entity": "false_positive_feedback", + "body": [ + { + "name": "ruleCode", + "type": "string", + "required": true, + "description": "The issue's issue_type from fetchAccessibilityIssues, e.g. role-required.", + "example": "role-required" + }, + { + "name": "pageUrl", + "type": "string", + "required": true, + "description": "The issue's affected_page, an absolute URL; its host is the site the entry applies to." + }, + { + "name": "component", + "type": "string", + "description": "The issue's component, as fetchAccessibilityIssues gave it." + }, + { + "name": "htmlSnippet", + "type": "string", + "description": "The issue's HTML_snippet, as fetchAccessibilityIssues gave it." + }, + { + "name": "verdict", + "type": "string", + "required": true, + "values": ["false_positive", "not_applicable", "accepted_risk"], + "description": "false_positive unless the user says the issue is real but does not apply, or is a risk they accept." + }, + { + "name": "scope", + "type": "string", + "required": true, + "values": [ + "element", + "component", + "component_family", + "site_predicate" + ], + "description": "The widest set of elements the user confirmed the root cause holds for. element alone recalls almost nothing on the next report." + }, + { + "name": "rootCause", + "type": "string", + "required": true, + "minLength": 20, + "description": "Why the rule is wrong, in terms of how the element is built (e.g. its role is set inside its shadow root), not only that it works for the user." + }, + { + "name": "pattern", + "type": "object", + "required": true, + "description": "What identifies the element and its kind. predicate is required; add what you observed.", + "fields": [ + { + "name": "predicate", + "type": "string", + "required": true, + "minLength": 10 + }, + { + "name": "tag", + "type": "string" + }, + { + "name": "tagPrefix", + "type": "string" + }, + { + "name": "isCustomElement", + "type": "boolean" + }, + { + "name": "hasShadowRoot", + "type": "boolean" + }, + { + "name": "classes", + "type": "array" + }, + { + "name": "designSystem", + "type": "string" + } + ] + }, + { + "name": "evidence", + "type": "object", + "description": "What you checked, as {observations: [...]}: computed role, shadow DOM content, screen reader output." + }, + { + "name": "counterExamples", + "type": "array", + "maxItems": 50, + "description": "Similar-looking elements the user says ARE real issues, as short strings; they are never removed." + }, + { + "name": "userStatement", + "type": "string", + "required": true, + "description": "The user's own words, as they said them." + }, + { + "name": "global", + "type": "boolean", + "default": false, + "description": "True only when the user says it holds on all the account's sites; otherwise it applies to pageUrl's site." + }, + { + "name": "product", + "type": "string", + "values": [ + "website-scanner", + "automated-tests", + "workflow-analyzer" + ], + "description": "The product of the report the issue came from." + }, + { + "name": "reportReference", + "type": "string", + "description": "The report the issue came from: its scanRunId, buildId or workflowReportId." + } + ], + "intent": "Save an accessibility issue the user confirmed is a false positive, so later fetchAccessibilityIssues answers filter the same pattern without asking the user again.", + "guidance": [ + "Before saving, inspect the element yourself (open the page if you can, else read the issue's HTML) and write rootCause and pattern.predicate from what you saw: a record that only says the element works for the user matches almost nothing on the next report.", + "Propose the widest scope the root cause supports (e.g. component_family for every custom element of one design system) and have the user confirm it; ask them for any similar element that is a real issue and send it in counterExamples.", + "Send the issue's fields as fetchAccessibilityIssues gave them; selectors are not taken, as they differ per page.", + "warnings in the reply ask you to re-check the scope with the user; resend at the wider scope if they agree.", + "A 409 reply means an active entry already covers this rule, site, scope and predicate: nothing was saved, and data.existingId names that entry; tell the user it is already saved." + ], + "returns": [ + "data.id", + "data.ruleCode", + "data.docsUrl", + "data.site", + "data.global", + "data.activeEntriesForRule", + "data.warnings", + "data.existingId" + ], + "responses": { + "201": { + "description": "Saved.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ruleCode": { + "type": "string" + }, + "docsUrl": { + "type": "string" + }, + "site": { + "type": "string" + }, + "global": { + "type": "boolean" + }, + "activeEntriesForRule": { + "type": "integer" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "FpMemoryDisabled" + }, + "409": { + "$response": "FpFeedbackDuplicate" + }, + "422": { + "$response": "FpFeedbackRejected" + }, + "500": { + "$response": "InternalServerError" + } + } + }, + { + "name": "get_false_positive_memory", + "method": "GET", + "path": "/api/fp-memory/v1/memory", + "mode": "read", + "entity": "false_positive_feedback", + "query": [ + { + "name": "ruleCodes", + "type": "string", + "description": "Comma-separated rule codes (issue_type) to read, at most 200; omit for every rule.", + "example": "role-required,image-alt" + }, + { + "name": "site", + "type": "string", + "description": "A host or page URL: only that site's entries, plus those saved for all sites." + }, + { + "name": "limit", + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 100, + "description": "Entries per page, newest first." + }, + { + "name": "cursor", + "type": "string", + "description": "The previous page's data.nextCursor, to read the next page." + } + ], + "intent": "Read the account's saved false positives, grouped by rule with each rule's docs link, to filter accessibility issues with or to find an entry's id.", + "guidance": [ + "fetchAccessibilityIssues already brings the memory for its issues' rules; read here only when it says some entries did not fit, or to find an entry to revoke.", + "Apply an entry only to issues of the same rule, on its site unless global, matching its pattern and rootCause, never the selector." + ], + "returns": [ + "data.rules.ruleCode", + "data.rules.docsUrl", + "data.rules.entryCount", + "data.rules.entries", + "data.nextCursor" + ], + "responses": { + "200": { + "description": "The memory, grouped by rule; newest entries first, one page at a time.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "nextCursor": { + "type": ["string", "null"], + "description": "Set when more entries follow: pass it as cursor." + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "ruleCode": { + "type": "string" + }, + "docsUrl": { + "type": "string" + }, + "entryCount": { + "type": "integer" + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "site": { + "type": "string" + }, + "global": { + "type": "boolean" + }, + "verdict": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "rootCause": { + "type": "string" + }, + "pattern": { + "type": "object" + }, + "counterExamples": { + "type": "array" + }, + "userStatement": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "component": { + "type": "string" + }, + "htmlSnippet": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "FpMemoryDisabled" + }, + "422": { + "description": "Invalid limit, or a cursor that is not an entry of the account." + }, + "500": { + "$response": "InternalServerError" + } + } + }, + { + "name": "revoke_false_positive_feedback", + "method": "POST", + "path": "/api/fp-memory/v1/feedback/{id}/revoke", + "mode": "write", + "entity": "false_positive_feedback", + "path_params": [ + { + "name": "id", + "type": "string", + "required": true, + "description": "The entry's id (fp_…), from the memory or the save reply.", + "example": "fp_Ab12Cd34Ef56" + } + ], + "intent": "Remove a saved false positive the user no longer agrees with, so the issues it covered show again in later reports.", + "guidance": [ + "Revoked entries are kept on the account's side but no longer returned or applied; there is no undo, so save a new entry instead if the user changes their mind." + ], + "returns": ["data.id", "data.status"], + "responses": { + "200": { + "description": "Revoked.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + } + } + } + } + } + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "FpMemoryDisabled" + }, + "404": { + "description": "No active entry with this id in the account." + }, + "500": { + "$response": "InternalServerError" + } + } } ], "entities": { @@ -430,9 +839,7 @@ "id_convention": "A plain integer, discovered through the report list; it is the workflowReportId the accessibility issue tool reads.", "parents": [], "relations": [], - "capabilities": [ - "list_workflow_analyzer_reports" - ], + "capabilities": ["list_workflow_analyzer_reports"], "key_facts": [ "Reports are shared across the account, so the list holds everyone's; one person's are picked out by the creator on each row.", "A report is created by scanning in the browser extension; there is nothing to create or delete here." @@ -460,9 +867,7 @@ "via": "runs once, or again on each recurrence" } ], - "capabilities": [ - "list_website_scanner_scans" - ], + "capabilities": ["list_website_scanner_scans"], "key_facts": [ "A scan's own row carries only its latest run; the earlier runs are listed per scan.", "Scans are shared across the account, so the list holds everyone's." @@ -480,18 +885,14 @@ "previous scan" ], "id_convention": "A plain integer, read together with its scan's id; discovered through the scan's run list.", - "parents": [ - "website_scan" - ], + "parents": ["website_scan"], "relations": [ { "entity": "website_scan", "via": "belongs to one scan" } ], - "capabilities": [ - "list_website_scanner_scan_runs" - ], + "capabilities": ["list_website_scanner_scan_runs"], "key_facts": [ "A page that did not load in a run is still counted as scanned in some places; the run's failure count says how many did not load." ] @@ -514,9 +915,7 @@ "via": "groups the builds reported under its name" } ], - "capabilities": [ - "list_automated_tests_projects" - ], + "capabilities": ["list_automated_tests_projects"], "key_facts": [ "A project is created by the first test run reported under its name; there is nothing to create or delete here." ] @@ -533,18 +932,14 @@ "accessibility test run" ], "id_convention": "A 40-character string of lowercase letters and digits, the thBuildId in the build's dashboard link; discovered through the build list.", - "parents": [ - "automated_tests_project" - ], + "parents": ["automated_tests_project"], "relations": [ { "entity": "automated_tests_project", "via": "belongs to one project" } ], - "capabilities": [ - "list_automated_tests_builds" - ], + "capabilities": ["list_automated_tests_builds"], "key_facts": [ "A build appears only if its tests ran with accessibility checks on.", "status is the accessibility report's; a completed report whose build ran no tests reads failed." @@ -563,11 +958,33 @@ "id_convention": "A plain integer, discovered through the report list.", "parents": [], "relations": [], + "capabilities": ["list_assisted_test_reports"], + "key_facts": [ + "Its individual issues are read on the Accessibility dashboard; the list gives its counts." + ] + }, + "false_positive_feedback": { + "entity": "false_positive_feedback", + "title": "False-positive feedback", + "description": "One issue pattern a user confirmed is not a real problem, saved per rule to filter it from later reports.", + "aliases": [ + "false positive", + "false positive memory", + "not an issue", + "fp feedback", + "ignored issue pattern" + ], + "id_convention": "A string starting with fp_, returned when the feedback is saved and on each entry of the memory.", + "parents": [], + "relations": [], "capabilities": [ - "list_assisted_test_reports" + "submit_false_positive_feedback", + "get_false_positive_memory", + "revoke_false_positive_feedback" ], "key_facts": [ - "Its individual issues are read on the Accessibility dashboard; the list gives its counts." + "The memory belongs to the account: every user's agent reads it. An entry applies to the site it was saved on, or to all sites when global.", + "fetchAccessibilityIssues already brings the memory for the rules of the issues it answers with." ] } }, @@ -1268,6 +1685,24 @@ "schema": { "$schema": "Failure" } + }, + "FpMemoryDisabled": { + "description": "The caller's role cannot act on issues, or false-positive memory is not enabled for this account (error says which).", + "schema": { + "$schema": "Failure" + } + }, + "FpFeedbackRejected": { + "description": "The feedback was not saved: unknown rule, a root cause or predicate too thin to match later issues by, a pageUrl without a host, or the rule already holds 1000 entries. error says which.", + "schema": { + "$schema": "Failure" + } + }, + "FpFeedbackDuplicate": { + "description": "Not saved: an active entry already covers the same rule, site (or all sites), scope and predicate. data.existingId names it.", + "schema": { + "$schema": "Failure" + } } } } diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 00000000..907b7914 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 2936bc04..e03c6585 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -36,6 +36,11 @@ import { BrowserStackConfig } from "../lib/types.js"; import { elicitCredentialsIfSupported } from "../lib/elicit-credentials.js"; import logger from "../logger.js"; import { NEEDS_A11Y_CONFIG_ID, NEEDS_A11Y_SCAN_ID } from "./tool-handoff.js"; +import { + applyMemory, + memoryForReport, + type MemoryView, +} from "./accessiblity-utils/fp-memory.js"; interface AuthCredentials { username: string; @@ -1048,18 +1053,31 @@ async function fetchAccessibilityIssues( nextPage: cursor, pageUrl, }); + // The memory for this page's rules comes with the issues, so it is applied whether or + // not the agent thinks to ask for it. + const fp = applyMemory( + await memoryForReport(setupAuth(config), Object.keys(rules ?? {}), pageUrl), + records, + pageUrl, + ); + const beforeRemoval = fp.marked + ? ", before removing saved false positives" + : ""; return createSuccessResponse([ + ...fp.lead, ...preface, ...processingNote(processing), ...(page ? pageNote(page) : []), - `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, + `Retrieved ${page_length} accessibility issues (Total: ${total_issues}${beforeRemoval})`, // The whole report's counts, said on its first page. ...(cursor ? [] : webSummaryNote(summary, total_issues)), ...excludedIssuesNote(duplicate_issues, unlinked_duplicates, hidden_issues), fullReportLine(reportLink), WEB_ISSUES_NOTE, - `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: records, rules }))}`, + ...fp.notes, + `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: fp.records, rules }))}`, + ...fp.closing, closingMessage(next_page, remaining, source, stillProcessing, pageArg), ]); } @@ -1474,14 +1492,22 @@ function createScanSuccessResponse( dashboardLink: string | undefined, settings: ScanSettings = {}, applied?: ScanSettings, + fp: MemoryView<(typeof report.records)[number]> = applyMemory( + undefined, + report.records, + ), ): CallToolResult { const cursor = report.next_page; + const beforeRemoval = fp.marked + ? " (before removing saved false positives)" + : ""; const messages = [ + ...fp.lead, `Accessibility scan "${name}" completed. ${scanDashboardNote(dashboardLink, scanId)}`, `Scan ID: ${scanId} and Scan Run ID: ${scanRunId}`, scanSettingsNote(settings, applied), fullReportLine(reportUrl), - `We found ${report.total_issues} issues. Below are the details of the ${report.page_length} most critical issues.`, + `We found ${report.total_issues} issues${beforeRemoval}. Below are the details of the ${report.page_length} most critical issues.`, ...webSummaryNote(report.summary, report.total_issues), ...excludedIssuesNote( report.duplicate_issues, @@ -1489,7 +1515,9 @@ function createScanSuccessResponse( report.hidden_issues, ), WEB_ISSUES_NOTE, - `Scan results: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: report.records, rules: report.rules }))}`, + ...fp.notes, + `Scan results: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: fp.records, rules: report.rules }))}`, + ...fp.closing, ]; if (cursor !== null) { @@ -1564,15 +1592,25 @@ async function runAccessibilityScan( throw error; } + const report = await parseAccessibilityReportFromCSV(reportLink); return createScanSuccessResponse( name, - await parseAccessibilityReportFromCSV(reportLink), + report, scanId, scanRunId, reportLink, dashboardLink, settings, applied, + applyMemory( + await memoryForReport( + setupAuth(config), + Object.keys(report.rules ?? {}), + pageURL, + ), + report.records, + pageURL, + ), ); } @@ -1765,7 +1803,7 @@ export default function addAccessibilityTools( tools.fetchAccessibilityIssues = server.tool( "fetchAccessibilityIssues", - "Fetch accessibility issues from one source: scanId+scanRunId, buildId, workflowReportId, appReportId or reportUrl." + + "Fetch a web or app accessibility report's issues. Remove the saved false positives it marks before reporting any." + NEEDS_A11Y_SCAN_ID, { scanId: z diff --git a/src/tools/accessiblity-utils/fp-memory.ts b/src/tools/accessiblity-utils/fp-memory.ts new file mode 100644 index 00000000..db9f0d86 --- /dev/null +++ b/src/tools/accessiblity-utils/fp-memory.ts @@ -0,0 +1,276 @@ +import { apiClient } from "../../lib/apiClient.js"; +import { getRegionalHost } from "../../lib/regional-host.js"; +import { wrapUntrusted } from "../../lib/untrusted-content.js"; +import logger from "../../logger.js"; + +/** + * The account's false-positive (FP) memory: what users and their agents said is not a real + * issue, saved per rule so later reports can be filtered without asking the user again. + * + * Saving, reading and revoking entries are web_accessibility capabilities (no tool of their + * own, so the catalog stays within its description budget). This module only reads the + * memory for the rules of the issues fetchAccessibilityIssues is answering with, so it + * arrives with them whether or not the agent thinks to ask for it. + */ + +/** The capabilities an answer names: keep in step with web_accessibility's index. */ +export const SUBMIT_CAPABILITY = "submit_false_positive_feedback"; +export const READ_CAPABILITY = "get_false_positive_memory"; + +/** A read made while answering with issues must not hold the issues back for long. */ +const MEMORY_READ_TIMEOUT_MS = 8_000; + +/** The most entries one read returns; more than the answer's budget below holds anyway. */ +const MEMORY_PAGE_SIZE = 200; + +/** How much of an answer the memory may take; a rule's oldest entries go first. */ +export const MEMORY_CHARACTERS = 24_000; +const SNIPPET_CHARACTERS = 200; + +export interface FpEntry { + id: string; + site: string; + global: boolean; + verdict: string; + scope: string; + rootCause: string; + pattern: Record; + evidence?: Record | null; + counterExamples?: unknown[] | null; + userStatement: string; + pageUrl?: string; + component?: string; + htmlSnippet?: string; +} + +export interface FpRuleMemory { + ruleCode: string; + docsUrl: string; + entryCount: number; + entries: FpEntry[]; +} + +export interface FpMemory { + rules: FpRuleMemory[]; + /** Set when the account holds more entries for these rules than one read returns. */ + nextCursor: string | null; +} + +/** + * The memory for the rules of the issues being answered with. Undefined when the account + * does not have it (403: its release flag is off) or it cannot be read in time: the issues + * are then answered without it, never not at all. + */ +export async function memoryForReport( + auth: { username: string; password: string }, + ruleCodes: string[], + page?: string, +): Promise { + if (ruleCodes.length === 0) return undefined; + // With the page known, other sites' entries stay out of the page of entries read. + const site = page ? `&site=${encodeURIComponent(page)}` : ""; + try { + const resp = await apiClient.get({ + url: `${getRegionalHost("accessibilityApi")}/api/fp-memory/v1/memory?ruleCodes=${encodeURIComponent(ruleCodes.join(","))}&limit=${MEMORY_PAGE_SIZE}${site}`, + headers: { + Authorization: + "Basic " + + Buffer.from(`${auth.username}:${auth.password}`).toString("base64"), + }, + timeout: MEMORY_READ_TIMEOUT_MS, + raise_error: false, + }); + if (resp.status === 403) return undefined; + if (!resp.ok || !resp.data?.success) { + logger.warn(`False-positive memory not read: HTTP ${resp.status}`); + return undefined; + } + const memory = resp.data.data; + if (!Array.isArray(memory?.rules)) { + logger.warn("False-positive memory not read: no rules in the response"); + return undefined; + } + return { + nextCursor: memory.nextCursor ?? null, + rules: memory.rules.map((rule: FpRuleMemory) => ({ + ...rule, + entries: Array.isArray(rule.entries) ? rule.entries : [], + })), + }; + } catch (error) { + logger.warn( + `False-positive memory not read: ${error instanceof Error ? error.message : String(error)}`, + ); + return undefined; + } +} + +function compactEntry(entry: FpEntry) { + const snippet = entry.htmlSnippet ?? ""; + return { + id: entry.id, + site: entry.global ? "all sites" : entry.site, + verdict: entry.verdict, + scope: entry.scope, + root_cause: entry.rootCause, + pattern: entry.pattern, + ...(entry.counterExamples?.length + ? { counter_examples: entry.counterExamples } + : {}), + ...(entry.evidence ? { evidence: entry.evidence } : {}), + user_statement: entry.userStatement, + seed_issue: { + page: entry.pageUrl, + component: entry.component, + ...(snippet + ? { + html: + snippet.length > SNIPPET_CHARACTERS + ? `${snippet.slice(0, SNIPPET_CHARACTERS)}…` + : snippet, + } + : {}), + }, + }; +} + +/** + * The memory as an answer carries it, within `budget` characters: each rule with its docs + * link and its entries, newest first. When it does not fit, the oldest entries of the + * fullest rule are left out first, and counted, so the agent knows to read the rest. + */ +export function memoryPayload( + memory: FpMemory, + budget = MEMORY_CHARACTERS, +): { json: string; omitted: { rule: string; count: number }[] } { + const rules = memory.rules.map((rule) => ({ + rule: rule.ruleCode, + docs: rule.docsUrl, + entries: rule.entries.map(compactEntry), + })); + const omitted = new Map(); + let json = JSON.stringify(rules); + while (json.length > budget) { + const fullest = rules.reduce((a, b) => + b.entries.length > a.entries.length ? b : a, + ); + if (fullest.entries.length <= 1) break; + fullest.entries.pop(); + omitted.set(fullest.rule, (omitted.get(fullest.rule) ?? 0) + 1); + json = JSON.stringify(rules); + } + return { + json, + omitted: [...omitted].map(([rule, count]) => ({ rule, count })), + }; +} + +export const FP_APPLY_NOTE = + "How to apply the false-positive memory: an issue marked saved_fp_entries has false positives saved for its rule (issue_type) on its site; compare it with those entries only. Match on an entry's pattern and root_cause, never on selector or exact HTML: ids, selectors and labels differ per page and language. Remove it when the entry covers it within the entry's scope (element: that element; component: the same component; component_family: any component its pattern.predicate describes; site_predicate: anything on the site the predicate describes), and do not ask the user about it again. Never remove a counter_example. A marked issue that looks like the same pattern but falls outside the scope is a likely false positive: ask the user once per group, and if they agree save it at the wider scope. An issue without saved_fp_entries is not covered by the memory. Each rule's docs link explains what the rule checks."; + +export const FP_FILING_GUIDE = `When the user says an issue is not a real problem, save it with the web_accessibility capability ${SUBMIT_CAPABILITY} (invokeCapability), so later reports are filtered without asking again. First inspect the element yourself (open the page if you have a browser, else read the issue's HTML): what it is built from (tag, custom element, shadow root, classes, design system) and why the rule is wrong about it. Write rootCause about how the element is built, not only that it works for the user, and pattern.predicate as the widest set of elements that root cause covers (e.g. every custom element of the same design system). Propose that scope and ask the user to confirm it, and to name any similar element that is a real issue (counterExamples). Quote the user as they said it in userStatement. Mark global only when the user says it holds on all their sites.`; + +/** An issue as an answer lists it: its rule, and the page it is on when the answer spans several. */ +interface IssueLike { + issue_type: string; + affected_page?: string; +} + +/** What an answer with issues says about the memory, and where. */ +export interface MemoryView { + /** The issues, those with saved entries for their rule and site marked saved_fp_entries. */ + records: (T & { saved_fp_entries?: string[] })[]; + /** How many of them are marked. */ + marked: number; + /** First in the answer: remove the false positives before giving any list or count. */ + lead: string[]; + /** Before the issues: how to apply the memory, the entries, how to save a new one. */ + notes: string[]; + /** After the issues: how to report what is left. */ + closing: string[]; +} + +function hostOf(page: string | undefined): string | undefined { + try { + return page ? new URL(page).hostname.toLowerCase() : undefined; + } catch { + return undefined; + } +} + +/** + * The memory applied to one answer's issues. An agent handed the memory as one note among + * many reported the report's own totals without filtering, so the issues the memory may + * cover are marked with their entry ids, the answer opens by saying they are to be removed + * before anything is reported, and it closes by saying how to report what is left. Only + * the entries of these issues' rules and sites (or saved for all sites) are shown. + * `page` is the page every issue is on, when the answer is about one. + */ +export function applyMemory( + memory: FpMemory | undefined, + records: T[], + page?: string, +): MemoryView { + if (!memory) return { records, marked: 0, lead: [], notes: [], closing: [] }; + const byRule = new Map(memory.rules.map((rule) => [rule.ruleCode, rule])); + const used = new Map>(); + let marked = 0; + const annotated = records.map((record) => { + const host = hostOf(record.affected_page ?? page); + const ids = (byRule.get(record.issue_type)?.entries ?? []) + .filter((entry) => entry.global || (host && entry.site === host)) + .map((entry) => entry.id); + if (ids.length === 0) return record; + marked++; + const seen = used.get(record.issue_type) ?? new Set(); + ids.forEach((id) => seen.add(id)); + used.set(record.issue_type, seen); + return { ...record, saved_fp_entries: ids }; + }); + if (marked === 0) { + const unread = memory.nextCursor + ? [ + `Not all saved false positives were read here. Before you tell the user an issue is real when they say it is a known false positive, read its rule's entries with the web_accessibility capability ${READ_CAPABILITY} (ruleCodes set to its rule; pass each page's nextCursor as cursor for the next).`, + ] + : []; + return { + records, + marked, + lead: [], + notes: [...unread, FP_FILING_GUIDE], + closing: [], + }; + } + const shown: FpMemory = { + nextCursor: memory.nextCursor, + rules: memory.rules + .filter((rule) => used.has(rule.ruleCode)) + .map((rule) => { + const entries = rule.entries.filter((e) => + used.get(rule.ruleCode)!.has(e.id), + ); + return { ...rule, entries, entryCount: entries.length }; + }), + }; + const { json, omitted } = memoryPayload(shown); + return { + records: annotated, + marked, + lead: [ + `⚠️ Saved false positives apply to this report: ${marked} of the ${records.length} issues below are marked saved_fp_entries, as this account's users already confirmed false positives of their rule on their site. Before you give the user any list, count or summary of issues, check each marked issue against its entries (memory and how to apply it are below) and remove the ones they cover. The user already confirmed these, so do not ask them to confirm again, and leave them out even when the user asks for the full list or all issues: their report is the issues that are left. Every count in this answer is from before that removal.`, + ], + notes: [ + FP_APPLY_NOTE, + `False-positive memory: ${wrapUntrusted("saved false positives (user-written)", json)}`, + ...(omitted.length || memory.nextCursor + ? [ + `Not all entries fit here${omitted.length ? ` (${omitted.map((o) => `${o.count} older entries of ${o.rule}`).join(", ")})` : ""}. When a marked issue is not covered by the entries above, read its rule's entries with the web_accessibility capability ${READ_CAPABILITY} (ruleCodes set to its rule; pass each page's nextCursor as cursor for the next).`, + ] + : []), + FP_FILING_GUIDE, + ], + closing: [ + "When you answer the user: list only the issues left after removing the saved false positives, and lead with how many are left; say how many you removed for each rule, with the entry ids; and say that the report's own totals above count them, so the issues left are that many fewer.", + ], + }; +} diff --git a/tests/.DS_Store b/tests/.DS_Store new file mode 100644 index 00000000..f27e03de Binary files /dev/null and b/tests/.DS_Store differ diff --git a/tests/setup.ts b/tests/setup.ts index bdafb54c..89ae42db 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -18,3 +18,13 @@ vi.mock("sharp", () => { vi.mock("../src/lib/account-region.js", () => ({ accountRegion: () => undefined, })); + +// No false-positive memory reads while fetching issues; tests/tools/fpMemory.test.ts +// unmocks the module to test it. +vi.mock( + "../src/tools/accessiblity-utils/fp-memory.js", + async (importOriginal) => ({ + ...(await importOriginal()), + memoryForReport: async () => undefined, + }), +); diff --git a/tests/tools/accessibilityFpMemoryAnswer.test.ts b/tests/tools/accessibilityFpMemoryAnswer.test.ts new file mode 100644 index 00000000..9ea077ce --- /dev/null +++ b/tests/tools/accessibilityFpMemoryAnswer.test.ts @@ -0,0 +1,230 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// An agent asked to "fetch the issues from this report" was handed the saved false +// positives as one note among many, and reported the report's own list and totals without +// removing them. What has to hold: when saved entries cover issues in the answer, the +// answer opens by saying to remove them before giving any list or count, marks each issue +// they may cover, labels its totals as before that removal, and closes by saying how to +// report what is left. Without saved entries, the answer is what it always was. + +const m = vi.hoisted(() => ({ + parse: vi.fn(), + memoryForReport: vi.fn(), +})); + +vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ + AccessibilityReportFetcher: class { + setAuth = vi.fn(); + getReportLink = vi.fn(async () => "https://exports.example/report.csv"); + reportStatus = vi.fn(async () => "completed"); + resolveBuild = vi.fn(async (source: unknown) => source); + }, + EXPORT_WAIT_MS: 45_000, + ExportPendingError: class extends Error {}, +})); +vi.mock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ + parseAccessibilityReportFromCSV: m.parse, + loadMainIssues: vi.fn(), + compareIssues: vi.fn(), +})); +vi.mock( + "../../src/tools/accessiblity-utils/fp-memory.js", + async (importOriginal) => ({ + ...(await importOriginal()), + memoryForReport: m.memoryForReport, + }), +); +vi.mock("../../src/lib/get-auth.js", () => ({ + getBrowserStackAuth: vi.fn().mockReturnValue("fake-user:fake-key"), +})); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); +vi.mock("../../src/lib/instrumentation.js", () => ({ trackMCP: vi.fn() })); +vi.mock("../../src/index.js", () => ({ + trackMCP: vi.fn(), + PackageJsonVersion: "0.0.0", +})); + +import addAccessibilityTools from "../../src/tools/accessibility.js"; + +function fetchIssues() { + let handler: (...args: any[]) => any = () => undefined; + const server: any = { + tool: vi.fn((...args: any[]) => { + if (args[0] === "fetchAccessibilityIssues") + handler = args[args.length - 1]; + }), + server: { getClientVersion: vi.fn().mockReturnValue({ version: "1.0" }) }, + }; + addAccessibilityTools(server, { + "browserstack-username": "fake-user", + "browserstack-access-key": "fake-key", + } as any); + return (args: Record) => + handler(args, { sendNotification: vi.fn(), _meta: {} }); +} + +const texts = (result: any): string[] => + result.content.map((c: { text: string }) => c.text); + +const PAGE = "https://www.shop.example/checkout"; +const issue = (issue_type: string, component: string) => ({ + issue_type, + component, + HTML_snippet: `<${component.replace("#", "")}>`, + severity: "serious", + affected_page: PAGE, +}); + +const REPORT = { + records: [ + issue("role-required", "acme-header-button#"), + issue("role-required", "acme-menu-trigger#"), + issue("image-alt", "img#"), + ], + rules: { + "role-required": { description: "Interactive elements need a role" }, + "image-alt": { description: "Images need alt text" }, + }, + summary: { + by_severity: { serious: 3 }, + by_rule: [ + { issue_type: "role-required", count: 2 }, + { issue_type: "image-alt", count: 1 }, + ], + needs_review: 0, + best_practice: 0, + }, + page_length: 3, + total_issues: 3, + next_page: null, + remaining: 0, + duplicate_issues: null, + unlinked_duplicates: 0, + hidden_issues: null, +}; + +const MEMORY = { + nextCursor: null, + rules: [ + { + ruleCode: "role-required", + docsUrl: + "https://www.browserstack.com/docs/accessibility/rules/a11y-engine/role-required/", + entryCount: 1, + entries: [ + { + id: "fp_41", + site: "www.shop.example", + global: false, + verdict: "false_positive", + scope: "component_family", + rootCause: + "Role is set inside each acme- custom element's shadow root; the host has none.", + pattern: { + tagPrefix: "acme-", + predicate: "acme- custom elements rendering a native control", + }, + userStatement: "Those Acme buttons work fine with a screen reader", + }, + ], + }, + ], +}; + +const RUN = { scanId: "7310042", scanRunId: "5520318" }; + +beforeEach(() => { + vi.clearAllMocks(); + m.parse.mockResolvedValue(structuredClone(REPORT)); +}); + +describe("fetchAccessibilityIssues with saved false positives", () => { + it("opens by saying to remove them before giving any list or count", async () => { + m.memoryForReport.mockResolvedValue(MEMORY); + + const lines = texts(await fetchIssues()(RUN)); + + expect(lines[0]).toMatch(/Saved false positives apply to this report/); + expect(lines[0]).toContain("2 of the 3 issues below"); + expect(lines[0]).toMatch(/Before you give the user any list, count/); + const firstCount = lines.findIndex((l) => l.startsWith("Retrieved")); + expect(firstCount).toBeGreaterThan(0); + expect(lines[firstCount]).toContain( + "Total: 3, before removing saved false positives", + ); + }); + + it("asks only for the rules of the issues it answers with", async () => { + m.memoryForReport.mockResolvedValue(MEMORY); + + await fetchIssues()(RUN); + + expect(m.memoryForReport.mock.calls[0][1]).toEqual([ + "role-required", + "image-alt", + ]); + }); + + it("marks the issues the entries may cover, and only those", async () => { + m.memoryForReport.mockResolvedValue(MEMORY); + + const block = texts(await fetchIssues()(RUN)).find((l) => + l.startsWith("Issues:"), + )!; + const issues = JSON.parse(block.split("\n")[2]).issues; + + expect(issues.map((i: any) => i.saved_fp_entries)).toEqual([ + ["fp_41"], + ["fp_41"], + undefined, + ]); + }); + + it("gives the memory before the issues and how to report what is left after them", async () => { + m.memoryForReport.mockResolvedValue(MEMORY); + + const lines = texts(await fetchIssues()(RUN)); + const memoryAt = lines.findIndex((l) => + l.startsWith("False-positive memory:"), + ); + const issuesAt = lines.findIndex((l) => l.startsWith("Issues:")); + const closingAt = lines.findIndex((l) => + l.includes("list only the issues left"), + ); + + expect(memoryAt).toBeGreaterThan(-1); + expect(memoryAt).toBeLessThan(issuesAt); + expect(closingAt).toBeGreaterThan(issuesAt); + }); + + it("answers as it always did when no saved entry covers these issues", async () => { + m.memoryForReport.mockResolvedValue({ + nextCursor: null, + rules: [ + { + ...MEMORY.rules[0], + entries: [ + { ...MEMORY.rules[0].entries[0], site: "elsewhere.example" }, + ], + }, + ], + }); + + const lines = texts(await fetchIssues()(RUN)); + + expect(lines[0]).not.toMatch(/Saved false positives/); + expect(lines.join("\n")).not.toContain("saved_fp_entries"); + expect(lines.join("\n")).toContain("Total: 3)"); + }); + + it("answers as it always did when the account has no memory", async () => { + m.memoryForReport.mockResolvedValue(undefined); + + const lines = texts(await fetchIssues()(RUN)); + + expect(lines.join("\n")).not.toMatch(/false positive/i); + expect(lines.join("\n")).toContain("Total: 3)"); + }); +}); diff --git a/tests/tools/capabilityWebAccessibility.test.ts b/tests/tools/capabilityWebAccessibility.test.ts index ef712b4a..b634688d 100644 --- a/tests/tools/capabilityWebAccessibility.test.ts +++ b/tests/tools/capabilityWebAccessibility.test.ts @@ -40,8 +40,11 @@ const queryNames = (name: string) => describe("the Web Accessibility capability index", () => { it("ships the six lists, as paged reads, each on an entity that names it", () => { - const names = web.capabilities.map((c: any) => c.name).sort(); - expect(names).toEqual([ + // The false-positive memory capabilities are tested in fpMemory.test.ts. + const lists = web.capabilities.filter((c: any) => + c.name.startsWith("list_"), + ); + expect(lists.map((c: any) => c.name).sort()).toEqual([ "list_assisted_test_reports", "list_automated_tests_builds", "list_automated_tests_projects", @@ -49,7 +52,7 @@ describe("the Web Accessibility capability index", () => { "list_website_scanner_scans", "list_workflow_analyzer_reports", ]); - for (const c of web.capabilities) { + for (const c of lists) { expect(c.mode).toBe("read"); expect(c.paginated).toBe(true); expect(c.max_page_size).toBeGreaterThan(0); @@ -69,7 +72,7 @@ describe("the Web Accessibility capability index", () => { expect(limit("list_assisted_test_reports").maximum).toBe(99); // Above 25 the page holds 25 scans and the next one starts past the ones in between. expect(limit("list_website_scanner_scans").maximum).toBe(25); - for (const c of web.capabilities) { + for (const c of web.capabilities.filter((c: any) => c.paginated)) { expect(c.max_page_size).toBe( c.query.find((q: any) => q.name === "limit").maximum, ); @@ -154,9 +157,15 @@ describe("the Web Accessibility capability index", () => { it.each([ ["list workflow analyzer reports", "list_workflow_analyzer_reports"], ["list website scanner scans", "list_website_scanner_scans"], - ["list the runs of a website scanner scan", "list_website_scanner_scan_runs"], + [ + "list the runs of a website scanner scan", + "list_website_scanner_scan_runs", + ], ["list automated tests projects", "list_automated_tests_projects"], - ["list the builds of an automated tests project", "list_automated_tests_builds"], + [ + "list the builds of an automated tests project", + "list_automated_tests_builds", + ], ["list assisted test reports", "list_assisted_test_reports"], ])("finds %j as %s", (query, expected) => { const { hits } = searchCapabilities(products, query, { diff --git a/tests/tools/fpMemory.test.ts b/tests/tools/fpMemory.test.ts new file mode 100644 index 00000000..26cdd696 --- /dev/null +++ b/tests/tools/fpMemory.test.ts @@ -0,0 +1,332 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// The false-positive memory: what users confirmed is not a real issue, read for the rules of +// the issues fetchAccessibilityIssues answers with, so the agent filters them without asking +// again. Saving, reading and revoking are web_accessibility capabilities. What has to hold: +// the memory never holds the issues back, it fits its budget oldest-first, it is quoted as +// untrusted, and the capabilities an answer names exist and can be invoked. + +vi.unmock("../../src/tools/accessiblity-utils/fp-memory.js"); + +const get = vi.hoisted(() => vi.fn()); +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { get } })); +vi.mock("../../src/logger.js", () => ({ + default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() }, +})); + +import { + FP_APPLY_NOTE, + FP_FILING_GUIDE, + READ_CAPABILITY, + SUBMIT_CAPABILITY, + memoryForReport, + applyMemory, + memoryPayload, + type FpEntry, + type FpMemory, +} from "../../src/tools/accessiblity-utils/fp-memory.js"; + +const AUTH = { username: "u", password: "k" }; +const ON_SITE = "https://www.shop.example/de/angebote"; + +function entry(id: number, extra: Partial = {}): FpEntry { + return { + id: `fp_${id}`, + site: "www.shop.example", + global: false, + verdict: "false_positive", + scope: "component_family", + rootCause: + "The role is set inside the custom element's shadow root; the host has no role attribute.", + pattern: { + tagPrefix: "acme-", + predicate: "acme- custom elements rendering a native control", + }, + counterExamples: ["div.to_sign_in"], + userStatement: "These Acme components work with VoiceOver", + pageUrl: "https://www.shop.example/de/angebote/halbtax", + component: "acme-header-button#", + htmlSnippet: "", + ...extra, + }; +} + +function memory(entries: FpEntry[], ruleCode = "role-required"): FpMemory { + return { + rules: [ + { + ruleCode, + docsUrl: `https://www.browserstack.com/docs/accessibility/rules/a11y-engine/${ruleCode}/`, + entryCount: entries.length, + entries, + }, + ], + nextCursor: null, + }; +} + +beforeEach(() => vi.clearAllMocks()); + +describe("memoryForReport", () => { + it("asks for the answer's rules only, with the user's credentials", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { success: true, data: memory([entry(1)]) }, + }); + + const result = await memoryForReport(AUTH, ["role-required", "image-alt"]); + + expect(result?.rules[0].entries[0].id).toBe("fp_1"); + const call = get.mock.calls[0][0]; + expect(call.url).toContain( + "/api/fp-memory/v1/memory?ruleCodes=role-required%2Cimage-alt&limit=200", + ); + expect(call.headers.Authorization).toBe( + "Basic " + Buffer.from("u:k").toString("base64"), + ); + expect(call.timeout).toBeGreaterThan(0); + }); + + it("reads only the page's site (and all-sites entries) when the page is known", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { success: true, data: memory([entry(1)]) }, + }); + + await memoryForReport(AUTH, ["role-required"], ON_SITE); + + expect(get.mock.calls[0][0].url).toContain( + `&site=${encodeURIComponent(ON_SITE)}`, + ); + }); + + it("answers without memory when the response has no rules, and reads a rule without entries as empty", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { success: true, data: {} }, + }); + expect(await memoryForReport(AUTH, ["role-required"])).toBeUndefined(); + + get.mockResolvedValue({ + ok: true, + status: 200, + data: { success: true, data: { rules: [{ ruleCode: "role-required" }] } }, + }); + const result = await memoryForReport(AUTH, ["role-required"]); + expect(result).toEqual({ + nextCursor: null, + rules: [{ ruleCode: "role-required", entries: [] }], + }); + }); + + it("does not ask when the answer names no rules", async () => { + expect(await memoryForReport(AUTH, [])).toBeUndefined(); + expect(get).not.toHaveBeenCalled(); + }); + + it("answers without memory when the account does not have it", async () => { + get.mockResolvedValue({ ok: false, status: 403, data: { success: false } }); + expect(await memoryForReport(AUTH, ["role-required"])).toBeUndefined(); + }); + + it("answers without memory when it cannot be read, rather than failing", async () => { + get.mockRejectedValue(new Error("timeout of 8000ms exceeded")); + expect(await memoryForReport(AUTH, ["role-required"])).toBeUndefined(); + get.mockResolvedValue({ ok: false, status: 500, data: {} }); + expect(await memoryForReport(AUTH, ["role-required"])).toBeUndefined(); + }); +}); + +const issue = (issue_type: string, affected_page = ON_SITE) => ({ + issue_type, + affected_page, + component: "acme-header-button#", +}); + +describe("applyMemory", () => { + it("leaves an answer as it was when the account has no memory", () => { + const records = [issue("role-required")]; + const view = applyMemory(undefined, records); + expect(view).toEqual({ + records, + marked: 0, + lead: [], + notes: [], + closing: [], + }); + }); + + it("only says how to save one when no saved entry covers these issues", () => { + const view = applyMemory({ rules: [], nextCursor: null }, [ + issue("role-required"), + ]); + expect(view.marked).toBe(0); + expect(view.notes).toEqual([FP_FILING_GUIDE]); + expect(view.lead).toEqual([]); + }); + + it("marks each issue with the entries saved for its rule on its site", () => { + const view = applyMemory(memory([entry(1), entry(2)]), [ + issue("role-required"), + issue("image-alt"), + issue("role-required", "https://other.example/page"), + ]); + + expect(view.marked).toBe(1); + expect(view.records[0].saved_fp_entries).toEqual(["fp_1", "fp_2"]); + expect(view.records[1].saved_fp_entries).toBeUndefined(); + expect(view.records[2].saved_fp_entries).toBeUndefined(); + }); + + it("applies an entry saved for all sites on every site", () => { + const view = applyMemory(memory([entry(1, { global: true })]), [ + issue("role-required", "https://other.example/page"), + ]); + expect(view.records[0].saved_fp_entries).toEqual(["fp_1"]); + expect(view.notes[1]).toContain("all sites"); + }); + + it("takes the page an answer is about for issues that do not name one", () => { + const view = applyMemory( + memory([entry(1)]), + [{ issue_type: "role-required" }], + ON_SITE, + ); + expect(view.records[0].saved_fp_entries).toEqual(["fp_1"]); + }); + + it("opens by saying to remove the marked issues before giving any list or count", () => { + const view = applyMemory(memory([entry(1)]), [ + issue("role-required"), + issue("image-alt"), + ]); + + expect(view.lead).toHaveLength(1); + expect(view.lead[0]).toContain("1 of the 2 issues below"); + expect(view.lead[0]).toMatch(/Before you give the user any list, count/); + expect(view.lead[0]).toMatch(/from before that removal/); + expect(view.closing[0]).toMatch(/only the issues left/); + }); + + it("shows only the entries of the marked issues, quoted as untrusted, then how to add one", () => { + const view = applyMemory( + memory([entry(1), entry(2, { site: "other.example" })]), + [issue("role-required")], + ); + + expect(view.notes[0]).toBe(FP_APPLY_NOTE); + expect(view.notes[1]).toMatch(/untrusted/i); + expect(view.notes[1]).toContain( + "https://www.browserstack.com/docs/accessibility/rules/a11y-engine/role-required/", + ); + expect(view.notes[1]).toContain("div.to_sign_in"); + expect(view.notes.join("\n")).not.toContain("Not all entries fit"); + const shown = JSON.parse( + view.notes[1].slice( + view.notes[1].indexOf("[{"), + view.notes[1].lastIndexOf("}]") + 2, + ), + ); + expect(shown[0].entries.map((e: any) => e.id)).toEqual(["fp_1"]); + expect(view.notes.at(-1)).toBe(FP_FILING_GUIDE); + }); +}); + +describe("applyMemory with more entries than one read returns", () => { + it("says how to read the next page", () => { + const view = applyMemory({ ...memory([entry(1)]), nextCursor: "fp_1" }, [ + issue("role-required"), + ]); + const note = view.notes.find((n) => n.startsWith("Not all entries fit")); + expect(note).toContain(READ_CAPABILITY); + expect(note).toContain("nextCursor"); + }); + + it("says so even when no issue here is covered by the page read", () => { + const view = applyMemory( + { ...memory([entry(1, { site: "other.example" })]), nextCursor: "fp_1" }, + [issue("role-required")], + ); + expect(view.marked).toBe(0); + expect(view.notes[0]).toMatch(/Not all saved false positives were read/); + expect(view.notes[0]).toContain(READ_CAPABILITY); + expect(view.notes.at(-1)).toBe(FP_FILING_GUIDE); + }); +}); + +describe("memoryPayload", () => { + it("leaves out the fullest rule's oldest entries first, and counts them", () => { + const many = Array.from({ length: 40 }, (_, i) => entry(100 - i)); + const { json, omitted } = memoryPayload(memory(many), 4_000); + + expect(json.length).toBeLessThanOrEqual(4_000); + const kept = JSON.parse(json)[0].entries.map((e: any) => e.id); + expect(kept[0]).toBe("fp_100"); + expect(omitted).toEqual([ + { rule: "role-required", count: 40 - kept.length }, + ]); + }); + + it("shortens long HTML snippets", () => { + const long = entry(1, { + htmlSnippet: "
" + "x".repeat(1000) + "
", + }); + const html = JSON.parse(memoryPayload(memory([long])).json)[0].entries[0] + .seed_issue.html; + expect(html.length).toBeLessThanOrEqual(201); + }); +}); + +describe("the capabilities an answer names", () => { + const index = JSON.parse( + readFileSync( + fileURLToPath( + new URL( + "../../capability/web_accessibility.capability-index.json", + import.meta.url, + ), + ), + "utf8", + ), + ).web_accessibility; + const byName = (name: string) => + index.capabilities.find((c: any) => c.name === name); + + it("exist in the Web Accessibility index", () => { + expect(byName(SUBMIT_CAPABILITY)).toBeDefined(); + expect(byName(READ_CAPABILITY)).toBeDefined(); + expect(FP_FILING_GUIDE).toContain(SUBMIT_CAPABILITY); + }); + + it("are never destructive, which invokeCapability refuses before binding", () => { + const fp = index.capabilities.filter( + (c: any) => c.entity === "false_positive_feedback", + ); + expect(fp.map((c: any) => [c.name, c.method, c.mode])).toEqual([ + ["submit_false_positive_feedback", "POST", "write"], + ["get_false_positive_memory", "GET", "read"], + ["revoke_false_positive_feedback", "POST", "write"], + ]); + }); + + it("require on submit what the API refuses without", () => { + const required = byName(SUBMIT_CAPABILITY) + .body.filter((b: any) => b.required) + .map((b: any) => b.name) + .sort(); + expect(required).toEqual([ + "pageUrl", + "pattern", + "rootCause", + "ruleCode", + "scope", + "userStatement", + "verdict", + ]); + }); +});