From ac13e08d69c607fdb6bf15346d4e72af268c484e Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 12:32:51 +0100 Subject: [PATCH 001/117] feat(observability-map): package scaffold and route scanner --- .../observability-map/package.json | 21 ++++ .../observability-map/src/scan.ts | 111 ++++++++++++++++++ .../observability-map/src/types.ts | 22 ++++ .../observability-map/test/scan.test.ts | 29 +++++ .../observability-map/tsconfig.build.json | 5 + .../observability-map/tsconfig.json | 17 +++ .../observability-map/vitest.config.ts | 5 + pnpm-lock.yaml | 13 ++ 8 files changed, 223 insertions(+) create mode 100644 internal-packages/observability-map/package.json create mode 100644 internal-packages/observability-map/src/scan.ts create mode 100644 internal-packages/observability-map/src/types.ts create mode 100644 internal-packages/observability-map/test/scan.test.ts create mode 100644 internal-packages/observability-map/tsconfig.build.json create mode 100644 internal-packages/observability-map/tsconfig.json create mode 100644 internal-packages/observability-map/vitest.config.ts diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json new file mode 100644 index 00000000000..3d6f695f67b --- /dev/null +++ b/internal-packages/observability-map/package.json @@ -0,0 +1,21 @@ +{ + "name": "@internal/observability-map", + "private": true, + "version": "0.0.1", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "typescript": "catalog:" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "rimraf": "6.0.1" + }, + "scripts": { + "clean": "rimraf dist", + "typecheck": "tsc --noEmit", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest" + } +} diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts new file mode 100644 index 00000000000..de4203e885d --- /dev/null +++ b/internal-packages/observability-map/src/scan.ts @@ -0,0 +1,111 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { EntryPoint } from "./types.js"; + +function calleeName(expr: ts.Expression): string | null { + if (ts.isIdentifier(expr)) return expr.text; + if (ts.isPropertyAccessExpression(expr)) return expr.name.text; + return null; +} + +export function scanFile(fileName: string, source: string): EntryPoint | null { + const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); + + const ep: EntryPoint = { + fileName, + source, + hasLoader: false, + hasAction: false, + loaderInitializerCallee: null, + actionInitializerCallee: null, + importedNames: [], + calleeNames: [], + hasTryCatch: false, + statementCount: 0, + }; + + const isExported = (n: ts.Node) => + ts.canHaveModifiers(n) && + ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; + + const visit = (node: ts.Node) => { + if (ts.isImportDeclaration(node) && node.importClause) { + const bindings = node.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) ep.importedNames.push(el.name.text); + } + if (node.importClause.name) ep.importedNames.push(node.importClause.name.text); + } + + if (ts.isVariableStatement(node) && isExported(node)) { + for (const decl of node.declarationList.declarations) { + const name = decl.name.getText(sf); + if (name !== "loader" && name !== "action") continue; + if (name === "loader") ep.hasLoader = true; + if (name === "action") ep.hasAction = true; + let init = decl.initializer; + if (init && ts.isPropertyAccessExpression(init)) init = init.expression; + if (init && ts.isCallExpression(init)) { + const cn = calleeName(init.expression); + if (name === "loader") ep.loaderInitializerCallee = cn; + else ep.actionInitializerCallee = cn; + } + } + } + + if (ts.isFunctionDeclaration(node) && node.name && isExported(node)) { + if (node.name.text === "loader") ep.hasLoader = true; + if (node.name.text === "action") ep.hasAction = true; + if (node.body) ep.statementCount += node.body.statements.length; + } + + if (ts.isTryStatement(node)) ep.hasTryCatch = true; + + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) ep.calleeNames.push(cn); + } + + ts.forEachChild(node, visit); + }; + + visit(sf); + + if (!ep.hasLoader && !ep.hasAction) return null; + if (ep.statementCount === 0) { + ep.statementCount = countArrowBodyStatements(sf); + } + return ep; +} + +function countArrowBodyStatements(sf: ts.SourceFile): number { + let count = 0; + const visit = (n: ts.Node) => { + if ((ts.isArrowFunction(n) || ts.isFunctionExpression(n)) && n.body && ts.isBlock(n.body)) { + count += n.body.statements.length; + } + ts.forEachChild(n, visit); + }; + visit(sf); + return count; +} + +export function scanDirectory(dir: string): { + entryPoints: EntryPoint[]; + parseFailures: string[]; +} { + const entryPoints: EntryPoint[] = []; + const parseFailures: string[] = []; + const files = readdirSync(dir).filter((f) => /\.(ts|tsx)$/.test(f) && !f.endsWith(".test.ts")); + + for (const fileName of files) { + try { + const ep = scanFile(fileName, readFileSync(join(dir, fileName), "utf8")); + if (ep) entryPoints.push(ep); + } catch { + parseFailures.push(fileName); + } + } + return { entryPoints, parseFailures }; +} diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts new file mode 100644 index 00000000000..7a331707e21 --- /dev/null +++ b/internal-packages/observability-map/src/types.ts @@ -0,0 +1,22 @@ +export type CheckStatus = "pass" | "fail" | "not-applicable"; + +export type CheckResult = { + id: string; + status: CheckStatus; + detail?: string; +}; + +export type EntryPoint = { + fileName: string; + source: string; + hasLoader: boolean; + hasAction: boolean; + /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + importedNames: string[]; + calleeNames: string[]; + hasTryCatch: boolean; + /** Statement count across loader/action bodies, used by the triviality rule. */ + statementCount: number; +}; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts new file mode 100644 index 00000000000..27f233311cf --- /dev/null +++ b/internal-packages/observability-map/test/scan.test.ts @@ -0,0 +1,29 @@ +import { scanFile } from "../src/scan.js"; + +const LOADER = ` +import { json } from "@remix-run/server-runtime"; +export async function loader() { return json({}); } +`; + +const COMPONENT_ONLY = ` +export default function Page() { return null; } +`; + +describe("scanFile", () => { + it("detects an exported loader as a server entry point", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(false); + }); + + it("ignores a route that only exports a component", () => { + expect(scanFile("_app.things.tsx", COMPONENT_ONLY)).toBeNull(); + }); + + it("detects a loader assigned from a call expression", () => { + const ep = scanFile("api.v1.x.ts", `export const loader = createLoaderApiRoute({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); +}); diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json new file mode 100644 index 00000000000..56ade258e18 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "noEmit": false, "outDir": "dist", "declaration": true }, + "exclude": ["node_modules", "dist", "test"] +} diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json new file mode 100644 index 00000000000..61669556f76 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019"], + "module": "ESNext", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "types": ["vitest/globals", "node"], + "customConditions": ["@triggerdotdev/source"] + }, + "exclude": ["node_modules", "dist"] +} diff --git a/internal-packages/observability-map/vitest.config.ts b/internal-packages/observability-map/vitest.config.ts new file mode 100644 index 00000000000..c1680ce67f9 --- /dev/null +++ b/internal-packages/observability-map/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { include: ["**/*.test.ts"], globals: true, isolate: true, testTimeout: 10_000 }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f37e3250982..b2b0c4a17c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1139,6 +1139,19 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/observability-map: + dependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 + devDependencies: + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + internal-packages/otlp-importer: dependencies: long: From 4fd2020de5a61ec7914d3350d6570d3b7f5f2620 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:15:06 +0100 Subject: [PATCH 002/117] fix(observability-map): find every route entry point and scope body analysis The scanner missed 30% of server entry points in apps/webapp/app/routes. It found 299 of 427; it now finds all 427 with 0 parse failures. - detect named export clauses (export { loader }, export { h as loader }), resolving a local binding back to its declaration for the builder callee - recurse one level into flat-route directories and key entry points by a path relative to the scan root, so route.tsx files stay distinct - count statements for the loader/action bodies only, recursing through try, if, loop and switch blocks so a try-wrapped body reports its real size - scope hasTryCatch and calleeNames to the entry-point bodies, leaving importedNames file-wide - resolve the initializer callee to the root of a call chain - throw on parse diagnostics so parseFailures can actually fire - scan .test.ts route files and exclude .d.ts instead --- .../observability-map/src/scan.ts | 413 +++++++++++++++--- .../observability-map/src/types.ts | 3 + .../observability-map/test/scan.test.ts | 329 +++++++++++++- 3 files changed, 686 insertions(+), 59 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index de4203e885d..cb506ce2cd3 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -3,92 +3,376 @@ import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import type { EntryPoint } from "./types.js"; +/** Thrown by `scanFile` when the source does not parse cleanly. */ +export class ParseFailureError extends Error { + constructor( + readonly fileName: string, + readonly diagnostic: string + ) { + super(`${fileName}: ${diagnostic}`); + this.name = "ParseFailureError"; + } +} + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +/** Strip wrappers that do not change which expression is really being referred to. */ +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +/** + * Root callee of a call, unwrapping chains: `createLoaderApiRoute({}).withCors()` + * resolves to `createLoaderApiRoute`, not `withCors`. + */ +function rootCalleeName(call: ts.CallExpression): string | null { + let current: ts.Expression = unwrap(call.expression); + for (;;) { + if (ts.isIdentifier(current)) return current.text; + if ( + ts.isCallExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) + ) { + current = unwrap(current.expression); + continue; + } + return null; + } +} + +/** Callee as recorded in `calleeNames`: the identifier, or the property for a member call. */ function calleeName(expr: ts.Expression): string | null { - if (ts.isIdentifier(expr)) return expr.text; - if (ts.isPropertyAccessExpression(expr)) return expr.name.text; + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (ts.isPropertyAccessExpression(target)) return target.name.text; return null; } +/** + * Functions on a `handler` property of an object argument, e.g. `createSSELoader({ handler })` and + * the per-method `{ POST: { handler } }` map. Only `handler` counts: the surrounding config also + * holds lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property) || !property.name) continue; + const value = unwrap(property.initializer); + if (ts.isObjectLiteralExpression(value)) { + collectNamedHandlers(value, out); + continue; + } + const name = + ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) + ? property.name.text + : null; + if (name === "handler" && isEntryFunction(value)) out.push(value); + } +} + +/** Every function passed as an argument anywhere in a call chain, e.g. the builder's handler. */ +function collectHandlerFunctions(expr: ts.Expression, out: EntryFunction[]): void { + const target = unwrap(expr); + if (ts.isCallExpression(target)) { + for (const arg of target.arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } + collectHandlerFunctions(target.expression, out); + return; + } + if (ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) { + collectHandlerFunctions(target.expression, out); + } +} + +type Initializer = { callee: string | null; functions: EntryFunction[] }; + +const NO_INITIALIZER: Initializer = { callee: null, functions: [] }; + +/** Top-level `function x` / `const x = ...` declarations, keyed by binding name. */ +type LocalDeclarations = Map; + +function analyzeInitializer( + expr: ts.Expression | undefined, + locals: LocalDeclarations, + seen: Set +): Initializer { + if (!expr) return NO_INITIALIZER; + const target = unwrap(expr); + + if (isEntryFunction(target)) return { callee: null, functions: [target] }; + + if (ts.isCallExpression(target)) { + const functions: EntryFunction[] = []; + collectHandlerFunctions(target, functions); + return { callee: rootCalleeName(target), functions }; + } + + // `export const action = route.action` where `const route = createActionApiRoute(...)`, and the + // plain alias `export const loader = h`. Resolve back to the declaration the name came from. + if (ts.isIdentifier(target)) return resolveLocal(target.text, locals, seen); + if (ts.isPropertyAccessExpression(target)) { + const root = unwrap(target.expression); + if (ts.isIdentifier(root)) return resolveLocal(root.text, locals, seen); + } + + return NO_INITIALIZER; +} + +function resolveLocal(name: string, locals: LocalDeclarations, seen: Set): Initializer { + if (seen.has(name)) return NO_INITIALIZER; + seen.add(name); + const local = locals.get(name); + if (!local) return NO_INITIALIZER; + if (ts.isFunctionDeclaration(local)) return { callee: null, functions: [local] }; + return analyzeInitializer(local, locals, seen); +} + +/** + * Statements in a statement, counting through block-bearing statements so a body wrapped in a + * single `try` reports its real size. Does not descend into nested function bodies. + */ +function countStatement(statement: ts.Statement): number { + if (ts.isBlock(statement)) { + return countStatements(statement.statements); + } + + let count = 1; + + if (ts.isTryStatement(statement)) { + count += countStatements(statement.tryBlock.statements); + if (statement.catchClause) count += countStatements(statement.catchClause.block.statements); + if (statement.finallyBlock) count += countStatements(statement.finallyBlock.statements); + return count; + } + + if (ts.isIfStatement(statement)) { + count += countStatement(statement.thenStatement); + if (statement.elseStatement) count += countStatement(statement.elseStatement); + return count; + } + + if ( + ts.isForStatement(statement) || + ts.isForInStatement(statement) || + ts.isForOfStatement(statement) || + ts.isWhileStatement(statement) || + ts.isDoStatement(statement) || + ts.isLabeledStatement(statement) || + ts.isWithStatement(statement) + ) { + count += countStatement(statement.statement); + return count; + } + + if (ts.isSwitchStatement(statement)) { + for (const clause of statement.caseBlock.clauses) { + count += countStatements(clause.statements); + } + return count; + } + + return count; +} + +function countStatements(statements: ts.NodeArray): number { + let count = 0; + for (const statement of statements) count += countStatement(statement); + return count; +} + +function countFunctionStatements(fn: EntryFunction): number { + if (!fn.body) return 0; + // A concise arrow body (`() => json({})`) is one expression, so one statement. + if (!ts.isBlock(fn.body)) return 1; + return countStatements(fn.body.statements); +} + +type EntryTarget = { + hasLoader: boolean; + hasAction: boolean; + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + functions: Set; +}; + +/** + * Top-level `function x` / `const x = ...` declarations by binding name, so a named export clause + * (`export { action }`) can be resolved back to the initializer it came from. + */ +function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name) { + locals.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } + } + } + + return locals; +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); - const ep: EntryPoint = { - fileName, - source, + // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are + // the only signal that a file did not parse. + const parseDiagnostics = (sf as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] }) + .parseDiagnostics; + if (parseDiagnostics && parseDiagnostics.length > 0) { + const first = parseDiagnostics[0]!; + throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); + } + + const importedNames: string[] = []; + for (const statement of sf.statements) { + if (!ts.isImportDeclaration(statement) || !statement.importClause) continue; + const bindings = statement.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) importedNames.push(el.name.text); + } + if (statement.importClause.name) importedNames.push(statement.importClause.name.text); + } + + const target: EntryTarget = { hasLoader: false, hasAction: false, loaderInitializerCallee: null, actionInitializerCallee: null, - importedNames: [], - calleeNames: [], - hasTryCatch: false, - statementCount: 0, + functions: new Set(), + }; + + const record = (name: string, initializer: Initializer) => { + if (name === "loader") { + target.hasLoader = true; + target.loaderInitializerCallee ??= initializer.callee; + } else { + target.hasAction = true; + target.actionInitializerCallee ??= initializer.callee; + } + for (const fn of initializer.functions) target.functions.add(fn); }; const isExported = (n: ts.Node) => ts.canHaveModifiers(n) && ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; - const visit = (node: ts.Node) => { - if (ts.isImportDeclaration(node) && node.importClause) { - const bindings = node.importClause.namedBindings; - if (bindings && ts.isNamedImports(bindings)) { - for (const el of bindings.elements) ep.importedNames.push(el.name.text); + const locals = collectLocalDeclarations(sf); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + const name = statement.name.text; + if (name === "loader" || name === "action") { + record(name, { callee: null, functions: [statement] }); } - if (node.importClause.name) ep.importedNames.push(node.importClause.name.text); + continue; } - if (ts.isVariableStatement(node) && isExported(node)) { - for (const decl of node.declarationList.declarations) { - const name = decl.name.getText(sf); - if (name !== "loader" && name !== "action") continue; - if (name === "loader") ep.hasLoader = true; - if (name === "action") ep.hasAction = true; - let init = decl.initializer; - if (init && ts.isPropertyAccessExpression(init)) init = init.expression; - if (init && ts.isCallExpression(init)) { - const cn = calleeName(init.expression); - if (name === "loader") ep.loaderInitializerCallee = cn; - else ep.actionInitializerCallee = cn; + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (!ts.isIdentifier(decl.name)) continue; + const name = decl.name.text; + if (name === "loader" || name === "action") { + record(name, analyzeInitializer(decl.initializer, locals, new Set())); } } + continue; } - if (ts.isFunctionDeclaration(node) && node.name && isExported(node)) { - if (node.name.text === "loader") ep.hasLoader = true; - if (node.name.text === "action") ep.hasAction = true; - if (node.body) ep.statementCount += node.body.statements.length; - } + if (ts.isExportDeclaration(statement) && statement.exportClause) { + // `export * from "./x"` has no clause and reaches nothing here; `export * as ns from "./x"` + // is a namespace clause, which cannot name a loader or action either. + if (!ts.isNamedExports(statement.exportClause)) continue; - if (ts.isTryStatement(node)) ep.hasTryCatch = true; - - if (ts.isCallExpression(node)) { - const cn = calleeName(node.expression); - if (cn) ep.calleeNames.push(cn); + for (const element of statement.exportClause.elements) { + const exportedName = element.name.text; + if (exportedName !== "loader" && exportedName !== "action") continue; + // A re-export (`export { loader } from "./x"`) has no local binding to resolve. + if (statement.moduleSpecifier) { + record(exportedName, NO_INITIALIZER); + continue; + } + const localName = element.propertyName?.text ?? exportedName; + record(exportedName, resolveLocal(localName, locals, new Set())); + } } + } - ts.forEachChild(node, visit); - }; + if (!target.hasLoader && !target.hasAction) return null; + + let statementCount = 0; + let hasTryCatch = false; + const calleeNames: string[] = []; - visit(sf); + for (const fn of target.functions) { + statementCount += countFunctionStatements(fn); - if (!ep.hasLoader && !ep.hasAction) return null; - if (ep.statementCount === 0) { - ep.statementCount = countArrowBodyStatements(sf); + if (!fn.body) continue; + const visit = (node: ts.Node) => { + if (ts.isTryStatement(node)) hasTryCatch = true; + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) calleeNames.push(cn); + } + ts.forEachChild(node, visit); + }; + visit(fn.body); } - return ep; -} -function countArrowBodyStatements(sf: ts.SourceFile): number { - let count = 0; - const visit = (n: ts.Node) => { - if ((ts.isArrowFunction(n) || ts.isFunctionExpression(n)) && n.body && ts.isBlock(n.body)) { - count += n.body.statements.length; - } - ts.forEachChild(n, visit); + return { + fileName, + source, + hasLoader: target.hasLoader, + hasAction: target.hasAction, + loaderInitializerCallee: target.loaderInitializerCallee, + actionInitializerCallee: target.actionInitializerCallee, + importedNames, + calleeNames, + hasTryCatch, + statementCount, }; - visit(sf); - return count; +} + +const SOURCE_FILE = /\.tsx?$/; + +function isScannableFile(fileName: string): boolean { + return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); } export function scanDirectory(dir: string): { @@ -97,15 +381,28 @@ export function scanDirectory(dir: string): { } { const entryPoints: EntryPoint[] = []; const parseFailures: string[] = []; - const files = readdirSync(dir).filter((f) => /\.(ts|tsx)$/.test(f) && !f.endsWith(".test.ts")); - for (const fileName of files) { + const scan = (absolutePath: string, relativeName: string) => { try { - const ep = scanFile(fileName, readFileSync(join(dir, fileName), "utf8")); + const ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); if (ep) entryPoints.push(ep); } catch { - parseFailures.push(fileName); + parseFailures.push(relativeName); + } + }; + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + scan(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); + } + continue; } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + scan(join(dir, entry.name), entry.name); } + return { entryPoints, parseFailures }; } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 7a331707e21..fcf037305fe 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -14,8 +14,11 @@ export type EntryPoint = { /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ loaderInitializerCallee: string | null; actionInitializerCallee: string | null; + /** Named and default imports, file-wide. */ importedNames: string[]; + /** Names of functions called inside the loader/action bodies. */ calleeNames: string[]; + /** Whether a `try` appears inside the loader/action bodies. */ hasTryCatch: boolean; /** Statement count across loader/action bodies, used by the triviality rule. */ statementCount: number; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 27f233311cf..758f2c35c71 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,4 +1,7 @@ -import { scanFile } from "../src/scan.js"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` import { json } from "@remix-run/server-runtime"; @@ -27,3 +30,327 @@ describe("scanFile", () => { expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); }); }); + +describe("scanFile: named export clauses", () => { + it("detects `export { loader }` and resolves the builder callee from the local declaration", () => { + const ep = scanFile( + "api.v1.query.ts", + ` + const { loader } = createLoaderApiRoute({ findResource: async () => 1 }, async () => { + const a = 1; + return json({ a }); + }); + export { loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(2); + }); + + it("detects an aliased named export `export { h as loader }`", () => { + const ep = scanFile( + "api.v1.aliased.ts", + ` + async function h() { return json({}); } + export { h as loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.statementCount).toBe(1); + }); + + it("reports both `export const loader` and a separate `export { action }`", () => { + const ep = scanFile( + "api.v1.both.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})); + const { action } = createActionApiRoute({}, async ({ body }) => { + const x = body.x; + return json({ x }); + }); + export { action }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("resolves an export assigned from a property of a local builder result", () => { + const ep = scanFile( + "api.v1.errors.$errorId.ignore.ts", + ` + const route = createActionApiRoute({ method: "POST" }, async ({ body }) => { + const a = 1; + const b = 2; + return json({ a, b }); + }); + export const action = route.action; + export const loader = route.loader; + ` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + // Both exports share one handler, so its statements are counted once. + expect(ep!.statementCount).toBe(3); + }); + + it("counts a `handler` property body but not the surrounding builder config lambdas", () => { + const ep = scanFile( + "engine.v1.dev.presence.ts", + ` + export const loader = createSSELoader({ + timeout: 1000, + findResource: async (params) => { + const a = 1; + const b = 2; + const c = 3; + return lookup(a, b, c); + }, + handler: async ({ request }) => { + const auth = await authenticate(request); + return stream(auth); + }, + }); + ` + ); + expect(ep!.statementCount).toBe(2); + expect(ep!.calleeNames).not.toContain("lookup"); + }); + + it("detects an action-only route", () => { + const ep = scanFile( + "api.v1.action-only.ts", + `export async function action() { return json({}); }` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(false); + }); + + it("does not crash on a re-export or a star export", () => { + expect(() => scanFile("re-export.ts", `export { loader } from "./other";`)).not.toThrow(); + expect(() => scanFile("star.ts", `export * from "./other";`)).not.toThrow(); + const ep = scanFile("re-export.ts", `export { loader } from "./other";`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: statement counting", () => { + it("counts only the loader's statements, not a fat exported component's", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + const a = 1; + const b = 2; + const c = 3; + const d = 4; + return null; + } + export function ErrorBoundary() { + const e = 1; + return null; + } + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("counts through a try/catch wrapper rather than reporting 1", () => { + const ep = scanFile( + "otel.v1.traces.ts", + ` + export async function action({ request }) { + try { + const body = await request.arrayBuffer(); + const result = await process(body); + return json(result); + } catch (e) { + logger.error(e); + return json({ error: true }, { status: 500 }); + } + } + ` + ); + expect(ep!.statementCount).toBeGreaterThan(4); + }); + + it("counts statements nested in if/for/while/switch blocks", () => { + const ep = scanFile( + "nested.ts", + ` + export async function loader() { + if (a) { + const x = 1; + doThing(x); + } + for (const i of list) { + use(i); + } + return json({}); + } + ` + ); + // if (1) + 2 nested + for (1) + 1 nested + return (1) + expect(ep!.statementCount).toBe(6); + }); +}); + +describe("scanFile: entry-point scoping", () => { + it("reports hasTryCatch false when the only try is in a non-entry-point export", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + report(e); + } + return null; + } + ` + ); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("reports hasTryCatch true when the try is inside the loader", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + try { + return json(await load()); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + }); + + it("excludes callees invoked outside the loader/action body", () => { + const ep = scanFile( + "route.tsx", + ` + const schema = z.object({}); + export async function loader() { + const data = await fetchThings(); + return json(data); + } + export default function Page() { + useFancyHook(); + return null; + } + ` + ); + expect(ep!.calleeNames).toContain("fetchThings"); + expect(ep!.calleeNames).toContain("json"); + expect(ep!.calleeNames).not.toContain("useFancyHook"); + expect(ep!.calleeNames).not.toContain("object"); + }); +}); + +describe("scanFile: callee resolution", () => { + it("records the root callee of a chained builder call", () => { + const ep = scanFile( + "api.v1.cors.ts", + `export const loader = createLoaderApiRoute({}).withCors();` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); + + it("leaves the callee null for a shape that cannot be named", () => { + const ep = scanFile("api.v1.anon.ts", `export const loader = async () => json({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: parse failures", () => { + it("throws on a malformed source rather than returning a clean entry point", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { const a = ; return json(`) + ).toThrow(); + }); + + it("does not throw on a well-formed tsx route", () => { + expect(() => + scanFile( + "route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi
; }` + ) + ).not.toThrow(); + }); +}); + +describe("scanDirectory", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "obs-map-scan-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("recurses into route directories and keeps file names distinct", () => { + writeFileSync(join(dir, "a.ts"), `export async function loader() { return json({}); }`); + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + mkdirSync(join(dir, "other.route")); + writeFileSync( + join(dir, "other.route", "route.tsx"), + `export async function action() { return json({}); }` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + const names = entryPoints.map((ep) => ep.fileName).sort(); + + expect(parseFailures).toEqual([]); + expect(names).toEqual(["a.ts", "nested.route/route.tsx", "other.route/route.tsx"]); + }); + + it("records a malformed file as a parse failure instead of an entry point", () => { + writeFileSync( + join(dir, "broken.ts"), + `export async function loader() { const a = ; return json(` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + + expect(entryPoints).toEqual([]); + expect(parseFailures).toEqual(["broken.ts"]); + }); + + it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { + writeFileSync( + join(dir, "projects.v3.$projectRef.test.ts"), + `export async function loader() { return json({}); }` + ); + writeFileSync(join(dir, "types.d.ts"), `export declare const x: number;`); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); + }); +}); From fa01e3e9adccaeaa760ae55610060adaf0115070 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:22:26 +0100 Subject: [PATCH 003/117] feat(observability-map): remix route adapter --- .../observability-map/src/adapters/remix.ts | 44 ++++++++++++++ .../observability-map/test/remix.test.ts | 57 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 internal-packages/observability-map/src/adapters/remix.ts create mode 100644 internal-packages/observability-map/test/remix.test.ts diff --git a/internal-packages/observability-map/src/adapters/remix.ts b/internal-packages/observability-map/src/adapters/remix.ts new file mode 100644 index 00000000000..7815fa3e6a4 --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.ts @@ -0,0 +1,44 @@ +export type Family = + | "api.v1" + | "api.other" + | "webhooks" + | "admin" + | "resources" + | "dashboard" + | "ingest" + | "other"; + +/** + * The name that carries routing meaning for a given `fileName`. + * + * Flat routes (`api.v1.runs.$runId.ts`) are already that name. Directory routes + * (`_app.orgs.$slug/route.tsx`) hold their module in a fixed `route.ts`/`route.tsx` file, so the + * directory segment before the slash is the meaningful name and `route.tsx` itself is not a path + * segment. + */ +function routeName(fileName: string): string { + const slashIndex = fileName.indexOf("/"); + return slashIndex === -1 ? fileName : fileName.slice(0, slashIndex); +} + +export function familyOf(fileName: string): Family { + const name = routeName(fileName); + // admin is checked first: admin.api.v1.* is an admin route, not an api.v1 one. + if (name.startsWith("admin.")) return "admin"; + if (name.startsWith("api.v1.")) return "api.v1"; + if (name.startsWith("api.")) return "api.other"; + if (name.startsWith("webhooks.")) return "webhooks"; + if (name.startsWith("resources.")) return "resources"; + if (name.startsWith("_app.")) return "dashboard"; + if (name.startsWith("otel.") || name.startsWith("engine.")) return "ingest"; + return "other"; +} + +export function routePathOf(fileName: string): string { + const withoutExt = routeName(fileName).replace(/\.(ts|tsx)$/, ""); + const segments = withoutExt + .split(".") + .filter((s) => s.length > 0) + .map((s) => (s.startsWith("$") ? `:${s.slice(1)}` : s)); + return `/${segments.join("/")}`; +} diff --git a/internal-packages/observability-map/test/remix.test.ts b/internal-packages/observability-map/test/remix.test.ts new file mode 100644 index 00000000000..4af418ef846 --- /dev/null +++ b/internal-packages/observability-map/test/remix.test.ts @@ -0,0 +1,57 @@ +import { familyOf, routePathOf } from "../src/adapters/remix.js"; + +describe("familyOf", () => { + it("classifies each family from the flat-route filename", () => { + expect(familyOf("api.v1.runs.$runId.ts")).toBe("api.v1"); + expect(familyOf("api.something.ts")).toBe("api.other"); + expect(familyOf("admin.api.v1.gc.ts")).toBe("admin"); + expect(familyOf("resources.queues.ts")).toBe("resources"); + expect(familyOf("_app.orgs.$slug.ts")).toBe("dashboard"); + expect(familyOf("otel.v1.logs.ts")).toBe("ingest"); + expect(familyOf("@.ts")).toBe("other"); + }); + + it("prefers admin over api.v1 for admin-prefixed api routes", () => { + expect(familyOf("admin.api.v1.environments.$id.ts")).toBe("admin"); + }); +}); + +describe("routePathOf", () => { + it("turns a flat-route filename into a path", () => { + expect(routePathOf("api.v1.runs.$runId.ts")).toBe("/api/v1/runs/:runId"); + }); + + it("strips a trailing method suffix", () => { + expect(routePathOf("api.v1.runs.ts")).toBe("/api/v1/runs"); + }); +}); + +// Directory routes: the scanner recurses into Remix directory routes, so `fileName` can be a +// relative path like `_app.orgs.$organizationSlug.projects.$projectParam/route.tsx` rather than a +// flat dot-separated name. The directory name (not `route.tsx`) carries the meaning. +describe("familyOf: directory routes", () => { + it("classifies a directory route by its directory name, not the flat rules", () => { + expect(familyOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx")).toBe( + "dashboard" + ); + }); + + it("classifies each family the same whether the route is flat or a directory", () => { + expect(familyOf("api.v1.runs.$runId/route.tsx")).toBe("api.v1"); + expect(familyOf("admin.api.v1.environments.$id/route.tsx")).toBe("admin"); + expect(familyOf("resources.queues/route.tsx")).toBe("resources"); + expect(familyOf("storybook.callout/route.tsx")).toBe("other"); + }); +}); + +describe("routePathOf: directory routes", () => { + it("does not emit a literal 'route' segment for a directory route", () => { + const path = routePathOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx"); + expect(path).not.toContain("route"); + expect(path).toBe("/_app/orgs/:organizationSlug/projects/:projectParam"); + }); + + it("produces the same path shape for a directory route as its flat equivalent", () => { + expect(routePathOf("api.v1.runs.$runId/route.tsx")).toBe(routePathOf("api.v1.runs.$runId.ts")); + }); +}); From 944f4cb3ac97aeddd33fdf13ad5f9223cc0b6009 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:30:07 +0100 Subject: [PATCH 004/117] feat(observability-map): import and callee based sensitivity classification --- .../observability-map/src/sensitivity.ts | 55 ++++++++++ .../test/sensitivity.test.ts | 101 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 internal-packages/observability-map/src/sensitivity.ts create mode 100644 internal-packages/observability-map/test/sensitivity.test.ts diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts new file mode 100644 index 00000000000..40f348afac7 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -0,0 +1,55 @@ +import type { EntryPoint } from "./types.js"; +import { routePathOf } from "./adapters/remix.js"; + +const SENSITIVE_SYMBOLS = [ + "clearImpersonation", + "setImpersonation", + "requireAdminApiRequest", + "createPersonalAccessToken", + "regenerateApiKey", + "createJWT", + "signJWT", + "updateEnvVars", + "createAuthorizationCode", +]; + +// Whole path segments only, so "authorship" does not match "auth". +const SENSITIVE_SEGMENTS = [ + "auth", + "jwt", + "token", + "tokens", + "envvars", + "billing", + "payment", + "invoices", + "secrets", + "impersonate", + "authorization-code", + "regenerate-api-key", +]; + +export type Sensitivity = { sensitive: boolean; reasons: string[] }; + +export function classifySensitivity(ep: EntryPoint): Sensitivity { + const reasons: string[] = []; + + // importedNames is file-wide; calleeNames is scoped to the loader/action body. A sensitive + // symbol called only at module scope is caught here only if it is also imported. + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + for (const s of SENSITIVE_SYMBOLS) { + if (symbols.has(s)) reasons.push(`calls ${s}`); + } + + // `routePathOf` turns both flat routes (`api.v1.envvars.ts`) and directory routes + // (`billing/route.tsx`) into real `/`-separated path segments, so this matches whole segments in + // either shape rather than splitting the raw fileName on ".". + const segments = routePathOf(ep.fileName) + .split("/") + .filter((s) => s.length > 0); + for (const seg of segments) { + if (SENSITIVE_SEGMENTS.includes(seg)) reasons.push(`path segment "${seg}"`); + } + + return { sensitive: reasons.length > 0, reasons }; +} diff --git a/internal-packages/observability-map/test/sensitivity.test.ts b/internal-packages/observability-map/test/sensitivity.test.ts new file mode 100644 index 00000000000..9a5c12e5f6a --- /dev/null +++ b/internal-packages/observability-map/test/sensitivity.test.ts @@ -0,0 +1,101 @@ +import { classifySensitivity } from "../src/sensitivity.js"; +import { scanFile } from "../src/scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("classifySensitivity", () => { + it("flags a route whose filename says nothing but which calls a sensitive helper", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("clearImpersonation"))).toBe(true); + }); + + it("flags on the path when the filename is explicit", () => { + const e = ep("api.v1.projects.$ref.envvars.ts", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag an ordinary read route", () => { + const e = ep( + "api.v1.timezones.ts", + `import { json } from "@remix-run/server-runtime"; + export async function loader() { return json({ timezones: [] }); }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); + + it("does not flag on a substring that merely contains a sensitive word", () => { + const e = ep( + "api.v1.authorship.ts", + `export async function loader() { return { author: "x" }; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// Directory routes: `fileName` can be `dirName/route.tsx` rather than a flat dotted name. A naive +// `fileName.split(".")` treats "billing/route" as one non-matching segment because the slash never +// gets split, so it misses the directory name entirely. The classifier must derive real path +// segments (via the same route-path logic the remix adapter uses) so both shapes work alike. +describe("classifySensitivity: directory routes", () => { + it("flags a single-segment directory route by its directory name", () => { + const e = ep("billing/route.tsx", `export async function loader() { return 1; }`); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("billing"))).toBe(true); + }); + + it("flags a multi-segment directory route via a segment among dynamic params", () => { + const e = ep( + "_app.orgs.$slug.billing/route.tsx", + `export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a directory route on a substring that merely contains a sensitive word", () => { + const e = ep("api.v1.authorship/route.tsx", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// calleeNames is scoped to the loader/action body, unlike importedNames which is file-wide. A +// sensitive symbol called only at module scope is invisible to calleeNames, so it is only caught +// when it also shows up as an import. +describe("classifySensitivity: calleeNames is body-scoped, importedNames is file-wide", () => { + it("flags a sensitive symbol invoked only at module scope, via the import rather than the callee", () => { + const e = ep( + "api.v1.setup.ts", + `import { setImpersonation } from "~/models/admin.server"; + setImpersonation(globalThis, "seed"); + export async function loader() { return 1; }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("setImpersonation"))).toBe(true); + }); + + it("flags a sensitive callee defined locally and invoked inside the loader, even without an import", () => { + const e = ep( + "api.v1.local-admin.ts", + `async function createJWT() { return "x"; } + export async function loader() { return createJWT(); }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a sensitive-named call made only at module scope outside the loader/action body", () => { + const e = ep( + "api.v1.module-scope.ts", + `function createJWT() { return "x"; } + createJWT(); + export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); From 68e09883d19b6154156631a4ca6c90a83e0dbc12 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:41:22 +0100 Subject: [PATCH 005/117] fix(observability-map): resolve one-hop helpers and narrow the scanner heuristics Follow-up to the adversarial review of the scanner fixes. - follow a call from a loader/action body to a same-file helper, one hop with a cycle guard, so a body that delegates reports the helper's statements, try/catch and callees rather than just the delegation. ph.$.ts goes from 2 statements and hasTryCatch false to 30 and true; 66 entry points gain statements, 6 gain hasTryCatch - only treat a ParseFailureError as a parse failure in scanDirectory and rethrow everything else, so an unreadable file is no longer reported as malformed source, and keep the diagnostic alongside the file name - match a builder handler only at the top of the config object or under methods..handler, not by name at any depth - read handler arguments from the root call of a builder chain only, so a callback given to a later decorator is not the route body - pin the loosened assertions and add negatives for the new resolution, the handler shapes and the chained builder --- .../observability-map/src/scan.ts | 148 +++++++++--- .../observability-map/src/types.ts | 10 +- .../observability-map/test/scan.test.ts | 225 +++++++++++++++++- 3 files changed, 344 insertions(+), 39 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index cb506ce2cd3..2ebe03b67f5 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -68,41 +68,75 @@ function calleeName(expr: ts.Expression): string | null { return null; } +const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); + +function propertyName(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) + return property.name.text; + return null; +} + +/** `methods: { POST: { handler } }`, the per-method shape of `createMultiMethodApiRoute`. */ +function collectMethodHandlers(methods: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const method of methods.properties) { + const name = propertyName(method); + if (!name || !HTTP_METHODS.has(name)) continue; + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (!ts.isObjectLiteralExpression(config)) continue; + for (const property of config.properties) { + if (!ts.isPropertyAssignment(property) || propertyName(property) !== "handler") continue; + const value = unwrap(property.initializer); + if (isEntryFunction(value)) out.push(value); + } + } +} + /** - * Functions on a `handler` property of an object argument, e.g. `createSSELoader({ handler })` and - * the per-method `{ POST: { handler } }` map. Only `handler` counts: the surrounding config also - * holds lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. + * The handler on an object argument, in the two shapes the route builders use: `handler` at the + * top level of the config (`createSSELoader({ handler })`) and `methods.POST.handler`. Matching by + * name at any depth would pick up an unrelated config callback that happens to be called + * `handler`, as well as the sibling lambdas (`findResource`, `authorization.resource`) that are + * not the entry-point body. */ function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { for (const property of object.properties) { - if (!ts.isPropertyAssignment(property) || !property.name) continue; + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyName(property); const value = unwrap(property.initializer); - if (ts.isObjectLiteralExpression(value)) { - collectNamedHandlers(value, out); - continue; - } - const name = - ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) - ? property.name.text - : null; if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + collectMethodHandlers(value, out); + } } } -/** Every function passed as an argument anywhere in a call chain, e.g. the builder's handler. */ -function collectHandlerFunctions(expr: ts.Expression, out: EntryFunction[]): void { - const target = unwrap(expr); - if (ts.isCallExpression(target)) { - for (const arg of target.arguments) { - const unwrapped = unwrap(arg); - if (isEntryFunction(unwrapped)) out.push(unwrapped); - else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); +/** The innermost call of a chain: the `createLoaderApiRoute(...)` in `createLoaderApiRoute(...).withCors(...)`. */ +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; } - collectHandlerFunctions(target.expression, out); - return; + return current; } - if (ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) { - collectHandlerFunctions(target.expression, out); +} + +/** + * The handler functions passed to a builder call. Only the root call of a chain is read: a + * callback given to a decorator further along the chain (`.withCors(cb)`) is not the route body. + */ +function collectHandlerFunctions(call: ts.CallExpression, out: EntryFunction[]): void { + for (const arg of rootCall(call).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); } } @@ -248,6 +282,30 @@ function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { return locals; } +/** + * Top-level functions by name, for resolving a body that delegates its work to a same-file helper + * (`export async function loader({ request }) { return proxyToPostHog(request); }`). + */ +function collectLocalFunctions(sf: ts.SourceFile): Map { + const functions = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && statement.body) { + functions.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + const value = unwrap(decl.initializer); + if (isEntryFunction(value)) functions.set(decl.name.text, value); + } + } + + return functions; +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); @@ -340,20 +398,41 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; const calleeNames: string[] = []; - for (const fn of target.functions) { + const localFunctions = collectLocalFunctions(sf); + // A body that delegates to a same-file helper does the work in that helper, so the helper's + // statements, try/catch and callees belong to the entry point. One hop only: a helper's own + // helpers are not followed, and the visited set stops a cycle and any double counting. + const visited = new Set(target.functions); + const helpers: EntryFunction[] = []; + + const walkBody = (fn: EntryFunction, followHelpers: boolean) => { statementCount += countFunctionStatements(fn); - if (!fn.body) continue; + if (!fn.body) return; const visit = (node: ts.Node) => { if (ts.isTryStatement(node)) hasTryCatch = true; if (ts.isCallExpression(node)) { const cn = calleeName(node.expression); if (cn) calleeNames.push(cn); + + if (followHelpers) { + const callee = unwrap(node.expression); + if (ts.isIdentifier(callee)) { + const helper = localFunctions.get(callee.text); + if (helper && !visited.has(helper)) { + visited.add(helper); + helpers.push(helper); + } + } + } } ts.forEachChild(node, visit); }; visit(fn.body); - } + }; + + for (const fn of target.functions) walkBody(fn, true); + for (const helper of helpers) walkBody(helper, false); return { fileName, @@ -383,12 +462,19 @@ export function scanDirectory(dir: string): { const parseFailures: string[] = []; const scan = (absolutePath: string, relativeName: string) => { + let ep: EntryPoint | null; try { - const ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); - if (ep) entryPoints.push(ep); - } catch { - parseFailures.push(relativeName); + ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); + } catch (error) { + // Only a genuinely malformed source is a parse failure. An unreadable file or a bug in the + // scanner must not be laundered into the same bucket, or a non-zero count means nothing. + if (error instanceof ParseFailureError) { + parseFailures.push(`${relativeName}: ${error.diagnostic}`); + return; + } + throw error; } + if (ep) entryPoints.push(ep); }; for (const entry of readdirSync(dir, { withFileTypes: true })) { diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index fcf037305fe..1f23f0e059f 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -16,10 +16,14 @@ export type EntryPoint = { actionInitializerCallee: string | null; /** Named and default imports, file-wide. */ importedNames: string[]; - /** Names of functions called inside the loader/action bodies. */ + /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; - /** Whether a `try` appears inside the loader/action bodies. */ + /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ hasTryCatch: boolean; - /** Statement count across loader/action bodies, used by the triviality rule. */ + /** + * Statement count across loader/action bodies, used by the triviality rule. A body that + * delegates to a same-file helper counts that helper's statements too, one hop only: work in a + * helper's own helpers, or in an imported module, is not counted. + */ statementCount: number; }; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 758f2c35c71..c727577a78c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,7 +1,7 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { scanDirectory, scanFile } from "../src/scan.js"; +import { ParseFailureError, scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` import { json } from "@remix-run/server-runtime"; @@ -122,6 +122,69 @@ describe("scanFile: named export clauses", () => { expect(ep!.calleeNames).not.toContain("lookup"); }); + it("counts the per-method `methods.POST.handler` bodies", () => { + const ep = scanFile( + "api.v1.prompts.$slug.override.ts", + ` + const { action, loader } = createMultiMethodApiRoute({ + params: ParamsSchema, + methods: { + POST: { + body: CreateBody, + handler: async ({ body }) => { + const created = await create(body); + return json(created); + }, + }, + DELETE: { + handler: async ({ params }) => { + return json({ ok: true }); + }, + }, + }, + }); + export { action, loader }; + ` + ); + expect(ep!.statementCount).toBe(3); + expect(ep!.calleeNames).toContain("create"); + }); + + it("ignores a nested config callback that happens to be named `handler`", () => { + const ep = scanFile( + "api.v1.named-collision.ts", + ` + export const loader = build({ + onError: { + handler: async () => { + const a = 1; + const b = 2; + const c = 3; + return null; + }, + }, + handler: async () => json({}), + }); + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("ignores a callback passed to a decorator further along the builder chain", () => { + const ep = scanFile( + "api.v1.chained.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})).withCors(async () => { + const a = 1; + const b = 2; + return a + b; + }); + ` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(1); + }); + it("detects an action-only route", () => { const ep = scanFile( "api.v1.action-only.ts", @@ -181,7 +244,125 @@ describe("scanFile: statement counting", () => { } ` ); - expect(ep!.statementCount).toBeGreaterThan(4); + // try (1) + 3 in the try block + 2 in the catch block + expect(ep!.statementCount).toBe(6); + }); + + it("counts a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxyToPostHog(request) { + const url = new URL(request.url); + try { + const upstream = await fetch(url); + return new Response(upstream.body); + } catch (e) { + logger.error(e); + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxyToPostHog(request); + } + export async function action({ request }) { + return proxyToPostHog(request); + } + ` + ); + // loader (1) + action (1) + helper: const url, try (1) + 2 + 2 + expect(ep!.statementCount).toBe(8); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.calleeNames).toContain("proxyToPostHog"); + expect(ep!.calleeNames).toContain("fetch"); + }); + + it("counts a shared helper once when both the loader and the action delegate to it", () => { + const ep = scanFile( + "shared.ts", + ` + function work() { + const a = 1; + const b = 2; + return a + b; + } + export async function loader() { return work(); } + export async function action() { return work(); } + ` + ); + // loader (1) + action (1) + helper (3), the helper counted once + expect(ep!.statementCount).toBe(5); + }); + + it("follows a delegating helper one hop only", () => { + const ep = scanFile( + "two-hop.ts", + ` + function deep() { + const a = 1; + const b = 2; + const c = 3; + return a + b + c; + } + function shallow() { + return deep(); + } + export async function loader() { return shallow(); } + ` + ); + // loader (1) + shallow (1). `deep` is a second hop and is not counted. + expect(ep!.statementCount).toBe(2); + }); + + it("terminates on a recursive helper", () => { + const ep = scanFile( + "recursive.ts", + ` + function recurse(n) { + if (n <= 0) return 0; + return recurse(n - 1); + } + export async function loader() { return recurse(3); } + ` + ); + // loader (1) + recurse: if (1) + return (1) + return (1) + expect(ep!.statementCount).toBe(4); + }); + + it("does not count an imported helper it cannot resolve", () => { + const ep = scanFile( + "imported.ts", + ` + import { proxy } from "./proxy.server"; + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("does not count a same-file function the body never calls", () => { + const ep = scanFile( + "unused-helper.ts", + ` + function unrelated() { + try { + const a = 1; + const b = 2; + return a + b; + } catch (e) { + return null; + } + } + export async function loader() { + return json({}); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); }); it("counts statements nested in if/for/while/switch blocks", () => { @@ -285,7 +466,7 @@ describe("scanFile: parse failures", () => { it("throws on a malformed source rather than returning a clean entry point", () => { expect(() => scanFile("broken.ts", `export async function loader() { const a = ; return json(`) - ).toThrow(); + ).toThrow(ParseFailureError); }); it("does not throw on a well-formed tsx route", () => { @@ -339,7 +520,41 @@ describe("scanDirectory", () => { const { entryPoints, parseFailures } = scanDirectory(dir); expect(entryPoints).toEqual([]); - expect(parseFailures).toEqual(["broken.ts"]); + expect(parseFailures).toHaveLength(1); + // The file name, then the diagnostic that made it a failure. + expect(parseFailures[0]).toMatch(/^broken\.ts: \S/); + }); + + // root ignores the mode bits, so the unreadable file would read fine. + it.skipIf(process.getuid?.() === 0)( + "rethrows an error that is not a parse failure instead of counting it as one", + () => { + const unreadable = join(dir, "unreadable.ts"); + writeFileSync(unreadable, `export async function loader() { return json({}); }`); + chmodSync(unreadable, 0o000); + + try { + expect(() => scanDirectory(dir)).toThrow(/EACCES|EPERM/); + } finally { + chmodSync(unreadable, 0o600); + } + } + ); + + it("skips a non-route file inside a route directory", () => { + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + writeFileSync( + join(dir, "nested.route", "loaders.server.ts"), + `export async function loader() { return json({}); }` + ); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["nested.route/route.tsx"]); }); it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { From 6ab9118c2854e4155419b26a6fc8974cf474f4b0 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 13:52:20 +0100 Subject: [PATCH 006/117] feat(observability-map): triviality rule and suppression comments --- .../observability-map/src/suppression.ts | 13 ++ .../observability-map/src/triviality.ts | 52 ++++++ .../test/suppression.test.ts | 45 +++++ .../observability-map/test/triviality.test.ts | 167 ++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 internal-packages/observability-map/src/suppression.ts create mode 100644 internal-packages/observability-map/src/triviality.ts create mode 100644 internal-packages/observability-map/test/suppression.test.ts create mode 100644 internal-packages/observability-map/test/triviality.test.ts diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts new file mode 100644 index 00000000000..bd807816254 --- /dev/null +++ b/internal-packages/observability-map/src/suppression.ts @@ -0,0 +1,13 @@ +// The reason runs to the end of the line: `.` does not match a newline, so a suppression on one +// line cannot pick up a reason from the next one. +const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/g; + +/** Check id to reason. A suppression without a reason is ignored. */ +export function suppressedChecks(source: string): Map { + const out = new Map(); + for (const match of source.matchAll(PATTERN)) { + const [, id, reason] = match; + if (id && reason && reason.trim().length > 0) out.set(id, reason.trim()); + } + return out; +} diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts new file mode 100644 index 00000000000..e36b80dc9b0 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.ts @@ -0,0 +1,52 @@ +import type { EntryPoint } from "./types.js"; + +/** + * Substrings that say the route touches a service, a datastore or the network. Matched against the + * callee names and the whole file, so an import of `prisma` disqualifies the file even when the + * query itself sits somewhere the scanner does not walk. + */ +const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; + +/** + * Calls a genuinely trivial body makes: parse the params, build a path, hand back a response. Every + * shape found in the real tree stays at or below three, so anything busier is doing work. Allowing + * a fourth admits `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. + */ +const MAX_CALLS = 3; + +/** + * Parse the params, build a path, redirect. Or an environment guard and two returns. Both real + * shapes need three. Allowing a fourth admits the routes that authenticate and then hand off to a + * presenter (`...tasks.stream/route.tsx`), which have real work behind them and belong in the + * report; allowing a fifth admits an admin route that calls a service and hand-rolls its own error + * responses. + */ +const MAX_STATEMENTS = 3; + +/** + * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed + * response, or hands off in a single call. Checks report not-applicable for these rather than + * failing, which is what stops `@.ts` being a finding. + * + * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the + * report again, so every signal that the body might be doing real work rules triviality out: + * + * - `statementCount` does not descend into inline callbacks, so a two-statement body can still hold + * a pile of work. `calleeNames` does descend, so the call count catches what the statement count + * misses. + * - An initializer callee means the route is wrapped in a builder, and the config passed to that + * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is + * not the whole route, so we cannot claim it is trivial. + * - A try/catch is exactly what the error-classification check reads, so a body with one has an + * error path worth reporting on however short it is. + */ +export function isTrivial(ep: EntryPoint): boolean { + if (ep.statementCount > MAX_STATEMENTS) return false; + if (ep.calleeNames.length > MAX_CALLS) return false; + if (ep.hasTryCatch) return false; + if (ep.loaderInitializerCallee !== null || ep.actionInitializerCallee !== null) return false; + + const callees = ep.calleeNames.join(" ").toLowerCase(); + const source = ep.source.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || source.includes(h)); +} diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts new file mode 100644 index 00000000000..8e8e88d5fa5 --- /dev/null +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -0,0 +1,45 @@ +import { suppressedChecks } from "../src/suppression.js"; + +describe("suppressedChecks", () => { + it("reads a suppression with its reason", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- liveness probe, deliberately silent + export async function loader() { return { ok: true }; }` + ); + expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); + }); + + it("ignores a suppression with no reason", () => { + const m = suppressedChecks(`// obs-map-disable-next-line error-classification`); + expect(m.size).toBe(0); + }); + + it("ignores a suppression whose reason is only whitespace", () => { + const m = suppressedChecks(`// obs-map-disable-next-line error-classification -- `); + expect(m.size).toBe(0); + }); + + it("reads several suppressions in one file", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- liveness probe + // obs-map-disable-next-line request-context -- no identifiers exist here + export async function loader() { return { ok: true }; }` + ); + expect(m.size).toBe(2); + expect(m.get("error-classification")).toBe("liveness probe"); + expect(m.get("request-context")).toBe("no identifiers exist here"); + }); + + it("returns nothing for a file with no suppressions", () => { + expect(suppressedChecks(`export async function loader() { return 1; }`).size).toBe(0); + }); + + it("does not carry a reason across lines", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification + // some other comment -- with a dash + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); diff --git a/internal-packages/observability-map/test/triviality.test.ts b/internal-packages/observability-map/test/triviality.test.ts new file mode 100644 index 00000000000..81c63b34eb7 --- /dev/null +++ b/internal-packages/observability-map/test/triviality.test.ts @@ -0,0 +1,167 @@ +import { isTrivial } from "../src/triviality.js"; +import { scanFile } from "../src/scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("isTrivial", () => { + it("treats a redirect-only route as trivial", () => { + const e = ep( + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat a route that queries the database as trivial", () => { + const e = ep( + "api.v1.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return rows; + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + // The motivating case from the design: four lines, one delegating call, nothing to instrument. + it("treats the impersonation-clearing route as trivial", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a static-response route as trivial", () => { + const e = ep( + "internal.webhooks.slack.interactivity.ts", + `export function action() { return new Response(null, { status: 200 }); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a guard and two fixed responses as trivial", () => { + const e = ep( + "api.v1.mock.ts", + `export async function action() { + if (process.env.NODE_ENV === "production") { + return new Response("Not found", { status: 404 }); + } + return new Response(JSON.stringify({ id: "123" }), { status: 200 }); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a params-parse and redirect as trivial", () => { + const e = ep( + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(isTrivial(e)).toBe(true); + }); +}); + +// statementCount deliberately does not descend into inline callbacks, so a two-statement body can +// still hold a pile of work. calleeNames does descend, which is what catches these. +describe("isTrivial: work hidden from the statement count", () => { + it("does not treat a short body holding a busy callback as trivial", () => { + const e = ep( + "api.v1.remote-build-provider-status.ts", + `export async function loader() { + const result = await fromPromise( + (async () => { + const response = await callProvider(); + const parsed = ProviderStatus.safeParse(await response.json()); + if (!parsed.success) return err("bad-payload"); + return ok(parsed.data); + })() + ); + return result.match(toJson, toError); + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a builder-wrapped route with a one-line handler as trivial", () => { + const e = ep( + "api.v1.deployments.current.ts", + `import { json } from "@remix-run/server-runtime"; + import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute( + { findResource: async (_params, auth) => lookup(auth) }, + async ({ resource }) => { return json(resource); } + );` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a body that delegates to a same-file helper as trivial", () => { + const e = ep( + "api.v1.proxy.ts", + `export async function loader({ request }) { return proxy(request); } + async function proxy(request) { + const url = buildUrl(request); + const response = await send(url); + const body = await response.text(); + return new Response(body); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +// Calibrated against apps/webapp/app/routes: three statements is the widest window that holds only +// redirects, fixed responses and single hand-offs. The fourth statement is where routes start +// authenticating and then calling a presenter, which is work worth reporting on. +describe("isTrivial: the statement boundary", () => { + it("treats a three-statement redirect as trivial", () => { + const e = ep( + "schedules._index/route.tsx", + `import { redirect } from "@remix-run/server-runtime"; + import { EnvironmentParamSchema, v3EnvironmentPath } from "~/utils/pathBuilder"; + export async function loader({ params }) { + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const tasksPath = v3EnvironmentPath({ slug: organizationSlug }, { slug: projectParam }, { slug: envParam }); + return redirect(\`\${tasksPath}?types=SCHEDULED\`); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat an authenticated hand-off to a presenter as trivial", () => { + const e = ep( + "tasks.stream/route.tsx", + `import { TasksStreamPresenter } from "~/presenters/v3/TasksStreamPresenter.server"; + import { requireUserId } from "~/services/session.server"; + import { EnvironmentParamSchema } from "~/utils/pathBuilder"; + export async function loader({ request, params }) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const presenter = new TasksStreamPresenter(); + return presenter.call({ request, projectParam, envParam, organizationSlug, userId }); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +describe("isTrivial: an error path is something to instrument", () => { + it("does not treat a short body with a try/catch as trivial", () => { + const e = ep( + "api.v1.ping.ts", + `export async function loader() { + try { return await ping(); } catch { return null; } + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); From 75eaaa8ffc9daaef161ab17c2d1b03bd843d3cbb Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:09:23 +0100 Subject: [PATCH 007/117] feat(observability-map): the four coverage checks Adds error-classification, auth-boundary, request-context and audit-trail, plus the CHECKS registry. Every check is a pure function of an EntryPoint and reads body-scoped evidence only. Two rules differ from the design. error-classification uses hasTryCatch as its gate rather than a regex over ep.source, which is the whole file including the React component; EntryPoint carries no evidence about what a catch does with the error, so the check reports the hand-rolled catch and says it has not been read. request-context looks for an identity resolved in the body rather than grepping the file for identifier names, for the same reason. Both deviations, and the calibration run over the 427 webapp entry points, are written up in the task 5 report. --- .../src/checks/auditTrail.ts | 25 ++ .../src/checks/authBoundary.ts | 42 ++ .../src/checks/errorClassification.ts | 65 ++++ .../observability-map/src/checks/index.ts | 12 + .../src/checks/requestContext.ts | 53 +++ .../observability-map/test/checks.test.ts | 360 ++++++++++++++++++ 6 files changed, 557 insertions(+) create mode 100644 internal-packages/observability-map/src/checks/auditTrail.ts create mode 100644 internal-packages/observability-map/src/checks/authBoundary.ts create mode 100644 internal-packages/observability-map/src/checks/errorClassification.ts create mode 100644 internal-packages/observability-map/src/checks/index.ts create mode 100644 internal-packages/observability-map/src/checks/requestContext.ts create mode 100644 internal-packages/observability-map/test/checks.test.ts diff --git a/internal-packages/observability-map/src/checks/auditTrail.ts b/internal-packages/observability-map/src/checks/auditTrail.ts new file mode 100644 index 00000000000..ee3dcdf45a7 --- /dev/null +++ b/internal-packages/observability-map/src/checks/auditTrail.ts @@ -0,0 +1,25 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; + +const ID = "audit-trail"; + +const AUDIT_SYMBOLS = ["auditLog", "recordAudit", "writeAuditEvent"]; + +/** + * Whether a sensitive mutation leaves a record of who did it. Nothing in the webapp writes one + * today, so every applicable entry point fails: the check states the gap rather than measuring + * variation between routes, which is why the score leaves it out. + */ +export const auditTrail = { + id: ID, + run(ep: EntryPoint): CheckResult { + // Mutations only: a sensitive read does not need an actor record. + if (!classifySensitivity(ep).sensitive || !ep.hasAction) { + return { id: ID, status: "not-applicable", detail: "not a sensitive mutation" }; + } + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + return AUDIT_SYMBOLS.some((s) => symbols.has(s)) + ? { id: ID, status: "pass", detail: "records an audit event" } + : { id: ID, status: "fail", detail: "sensitive mutation with no audit record" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts new file mode 100644 index 00000000000..11030d1d2d8 --- /dev/null +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -0,0 +1,42 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { usesBuilder } from "./errorClassification.js"; + +const ID = "auth-boundary"; + +/** + * The two shapes the webapp's guards take: `requireUserId`, `requireAdminApiRequest`, + * `authenticateApiRequest`, `authenticateProjectApiKey`. Matched against `calleeNames`, which is + * scoped to the loader/action bodies and follows one hop into a same-file helper, so a guard the + * route only imports and never calls does not count. + */ +const GUARD = /^(require|authenticate)/; + +/** + * Whether a route that handles credentials, tokens or money checks who is asking. + * + * The design matched `importedNames` as well as `calleeNames`. Across the 67 sensitive entry points + * in the real tree that widening changes nothing: every route with a `require*` import calls it + * from the body too. So the file-wide half only ever stood to hand out a pass for a dead import, + * and it is gone. + */ +export const authBoundary = { + id: ID, + run(ep: EntryPoint): CheckResult { + const sensitivity = classifySensitivity(ep); + if (!sensitivity.sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "authenticated by the builder" }; + } + if (ep.calleeNames.some((n) => GUARD.test(n))) { + return { id: ID, status: "pass", detail: "guarded in the body" }; + } + return { + id: ID, + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body`, + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts new file mode 100644 index 00000000000..637656f0d62 --- /dev/null +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -0,0 +1,65 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "error-classification"; + +/** + * The route builders that own the failure path: they authenticate, they catch, they pass a thrown + * `Response` through untouched and report anything else through `logBoundaryError` before + * answering 500. A route wrapped in one of these has its errors classified for it. + * + * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not + * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. + * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. + */ +export const BUILDERS = new Set([ + "createLoaderApiRoute", + "createActionApiRoute", + "createLoaderPATApiRoute", + "createActionPATApiRoute", + "createMultiMethodApiRoute", + "createLoaderWorkerApiRoute", + "createActionWorkerApiRoute", + "dashboardLoader", + "dashboardAction", +]); + +export function usesBuilder(ep: EntryPoint): boolean { + return ( + (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || + (ep.actionInitializerCallee !== null && BUILDERS.has(ep.actionInitializerCallee)) + ); +} + +/** + * Who decides what a failure means. Three answers count as covered: the builder does it, the route + * declines to interfere and the error reaches the global handler, or the route is trivial. + * + * The fourth answer is the finding: a route outside the builders that catches its own errors. What + * that catch then does is the question the check would like to answer and cannot. `EntryPoint` + * carries `hasTryCatch`, a body-scoped boolean, and nothing about the catch clause itself, so a + * rethrow and a swallow look identical from here. Reading the shape of the catch out of `ep.source` + * would mean matching the whole file, React component included, which is how a component's + * try/catch ends up deciding a loader's verdict. Coarse and honest beats precise and wrong: the + * check reports the hand-rolled catch and says it has not been read. + */ +export const errorClassification = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "classified by the builder" }; + } + if (!ep.hasTryCatch) { + return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; + } + return { + id: ID, + status: "fail", + detail: + "handles its own errors outside a route builder, and the catch has not been read: check it distinguishes expected from unexpected", + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts new file mode 100644 index 00000000000..b94c8ad4867 --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.ts @@ -0,0 +1,12 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { errorClassification } from "./errorClassification.js"; +import { authBoundary } from "./authBoundary.js"; +import { requestContext } from "./requestContext.js"; +import { auditTrail } from "./auditTrail.js"; + +export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; + +/** audit-trail is scored separately, see score.ts. */ +export const CHECKS: Check[] = [errorClassification, authBoundary, requestContext, auditTrail]; +export const SCORED_CHECK_IDS = ["error-classification", "auth-boundary", "request-context"]; +export { usesBuilder, BUILDERS } from "./errorClassification.js"; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts new file mode 100644 index 00000000000..c224b3f413d --- /dev/null +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -0,0 +1,53 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; +import { usesBuilder } from "./errorClassification.js"; + +const ID = "request-context"; + +/** `requireUserId`, `authenticateApiRequest`: the request comes back with an identity attached. */ +const GUARD = /^(require|authenticate)/; + +/** `findProjectBySlug`, `loadProjectEnvironmentFromRequest`, `getUserSession`, `findWaitpoint`. */ +const RESOLVE = /^(find|get|load|resolve|lookup)/; +const SCOPE = + /(environment|project|organization|org|run|user|account|waitpoint|deployment|batch|schedule|task)/i; + +/** + * Whether the body ever works out who or what the request is for. A failure in a route that has + * resolved a user, an environment or a project can be attributed to one when it is reported; a + * route that resolves none of them has nothing to attribute a failure to, wherever it is logged. + */ +function resolvesRequestIdentity(ep: EntryPoint): boolean { + return ep.calleeNames.some((n) => GUARD.test(n) || (RESOLVE.test(n) && SCOPE.test(n))); +} + +/** + * The design asked whether an identifier reaches the failure path, and looked for one by matching + * `environmentId` and friends against `ep.source`. That is the whole file: a route whose React + * component renders `runId` would pass a check about its loader's error handling. + * + * The body-scoped substitute is weaker. `EntryPoint` records the names of the functions a body + * calls, not the arguments passed to them, so `logger.error("failed", { environmentId })` and + * `logger.error("failed")` are the same to this check. What is left is the identity the body + * resolves, which the callee names do carry. It answers a related question rather than the + * original one, and its evidence overlaps heavily with `auth-boundary`: see the task 5 report. + */ +export const requestContext = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + if (usesBuilder(ep)) { + return { id: ID, status: "pass", detail: "attributed by the builder" }; + } + if (resolvesRequestIdentity(ep)) { + return { id: ID, status: "pass", detail: "resolves a request identity" }; + } + return { + id: ID, + status: "fail", + detail: "resolves no user, environment or project, so a failure here names nobody", + }; + }, +}; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts new file mode 100644 index 00000000000..57510e656ec --- /dev/null +++ b/internal-packages/observability-map/test/checks.test.ts @@ -0,0 +1,360 @@ +import { CHECKS, SCORED_CHECK_IDS } from "../src/checks/index.js"; +import { scanFile } from "../src/scan.js"; + +const run = (id: string, fileName: string, source: string) => { + const ep = scanFile(fileName, source)!; + return CHECKS.find((c) => c.id === id)!.run(ep); +}; + +/** + * The React component that lives alongside the loader in a `.tsx` route. Every check reads + * body-scoped evidence, so nothing in here may change a verdict: it try/catches, it logs, it names + * every request identifier the checks look for, and it calls an auth helper. + */ +const COMPONENT = ` + export default function Page() { + const { environmentId, organizationId, projectId, runId } = useTypedLoaderData(); + useEffect(() => { + try { + requireUserId(environmentId); + logger.error("render failed", { environmentId, organizationId, projectId, runId }); + } catch (e) { + if (e instanceof Error) return; + throw e; + } + }, [environmentId]); + return
{runId}
; + } +`; + +describe("registry", () => { + it("holds the four checks, with audit-trail left out of the score", () => { + expect(CHECKS.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "request-context", + "audit-trail", + ]); + expect(SCORED_CHECK_IDS).toEqual(["error-classification", "auth-boundary", "request-context"]); + }); +}); + +describe("error-classification", () => { + it("passes a builder-wrapped route with no local try/catch", () => { + const r = run( + "error-classification", + "api.v1.x.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a raw route whose catch swallows every error identically", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The brief expected a pass here. `EntryPoint` carries `hasTryCatch` and nothing about what the + // catch does with the error, so the check cannot see the rethrow, and reading it out of + // `ep.source` would read the whole file. Known false positive, see the task 5 report. + it("also fails a raw route whose catch branches on the error", () => { + const r = run( + "error-classification", + "api.v1.z.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof NotFound) return null; throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes a raw route that lets its errors propagate", () => { + const r = run( + "error-classification", + "api.v1.w.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "error-classification", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the only try/catch in the file belongs to the component. + it("does not flag a route whose try/catch is in the React component", () => { + const r = run( + "error-classification", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return typedjson({ rows }); + } + ${COMPONENT}` + ); + expect(r.status).toBe("pass"); + }); +}); + +describe("auth-boundary", () => { + it("passes a sensitive route guarded by a require helper", () => { + const r = run( + "auth-boundary", + "admin.api.v1.gc.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return prisma.thing.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a sensitive route guarded by an authenticate helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + if (!auth) throw new Response(null, { status: 401 }); + return prisma.token.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a sensitive route with no guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a non-sensitive route", () => { + const r = run( + "auth-boundary", + "api.v1.timezones.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.tz.findMany(); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the guard sits one hop away, in a same-file helper. + it("does not flag a sensitive route whose guard is in a same-file helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); }` + ); + expect(r.status).toBe("pass"); + }); + + // The guard has to be called, not merely imported: importedNames is file-wide. + it("fails a sensitive route that imports a guard it never calls", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + } + export function meta() { return requireUserId; }` + ); + expect(r.status).toBe("fail"); + }); +}); + +describe("request-context", () => { + it("passes a builder-wrapped route", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async () => json(await prisma.thing.findMany()));` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route that resolves the request identity", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route that resolves a tenant without authenticating", () => { + const r = run( + "request-context", + "resources.things.ts", + `import { loadProjectEnvironmentFromRequest } from "~/services/environment.server"; + import { prisma } from "~/db.server"; + export async function loader({ request, params }) { + const environment = await loadProjectEnvironmentFromRequest(request, params); + return json(await prisma.thing.findMany({ where: { environmentId: environment.id } })); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a raw route that resolves nothing about the request", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { logger.error("failed"); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The brief expected a pass here, on the strength of the identifier in the log call. `EntryPoint` + // records callee names, not their arguments, so that identifier is not body-scoped evidence and + // the check cannot see it. Known false positive, see the task 5 report. + it("also fails a raw route that only names an identifier in a log call", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("failed", { environmentId: params.envId }); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture: identifiers all over the component, none in the loader. The check must + // read the loader, so the verdict has to come from the loader's own call to requireUserId. + it("does not flag a route whose identifiers are in the React component", () => { + const r = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return typedjson(await prisma.thing.findMany({ where: { userId } })); + } + ${COMPONENT}` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a route whose identifiers are only in the React component", () => { + const r = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + return typedjson(await prisma.thing.findMany()); + } + ${COMPONENT}` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "request-context", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); +}); + +describe("audit-trail", () => { + it("is applicable only to sensitive mutations", () => { + const readOnly = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `export async function loader() { return 1; }` + ); + expect(readOnly.status).toBe("not-applicable"); + + const mutation = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ); + expect(mutation.status).toBe("fail"); + }); + + // False positive fixture: an ordinary mutation is not an audit target. + it("is not applicable to a non-sensitive mutation", () => { + const r = run( + "audit-trail", + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.thing.create({ data: {} }); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("passes a sensitive mutation that records an audit event", () => { + const r = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }` + ); + expect(r.status).toBe("pass"); + }); +}); From 6a07d6d2f4063109651f3c2f2200c28e4a3bbf9b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:16:40 +0100 Subject: [PATCH 008/117] feat(observability-map): record catch clause and callee evidence on entry points The error-classification and request-context checks could not tell a rethrow from a swallow, or a database call from any method with a common name. Four additive fields, all body scoped through the existing one-hop helper resolution. - catchRethrows and catchBranches: whether a catch clause in the bodies contains a throw, or branches with if, switch or instanceof. Of the 190 routes that catch, 140 do one of those and 50 take one path out - calleeTexts: the full callee path (prisma.organization.findFirst), index aligned with calleeNames, which is unchanged - logCalls: logger.* and log.* calls with their object argument field names and whether the call sits in a catch, so a check can ask whether the failure path logs an identifier No existing field changes value on any of the 427 route entry points. --- .../observability-map/src/scan.ts | 100 ++++++- .../observability-map/src/types.ts | 31 ++ .../observability-map/test/scan.test.ts | 283 ++++++++++++++++++ 3 files changed, 409 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 2ebe03b67f5..4702a7d7771 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -1,7 +1,7 @@ import ts from "typescript"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import type { EntryPoint } from "./types.js"; +import type { EntryPoint, LogCall } from "./types.js"; /** Thrown by `scanFile` when the source does not parse cleanly. */ export class ParseFailureError extends Error { @@ -68,6 +68,65 @@ function calleeName(expr: ts.Expression): string | null { return null; } +/** + * Callee as recorded in `calleeTexts`: the whole path, `prisma.organization.findFirst` rather than + * `findFirst`. Null when the path runs through something with no name of its own, e.g. + * `new PromptService().createOverride`, where the caller falls back to the bare name. + */ +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +/** `logger.error`, `log.info`, `this.logger.debug`. */ +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +/** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ +function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields: string[] } { + for (const arg of call.arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + const fields: string[] = []; + for (const property of target.properties) { + const name = propertyName(property); + if (name) fields.push(name); + } + return { found: true, fields }; + } + return { found: false, fields: [] }; +} + +/** What a catch clause does with the error, beyond the fact that it caught one. */ +function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { + let rethrows = false; + let branches = false; + + const visit = (node: ts.Node) => { + if (ts.isThrowStatement(node)) rethrows = true; + if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ) { + branches = true; + } + ts.forEachChild(node, visit); + }; + visit(clause.block); + + return { rethrows, branches }; +} + const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); function propertyName(property: ts.ObjectLiteralElementLike): string | null { @@ -396,7 +455,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; + let catchRethrows = false; + let catchBranches = false; const calleeNames: string[] = []; + const calleeTexts: string[] = []; + const logCalls: LogCall[] = []; const localFunctions = collectLocalFunctions(sf); // A body that delegates to a same-file helper does the work in that helper, so the helper's @@ -409,11 +472,34 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += countFunctionStatements(fn); if (!fn.body) return; - const visit = (node: ts.Node) => { + const visit = (node: ts.Node, inCatch: boolean) => { if (ts.isTryStatement(node)) hasTryCatch = true; + + if (ts.isCatchClause(node)) { + const evidence = catchClauseEvidence(node); + catchRethrows ||= evidence.rethrows; + catchBranches ||= evidence.branches; + ts.forEachChild(node, (child) => visit(child, true)); + return; + } + if (ts.isCallExpression(node)) { const cn = calleeName(node.expression); - if (cn) calleeNames.push(cn); + if (cn) { + const text = calleeText(node.expression) ?? cn; + calleeNames.push(cn); + calleeTexts.push(text); + + if (LOGGER_CALLEE.test(text)) { + const argument = objectArgumentFields(node); + logCalls.push({ + callee: text, + hasObjectArgument: argument.found, + fields: argument.fields, + inCatch, + }); + } + } if (followHelpers) { const callee = unwrap(node.expression); @@ -426,9 +512,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } } } - ts.forEachChild(node, visit); + ts.forEachChild(node, (child) => visit(child, inCatch)); }; - visit(fn.body); + visit(fn.body, false); }; for (const fn of target.functions) walkBody(fn, true); @@ -443,7 +529,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { actionInitializerCallee: target.actionInitializerCallee, importedNames, calleeNames, + calleeTexts, hasTryCatch, + catchRethrows, + catchBranches, + logCalls, statementCount, }; } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 1f23f0e059f..15b8d581132 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -6,6 +6,18 @@ export type CheckResult = { detail?: string; }; +/** A logging call made from a loader/action body, or from a same-file helper the body calls. */ +export type LogCall = { + /** Full callee path, e.g. `logger.error`. */ + callee: string; + /** Whether an object literal was passed as an argument. */ + hasObjectArgument: boolean; + /** Property names on that object literal, e.g. `["environmentId", "error"]`. */ + fields: string[]; + /** Whether the call sits inside a catch clause, i.e. on the failure path. */ + inCatch: boolean; +}; + export type EntryPoint = { fileName: string; source: string; @@ -18,8 +30,27 @@ export type EntryPoint = { importedNames: string[]; /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; + /** + * The same calls as `calleeNames`, same order and same length, but as the whole callee path: + * `prisma.organization.findFirst` where `calleeNames` has `findFirst`. A path that runs through + * something unnameable (`new PromptService().createOverride`) falls back to the bare name. + */ + calleeTexts: string[]; /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ hasTryCatch: boolean; + /** + * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided + * the error is not its to answer, which is a different act from swallowing it. + */ + catchRethrows: boolean; + /** + * Whether any catch clause in those bodies branches on the error: an `if`, a `switch`, or an + * `instanceof`. With `catchRethrows` both false while `hasTryCatch` is true, every catch in the + * entry point takes one path out regardless of what was thrown. + */ + catchBranches: boolean; + /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ + logCalls: LogCall[]; /** * Statement count across loader/action bodies, used by the triviality rule. A body that * delegates to a same-file helper counts that helper's statements too, one hop only: work in a diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index c727577a78c..ca92dd26182 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -569,3 +569,286 @@ describe("scanDirectory", () => { expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); }); }); + +describe("scanFile: catch clause evidence", () => { + it("sets catchRethrows when a catch rethrows", () => { + const ep = scanFile( + "rethrow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error(e); + throw e; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(false); + }); + + it("leaves both flags false when the catch only returns", () => { + const ep = scanFile( + "swallow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("sets catchBranches for an `if` on the error", () => { + const ep = scanFile( + "branch-if.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) { + return json({ error: "bad json" }, { status: 400 }); + } + return json({ error: "failed" }, { status: 500 }); + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + expect(ep!.catchRethrows).toBe(false); + }); + + it("sets catchBranches for a bare instanceof with no `if`", () => { + const ep = scanFile( + "branch-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return e instanceof Response ? e : json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + }); + + it("sets catchBranches for a switch in the catch", () => { + const ep = scanFile( + "branch-switch.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + switch (e.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catchBranches).toBe(true); + }); + + it("ignores a catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + if (e instanceof RenderError) throw e; + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("reads a catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxy(request) { + try { + return await fetch(request.url); + } catch (e) { + if (e.name === "AbortError") throw e; + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(true); + }); + + it("leaves both flags false for a try with no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); +}); + +describe("scanFile: callee texts", () => { + it("keeps the full callee expression alongside the bare name", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + logger.error("nope", { organizationId: org.id }); + return json(org); + } + ` + ); + expect(ep!.calleeNames).toContain("findFirst"); + expect(ep!.calleeTexts).toContain("prisma.organization.findFirst"); + expect(ep!.calleeTexts).toContain("logger.error"); + expect(ep!.calleeTexts).toContain("json"); + // Index-aligned with calleeNames, so a consumer can read either. + expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + }); + + it("does not leak calls made in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await prisma.run.findMany()); + } + export default function Page() { + useFancyHook(); + analytics.track("viewed"); + return null; + } + ` + ); + expect(ep!.calleeTexts).toContain("prisma.run.findMany"); + expect(ep!.calleeTexts).not.toContain("analytics.track"); + expect(ep!.calleeTexts).not.toContain("useFancyHook"); + }); + + it("records callee texts from a same-file helper the body delegates to", () => { + const ep = scanFile( + "delegating.ts", + ` + async function load(id) { + return prisma.project.findUnique({ where: { id } }); + } + export async function loader({ params }) { + return json(await load(params.id)); + } + ` + ); + expect(ep!.calleeTexts).toContain("prisma.project.findUnique"); + }); + + it("falls back to the bare name for a callee it cannot render as a path", () => { + const ep = scanFile( + "new-expression.ts", + ` + export async function action({ request }) { + return json(await new PromptService().createOverride(request)); + } + ` + ); + expect(ep!.calleeNames).toContain("createOverride"); + expect(ep!.calleeTexts).toContain("createOverride"); + expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + }); +}); + +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { + const ep = scanFile( + "logging.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + hasObjectArgument: true, + fields: ["environmentId", "error"], + inCatch: true, + }); + }); + + it("records a log call with no object argument, outside a catch", () => { + const ep = scanFile( + "logging-plain.ts", + ` + export async function loader() { + log.info("starting"); + return json({}); + } + ` + ); + expect(ep!.logCalls).toEqual([ + { callee: "log.info", hasObjectArgument: false, fields: [], inCatch: false }, + ]); + }); + + it("ignores a non-logger call and a log call in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await load()); + } + export default function Page() { + logger.debug("rendered", { runId: 1 }); + return null; + } + ` + ); + expect(ep!.logCalls).toEqual([]); + }); +}); From 751ea1b3b4f54942ef6e2448d9e33b01ec6f8390 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:28:52 +0100 Subject: [PATCH 009/117] feat(observability-map): rework two checks on the enriched scanner error-classification now reads catchRethrows and catchBranches instead of the mere presence of a try. It fails only where every catch in the bodies takes one way out regardless of what was thrown, which drops the finding count from 130 to 50. The swallow is read before the builder is credited: a swallow inside a builder-wrapped handler never reaches the builder, and 18 of the 50 are that shape. request-context now asks whether a failure-path log names a tenant, using logCalls with inCatch and the field names. The builder pass is gone, because the builders log { error, url } at their boundary and the logger only attaches http context ambiently, so a wrapped route is not attributed either. The check no longer echoes auth-boundary: 17 entry points of 427 are scored by both, and they disagree on 10 of those. auth-boundary and audit-trail are unchanged. --- .../src/checks/errorClassification.ts | 39 +++-- .../src/checks/requestContext.ts | 62 ++++--- .../observability-map/test/checks.test.ts | 158 +++++++++++++----- 3 files changed, 164 insertions(+), 95 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 637656f0d62..cfa8e389a82 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -32,16 +32,17 @@ export function usesBuilder(ep: EntryPoint): boolean { } /** - * Who decides what a failure means. Three answers count as covered: the builder does it, the route - * declines to interfere and the error reaches the global handler, or the route is trivial. + * Who decides what a failure means, and on what evidence. * - * The fourth answer is the finding: a route outside the builders that catches its own errors. What - * that catch then does is the question the check would like to answer and cannot. `EntryPoint` - * carries `hasTryCatch`, a body-scoped boolean, and nothing about the catch clause itself, so a - * rethrow and a swallow look identical from here. Reading the shape of the catch out of `ep.source` - * would mean matching the whole file, React component included, which is how a component's - * try/catch ends up deciding a loader's verdict. Coarse and honest beats precise and wrong: the - * check reports the hand-rolled catch and says it has not been read. + * The swallow is read before the builder is credited, which looks like the wrong order until you + * read `api.v2.runs.$runParam.cancel.ts`: a `createActionApiRoute` handler wrapping its service + * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that + * error never does. Crediting the wrapper would hide the one case in this family worth finding. + * + * `catchRethrows` and `catchBranches` are OR-ed across every catch clause in the bodies, so both + * false means every catch in the entry point takes the same way out whatever was thrown. The + * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses + * those rather than inventing them. */ export const errorClassification = { id: ID, @@ -49,17 +50,19 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches) { + return { + id: ID, + status: "fail", + detail: "catches its errors and takes one way out regardless of what was thrown", + }; + } + if (ep.catchRethrows || ep.catchBranches) { + return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; + } if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "classified by the builder" }; } - if (!ep.hasTryCatch) { - return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; - } - return { - id: ID, - status: "fail", - detail: - "handles its own errors outside a route builder, and the catch has not been read: check it distinguishes expected from unexpected", - }; + return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; }, }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index c224b3f413d..248ec1374ff 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,53 +1,51 @@ -import type { CheckResult, EntryPoint } from "../types.js"; -import { isTrivial } from "../triviality.js"; -import { usesBuilder } from "./errorClassification.js"; +import type { CheckResult, EntryPoint, LogCall } from "../types.js"; const ID = "request-context"; -/** `requireUserId`, `authenticateApiRequest`: the request comes back with an identity attached. */ -const GUARD = /^(require|authenticate)/; - -/** `findProjectBySlug`, `loadProjectEnvironmentFromRequest`, `getUserSession`, `findWaitpoint`. */ -const RESOLVE = /^(find|get|load|resolve|lookup)/; -const SCOPE = - /(environment|project|organization|org|run|user|account|waitpoint|deployment|batch|schedule|task)/i; - /** - * Whether the body ever works out who or what the request is for. A failure in a route that has - * resolved a user, an environment or a project can be attributed to one when it is reported; a - * route that resolves none of them has nothing to attribute a failure to, wherever it is logged. + * A field name that says which tenant, request or resource the failure belongs to. Matched on the + * suffix, in the camelCase the webapp writes: `environmentId`, `organizationSlug`, `runFriendlyId`, + * `projectParam`, `taskIdentifier`. Lowercase `id` inside a word is not a suffix, so `valid` and + * `paid` do not qualify. */ -function resolvesRequestIdentity(ep: EntryPoint): boolean { - return ep.calleeNames.some((n) => GUARD.test(n) || (RESOLVE.test(n) && SCOPE.test(n))); +const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; + +function failurePathLogs(ep: EntryPoint): LogCall[] { + return ep.logCalls.filter((l) => l.inCatch); } /** - * The design asked whether an identifier reaches the failure path, and looked for one by matching - * `environmentId` and friends against `ep.source`. That is the whole file: a route whose React - * component renders `runId` would pass a check about its loader's error handling. + * Whether a failure this route reports itself can be traced to whoever it happened to. * - * The body-scoped substitute is weaker. `EntryPoint` records the names of the functions a body - * calls, not the arguments passed to them, so `logger.error("failed", { environmentId })` and - * `logger.error("failed")` are the same to this check. What is left is the identity the body - * resolves, which the callee names do carry. It answers a related question rather than the - * original one, and its evidence overlaps heavily with `auth-boundary`: see the task 5 report. + * Everything the platform attaches centrally is already accounted for, which is what makes this + * worth asking. `logger` pushes the http context (requestId, path, host, method) onto every line + * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a + * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is + * `logBoundaryError(message, error, url)`, which is a url and an error. So a builder-wrapped route + * is not attributed either and gets no free pass here. An incident tells you which route and which + * request; whose environment it was is only ever in the fields the route passes itself. + * + * Applicable only where the route reports a failure itself, meaning it logs from inside a catch. A + * route that rethrows silently has handed the report to Sentry and there is nothing here to + * inspect. That gate has a perverse edge, noted in the task 5 report: deleting a log line moves a + * route from fail to not-applicable. */ export const requestContext = { id: ID, run(ep: EntryPoint): CheckResult { - if (isTrivial(ep)) { - return { id: ID, status: "not-applicable", detail: "trivial route" }; - } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "attributed by the builder" }; + const logs = failurePathLogs(ep); + if (logs.length === 0) { + return { id: ID, status: "not-applicable", detail: "reports no failure of its own" }; } - if (resolvesRequestIdentity(ep)) { - return { id: ID, status: "pass", detail: "resolves a request identity" }; + const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + if (named) { + const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); + return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } return { id: ID, status: "fail", - detail: "resolves no user, environment or project, so a failure here names nobody", + detail: "logs its failure without naming an environment, project, organization, run or user", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 57510e656ec..cf48a61a21c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -62,10 +62,8 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - // The brief expected a pass here. `EntryPoint` carries `hasTryCatch` and nothing about what the - // catch does with the error, so the check cannot see the rethrow, and reading it out of - // `ep.source` would read the whole file. Known false positive, see the task 5 report. - it("also fails a raw route whose catch branches on the error", () => { + // Restored from the brief: `catchBranches` sees the `instanceof` and the `if`. + it("passes a raw route whose catch branches on the error", () => { const r = run( "error-classification", "api.v1.z.ts", @@ -75,6 +73,39 @@ describe("error-classification", () => { catch (e) { if (e instanceof NotFound) return null; throw e; } }` ); + expect(r.status).toBe("pass"); + }); + + it("passes a raw route whose catch rethrows without branching", () => { + const r = run( + "error-classification", + "api.v1.v.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } + }` + ); + expect(r.status).toBe("pass"); + }); + + // The builder only classifies what reaches it. A swallow inside the handler never does, so the + // swallow is read before the builder is credited. + it("fails a builder-wrapped route whose handler swallows", () => { + const r = run( + "error-classification", + "api.v2.runs.$runParam.cancel.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { CancelTaskRunService } from "~/services/cancelTaskRun.server"; + const { action } = createActionApiRoute({}, async ({ params }) => { + const service = new CancelTaskRunService(); + try { await service.call(params.runParam); } + catch { return json({ error: "Internal Server Error" }, { status: 500 }); } + return json({ ok: true }); + }); + export { action };` + ); expect(r.status).toBe("fail"); }); @@ -204,46 +235,55 @@ describe("auth-boundary", () => { }); describe("request-context", () => { - it("passes a builder-wrapped route", () => { + it("passes a route whose failure log names an identifier", () => { const r = run( "request-context", - "api.v1.q.ts", - `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + "engine.v1.dev.config.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export const loader = createLoaderApiRoute({}, async () => json(await prisma.thing.findMany()));` + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("dev config failed", { environmentId: params.envId, error }); + throw error; + } + }` ); expect(r.status).toBe("pass"); }); - it("passes a raw route that resolves the request identity", () => { + it("passes on a route param, which names the tenant just as well", () => { const r = run( "request-context", - "api.v1.q.ts", - `import { authenticateApiRequest } from "~/services/apiAuth.server"; + "resources.things.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ request }) { - const auth = await authenticateApiRequest(request); - return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { organizationSlug: params.organizationSlug, error }); + throw error; + } }` ); expect(r.status).toBe("pass"); }); - it("passes a raw route that resolves a tenant without authenticating", () => { + it("fails a failure log that carries only the error", () => { const r = run( "request-context", - "resources.things.ts", - `import { loadProjectEnvironmentFromRequest } from "~/services/environment.server"; + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ request, params }) { - const environment = await loadProjectEnvironmentFromRequest(request, params); - return json(await prisma.thing.findMany({ where: { environmentId: environment.id } })); + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { error }); throw error; } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); - it("fails a raw route that resolves nothing about the request", () => { + it("fails a bare failure log with no object argument at all", () => { const r = run( "request-context", "api.v1.r.ts", @@ -256,51 +296,79 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - // The brief expected a pass here, on the strength of the identifier in the log call. `EntryPoint` - // records callee names, not their arguments, so that identifier is not body-scoped evidence and - // the check cannot see it. Known false positive, see the task 5 report. - it("also fails a raw route that only names an identifier in a log call", () => { + // The builder logs `{ error, url }` at its boundary and nothing that names a tenant, so being + // wrapped in one earns no pass here. This is what stops the check echoing `auth-boundary`. + it("fails a builder-wrapped route whose own failure log names nobody", () => { const r = run( "request-context", "api.v1.q.ts", - `import { logger } from "~/services/logger.server"; + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function loader({ params }) { - try { return await prisma.thing.findMany(); } - catch (e) { logger.error("failed", { environmentId: params.envId }); throw e; } - }` + export const loader = createLoaderApiRoute({}, async () => { + try { return json(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + });` ); expect(r.status).toBe("fail"); }); - // False positive fixture: identifiers all over the component, none in the loader. The check must - // read the loader, so the verdict has to come from the loader's own call to requireUserId. - it("does not flag a route whose identifiers are in the React component", () => { + it("is not applicable to a route that logs nothing on its failure path", () => { const r = run( "request-context", - "_app.orgs.$organizationSlug.things/route.tsx", - `import { requireUserId } from "~/services/session.server"; + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; import { prisma } from "~/db.server"; export async function loader({ request }) { - const userId = await requireUserId(request); - return typedjson(await prisma.thing.findMany({ where: { userId } })); - } - ${COMPONENT}` + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); - it("fails a route whose identifiers are only in the React component", () => { + it("is not applicable when the only log sits outside the catch", () => { const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + logger.info("starting", { environmentId: "env_1" }); + try { return await prisma.thing.findMany(); } catch (e) { throw e; } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture: the component logs every identifier there is, inside its own catch. + // Only the loader's own failure log may decide this. + it("does not read the React component's log calls", () => { + const bare = run( "request-context", "_app.orgs.$organizationSlug.things/route.tsx", - `import { prisma } from "~/db.server"; + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; export async function loader() { - return typedjson(await prisma.thing.findMany()); + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } } ${COMPONENT}` ); - expect(r.status).toBe("fail"); + expect(bare.status).toBe("fail"); + + const attributed = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { projectParam: params.projectParam, error }); throw error; } + } + ${COMPONENT}` + ); + expect(attributed.status).toBe("pass"); }); it("is not applicable to a trivial redirect", () => { From 3b19436d649f390cecc496917cd200c2a098eb8d Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:35:29 +0100 Subject: [PATCH 010/117] feat(observability-map): flag entry points whose catches guard one operation error-classification cannot tell the deliberate narrow guard, e.g. try { body = await request.json() } catch { 400 }, from a catch that swallows the whole handler. Both take one path out. catchesNarrowly is true when an entry point has at least one catch clause and no try block with a catch holds more than two statements, counted in the loader/action bodies and the same one-hop helpers as the other fields. Every catch has to qualify: one broad catch anywhere makes it false, so a route that guards a JSON.parse and also wraps its handler is still reported. Two statements lets the guarded operation bind its result and stops short of the three-statement try that covers a handler. 55 of the 427 route entry points, and 11 of the 32 error-classification failures, all eleven hand-read as the deliberate idiom. No existing field changes value. --- .../observability-map/src/scan.ts | 18 +- .../observability-map/src/types.ts | 8 + .../observability-map/test/scan.test.ts | 173 ++++++++++++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 4702a7d7771..1cb69bcdc27 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -106,6 +106,13 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields return { found: false, fields: [] }; } +/** + * How much a try block may guard and still count as narrow. Two, so that the guarded operation can + * bind its result (`const stripped = ...; new RegExp(stripped);`) but a third statement means the + * try has started to cover the handler rather than one operation. + */ +const NARROW_TRY_STATEMENTS = 2; + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -457,6 +464,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; let catchRethrows = false; let catchBranches = false; + let catchClauseCount = 0; + let broadCatch = false; const calleeNames: string[] = []; const calleeTexts: string[] = []; const logCalls: LogCall[] = []; @@ -473,7 +482,13 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!fn.body) return; const visit = (node: ts.Node, inCatch: boolean) => { - if (ts.isTryStatement(node)) hasTryCatch = true; + if (ts.isTryStatement(node)) { + hasTryCatch = true; + if (node.catchClause) { + catchClauseCount += 1; + if (countStatements(node.tryBlock.statements) > NARROW_TRY_STATEMENTS) broadCatch = true; + } + } if (ts.isCatchClause(node)) { const evidence = catchClauseEvidence(node); @@ -533,6 +548,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { hasTryCatch, catchRethrows, catchBranches, + catchesNarrowly: catchClauseCount > 0 && !broadCatch, logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 15b8d581132..5d7dc2292ae 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -49,6 +49,14 @@ export type EntryPoint = { * entry point takes one path out regardless of what was thrown. */ catchBranches: boolean; + /** + * Whether every catch in those bodies guards a specific operation rather than the handler: the + * entry point has at least one catch clause, and no try block with a catch holds more than two + * statements. The `try { body = await request.json() } catch { 400 }` idiom, which takes one path + * out and is still deliberate. False when any catch wraps the bulk of a body, and false when + * there is no catch clause at all. + */ + catchesNarrowly: boolean; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index ca92dd26182..943c4a56f3e 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -852,3 +852,176 @@ describe("scanFile: log calls", () => { expect(ep!.logCalls).toEqual([]); }); }); + +describe("scanFile: narrow catches", () => { + it("flags a try that guards a single request.json()", () => { + const ep = scanFile( + "admin.api.v1.platform-notifications.ts", + ` + export async function action({ request }) { + const user = await requireUser(request); + let body; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const result = await createPlatformNotification(body); + return json(result); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("does not flag a catch wrapping the whole handler", () => { + const ep = scanFile( + "otel.v1.logs.ts", + ` + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type") ?? ""; + const body = await request.json(); + await exporter.export(body); + return json({ ok: true }); + } catch (e) { + logger.error(e); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("does not flag a body that has both a narrow catch and a broad one", () => { + const ep = scanFile( + "mixed.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("allows a guarded operation with its own local binding", () => { + const ep = scanFile( + "regex.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + const stripped = pattern.startsWith("(?i)") ? pattern.slice(4) : pattern; + new RegExp(stripped); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("does not flag a try of three statements", () => { + const ep = scanFile( + "three.ts", + ` + export async function loader({ request }) { + try { + const raw = await request.json(); + const parsed = Schema.parse(raw); + return json(parsed); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("is false when there is no try at all", () => { + const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("is false for a try with a finally and no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("reads a narrow catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "helper-narrow.ts", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchesNarrowly).toBe(true); + }); + + it("ignores a narrow catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + JSON.parse(raw); + } catch { + return null; + } + return null; + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); +}); From 3cb31ccb4b63dd1e54de740949a9253aef78335f Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:47:58 +0100 Subject: [PATCH 011/117] fix(observability-map): stop request-context excusing routes that log nothing Applicability keyed off the presence of a failure-path log, so a route that kept its errors and recorded nothing was not-applicable rather than reported, and deleting a log line took a route out of the report. Every non-trivial entry point is now judged: no catch at all passes, since the error reaches the central handler, and a catch has to name whose failure it was. 87 of the 169 failures are routes that record nothing, which is what the old gate was hiding. Verified over the real tree that removing logging cannot help: re-running all four checks against every entry point with log calls deleted, failure-path logs deleted, and log fields stripped moves 63 verdicts, none of them for the better. error-classification now uses catchesNarrowly to excuse the guard that wraps a single parse. Applied on its own the field also excuses a one-statement try around a service call, which passes the design's own swallow fixture and four findings that were hand-read as real, so the exemption also asks that the body parse something. That clears the nine verbatim request.json guards and keeps the rest: 50 failures become 35. --- .../src/checks/errorClassification.ts | 22 ++++- .../src/checks/requestContext.ts | 33 +++++-- .../observability-map/test/checks.test.ts | 87 ++++++++++++++++++- 3 files changed, 129 insertions(+), 13 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index cfa8e389a82..caf18b3cded 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -24,6 +24,12 @@ export const BUILDERS = new Set([ "dashboardAction", ]); +/** + * A parse: `await request.json()`, `JSON.parse(raw)`. The dot matters, it keeps Remix's `json({})` + * response helper out. Matched against `calleeTexts`, which carries the whole callee path. + */ +const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; + export function usesBuilder(ep: EntryPoint): boolean { return ( (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || @@ -43,6 +49,16 @@ export function usesBuilder(ep: EntryPoint): boolean { * false means every catch in the entry point takes the same way out whatever was thrown. The * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses * those rather than inventing them. + * + * `catchesNarrowly` excuses the guard that wraps one operation and answers for that operation: + * `try { body = await request.json() } catch { 400 }` neither branches nor rethrows and does not + * need to. On its own it excuses too much, because a one-statement try around an awaited service + * call is exactly as narrow as one around a parse: applied unencumbered it passes + * `try { await service.call(run) } catch { 500 }`, and it passes the design's own swallow fixture, + * `try { return await prisma.thing.findMany() } catch { return null }`. So the exemption also asks + * that the body parse something, which is the idiom the exemption was justified by. Over the real + * tree that combination clears the nine verbatim `request.json()` guards and holds back the four + * hand-read findings, see the task 5 report. */ export const errorClassification = { id: ID, @@ -50,13 +66,17 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches) { + const guardsAParse = ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); + if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { return { id: ID, status: "fail", detail: "catches its errors and takes one way out regardless of what was thrown", }; } + if (guardsAParse) { + return { id: ID, status: "pass", detail: "guards a parse, not the handler" }; + } if (ep.catchRethrows || ep.catchBranches) { return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; } diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 248ec1374ff..c8694b5c168 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,4 +1,5 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; +import { isTrivial } from "../triviality.js"; const ID = "request-context"; @@ -25,18 +26,31 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * is not attributed either and gets no free pass here. An incident tells you which route and which * request; whose environment it was is only ever in the fields the route passes itself. * - * Applicable only where the route reports a failure itself, meaning it logs from inside a catch. A - * route that rethrows silently has handed the report to Sentry and there is nothing here to - * inspect. That gate has a perverse edge, noted in the task 5 report: deleting a log line moves a - * route from fail to not-applicable. + * Applicability turns on whether the route keeps its own failures, never on whether it logs. The + * first version made a route not-applicable when it had no failure-path log, which excused the very + * thing the check exists to find and meant deleting a log line took a route out of the report. + * Every non-trivial entry point is now judged: + * + * - no catch at all: pass. The error reaches the central handler, which is the intended path in + * this codebase, and the tenant it does not name there is a platform-level gap reported once + * rather than against each of 222 routes. + * - a catch: the route decided the outcome itself, so it has to say whose failure it was. + * + * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch + * without a report is a regression the check reports, which is the direction the incentive should + * run in. The one move that still improves a verdict is deleting the try/catch outright, and that + * hands the error back to the central handler, which `error-classification` also treats as correct. */ export const requestContext = { id: ID, run(ep: EntryPoint): CheckResult { - const logs = failurePathLogs(ep); - if (logs.length === 0) { - return { id: ID, status: "not-applicable", detail: "reports no failure of its own" }; + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (!ep.hasTryCatch) { + return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; + } + const logs = failurePathLogs(ep); const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); if (named) { const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); @@ -45,7 +59,10 @@ export const requestContext = { return { id: ID, status: "fail", - detail: "logs its failure without naming an environment, project, organization, run or user", + detail: + logs.length === 0 + ? "keeps its failures and records nothing about whose they were" + : "logs its failure without naming an environment, project, organization, run or user", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index cf48a61a21c..97fb21991e9 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -132,6 +132,47 @@ describe("error-classification", () => { expect(r.status).toBe("not-applicable"); }); + // A narrow guard around one operation classifies an expected failure without needing to branch + // or rethrow. `catchesNarrowly` is what tells it apart from a handler-wide catch. + it("passes a narrow guard around a single parse", () => { + const r = run( + "error-classification", + "resources.timezone.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + const saved = await prisma.preference.create({ data }); + return json({ saved }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad + // handler catch sitting next to it. `catchesNarrowly` is false when any guarded try is broad. + it("still fails when a narrow guard sits beside a handler-wide swallow", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + return json({ thing, audit }); + } catch (error) { + return json({ error: "Something went wrong" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not flag a route whose try/catch is in the React component", () => { const r = run( @@ -313,7 +354,7 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - it("is not applicable to a route that logs nothing on its failure path", () => { + it("passes a route that hands its failures to the central handler", () => { const r = run( "request-context", "api.v1.q.ts", @@ -324,10 +365,10 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); }` ); - expect(r.status).toBe("not-applicable"); + expect(r.status).toBe("pass"); }); - it("is not applicable when the only log sits outside the catch", () => { + it("fails a route that catches but only names an identifier outside the catch", () => { const r = run( "request-context", "api.v1.q.ts", @@ -338,7 +379,45 @@ describe("request-context", () => { try { return await prisma.thing.findMany(); } catch (e) { throw e; } }` ); - expect(r.status).toBe("not-applicable"); + expect(r.status).toBe("fail"); + }); + + // A route that catches and reports nothing at all is the case the log-based applicability gate + // used to excuse. It is a finding, not an exemption. + it("fails a route that catches and reports nothing", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { return json({ error: "Internal Server Error" }, { status: 500 }); } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. + // Deleting that line must never improve the verdict or drop the route out of the report. + it("never improves a verdict when the log call is deleted", () => { + const body = (log: string) => + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { ${log} throw error; } + }`; + + const withLog = run( + "request-context", + "api.v1.q.ts", + body(`logger.error("failed", { environmentId: params.envId, error });`) + ); + const withoutLog = run("request-context", "api.v1.q.ts", body("")); + + expect(withLog.status).toBe("pass"); + expect(withoutLog.status).toBe("fail"); + expect(withoutLog.status).not.toBe("not-applicable"); }); // False positive fixture: the component logs every identifier there is, inside its own catch. From ed52e17277d05ea4a81a7a026a63a805d6b5be36 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 14:56:07 +0100 Subject: [PATCH 012/117] feat(observability-map): add scoring and aggregation with an audit gap and unmeasured tracking --- .../observability-map/src/score.ts | 116 ++++++++++++++++++ .../observability-map/test/score.test.ts | 109 ++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 internal-packages/observability-map/src/score.ts create mode 100644 internal-packages/observability-map/test/score.test.ts diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts new file mode 100644 index 00000000000..542b968177c --- /dev/null +++ b/internal-packages/observability-map/src/score.ts @@ -0,0 +1,116 @@ +import type { CheckResult, EntryPoint } from "./types.js"; +import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; +import { suppressedChecks } from "./suppression.js"; +import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; +import { classifySensitivity } from "./sensitivity.js"; + +export type ScoredEntry = { + fileName: string; + routePath: string; + family: Family; + sensitive: boolean; + checks: CheckResult[]; + /** + * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable. + * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a + * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that + * default cannot inflate a figure nobody checked. + */ + measured: boolean; + /** Passed over applicable, across scored checks only. 100 when nothing applies. */ + score: number; +}; + +export type MapReport = { + global: number; + /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ + measured: number; + /** Entry points every scored check reported not-applicable for; excluded from `global`. */ + unmeasured: number; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number }; + auditGap: { sensitiveMutations: number; withAudit: number }; + entries: ScoredEntry[]; + parseFailures: string[]; +}; + +export function scoreEntry(ep: EntryPoint): ScoredEntry { + const suppressed = suppressedChecks(ep.source); + const checks = CHECKS.map((c) => { + const result = c.run(ep); + const reason = suppressed.get(c.id); + // A suppression always lands on not-applicable, never on pass: suppressing a check must remove + // it from the score, not launder it into a point in the entry's favor. + return reason + ? { id: c.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + : result; + }); + + const scored = checks.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const applicable = scored.filter((c) => c.status !== "not-applicable"); + const passed = applicable.filter((c) => c.status === "pass").length; + + return { + fileName: ep.fileName, + routePath: routePathOf(ep.fileName), + family: familyOf(ep.fileName), + sensitive: classifySensitivity(ep).sensitive, + checks, + measured: applicable.length > 0, + score: applicable.length === 0 ? 100 : Math.round((passed / applicable.length) * 100), + }; +} + +const mean = (xs: number[]) => + xs.length === 0 ? 100 : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); + +/** + * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an + * entry point nothing applied to cannot drag a family's or cohort's figure toward 100. `measured` + * is reported alongside so a reader can tell a family scoring high because it is clean apart from + * a family scoring high because most of it was never measured. + */ +function groupStats(entries: ScoredEntry[]): { n: number; measured: number; mean: number } { + const measuredEntries = entries.filter((e) => e.measured); + return { + n: entries.length, + measured: measuredEntries.length, + mean: mean(measuredEntries.map((e) => e.score)), + }; +} + +export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapReport { + const entries = eps.map(scoreEntry); + const measuredEntries = entries.filter((e) => e.measured); + + const byFamily: Record = {}; + for (const family of new Set(entries.map((e) => e.family))) { + byFamily[family] = groupStats(entries.filter((e) => e.family === family)); + } + + const sensitive = entries.filter((e) => e.sensitive); + + // audit-trail is excluded from the score (see checks/index.ts and scoreEntry above), and is + // reported here as its own architectural figure instead: how many sensitive mutations have an + // audit record, out of how many. Folding it into the score would tank every sensitive route on a + // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + const auditApplicable = entries.filter((e) => + e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + ); + + return { + global: mean(measuredEntries.map((e) => e.score)), + measured: measuredEntries.length, + unmeasured: entries.length - measuredEntries.length, + byFamily, + sensitiveCohort: groupStats(sensitive), + auditGap: { + sensitiveMutations: auditApplicable.length, + withAudit: auditApplicable.filter((e) => + e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") + ).length, + }, + entries, + parseFailures, + }; +} diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts new file mode 100644 index 00000000000..ef27f5473c4 --- /dev/null +++ b/internal-packages/observability-map/test/score.test.ts @@ -0,0 +1,109 @@ +import { scoreEntry, buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const BUILDER = `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +export const loader = createLoaderApiRoute({}, async () => new Response("ok"));`; + +const RAW = `import { prisma } from "~/db.server"; +export async function loader() { return prisma.thing.findMany(); }`; + +/** Trivial and not sensitive: every scored check reports not-applicable. */ +const TRIVIAL = `export const loader = () => new Response("ok");`; + +/** Not trivial (touches prisma, has a try/catch) and not sensitive: swallows every error and + * records nothing about whose failure it was, so both applicable scored checks fail. */ +const BUSY_AND_FAILING = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + +describe("scoreEntry", () => { + it("scores a builder route 100", () => { + expect(scoreEntry(scanFile("api.v1.a.ts", BUILDER)!).score).toBe(100); + }); + + it("excludes audit-trail from the per-entry score", () => { + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", BUILDER)!); + expect(scored.checks.some((c) => c.id === "audit-trail")).toBe(true); + expect(scored.score).toBe(100); + }); + + it("counts a suppressed check as not-applicable", () => { + const suppressed = `// obs-map-disable-next-line error-classification -- health probe +${RAW}`; + const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); + const ec = scored.checks.find((c) => c.id === "error-classification")!; + expect(ec.status).toBe("not-applicable"); + }); + + it("a suppressed-to-passing entry point does not read as unmeasured", () => { + // error-classification would fail here; auth-boundary is not-applicable (not sensitive). + // Suppressing the only applicable scored check must not be indistinguishable from an entry + // point nothing applies to: it is still reported, just not scored on that axis. + const suppressed = `// obs-map-disable-next-line error-classification -- health probe +${BUSY_AND_FAILING}`; + const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + }); + + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { + const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); + expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); + expect(scored.measured).toBe(false); + expect(scored.score).toBe(100); + }); + + it("marks an entry point with at least one applicable scored check as measured", () => { + const scored = scoreEntry(scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!); + expect(scored.measured).toBe(true); + }); +}); + +describe("buildReport", () => { + it("reports the audit gap separately from the score", () => { + const eps = [ + scanFile("api.v1.a.ts", BUILDER)!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ]; + const report = buildReport(eps, []); + expect(report.auditGap.sensitiveMutations).toBe(1); + expect(report.auditGap.withAudit).toBe(0); + expect(report.global).toBeGreaterThan(0); + }); + + it("records parse failures", () => { + const report = buildReport([scanFile("api.v1.a.ts", BUILDER)!], ["broken.ts"]); + expect(report.parseFailures).toEqual(["broken.ts"]); + }); + + it("excludes an unmeasured entry point from the global mean", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + expect(report.measured).toBe(1); + expect(report.unmeasured).toBe(1); + // If the trivial entry's vacuous 100 counted toward the mean, the global score would be 50 + // instead of matching the one entry that was actually measured. + expect(report.global).toBe(scoreEntry(busy).score); + }); + + it("excludes an unmeasured entry point from its family mean too", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("resources.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + const family = report.byFamily["resources"]!; + expect(family.n).toBe(2); + expect(family.measured).toBe(1); + expect(family.mean).toBe(scoreEntry(busy).score); + }); +}); From f3b582fd7cf3ef76851f2f5142b7df76587fe7f8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:06:05 +0100 Subject: [PATCH 013/117] feat(observability-map): terminal and json reports --- .../observability-map/src/report/json.ts | 5 + .../observability-map/src/report/terminal.ts | 76 ++++++++++++ .../observability-map/test/report.test.ts | 112 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 internal-packages/observability-map/src/report/json.ts create mode 100644 internal-packages/observability-map/src/report/terminal.ts create mode 100644 internal-packages/observability-map/test/report.test.ts diff --git a/internal-packages/observability-map/src/report/json.ts b/internal-packages/observability-map/src/report/json.ts new file mode 100644 index 00000000000..8a36465abd5 --- /dev/null +++ b/internal-packages/observability-map/src/report/json.ts @@ -0,0 +1,5 @@ +import type { MapReport } from "../score.js"; + +export function renderJson(report: MapReport): string { + return JSON.stringify(report, null, 2); +} diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts new file mode 100644 index 00000000000..c5d19e491f9 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -0,0 +1,76 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { SCORED_CHECK_IDS } from "../checks/index.js"; + +const bar = (score: number) => { + const filled = Math.round(score / 10); + return "▰".repeat(filled) + "▱".repeat(10 - filled); +}; + +/** + * Failing checks that actually feed `score`. `audit-trail` is deliberately excluded here: it is + * excluded from the score for the same reason (see `score.ts`), and every sensitive mutation fails + * it today, so folding it in would flood this list with the same finding repeated 52 times instead + * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as + * `AUDIT`, below. + */ +const scoredFailures = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); + +export function renderTerminal(report: MapReport): string { + const lines: string[] = []; + + lines.push( + `score ${report.global}/100 ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` + ); + lines.push(""); + lines.push("COVERAGE"); + for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { + lines.push( + ` ${family.padEnd(12)} ${bar(stats.mean)} ${String(stats.mean).padStart(3)} ${stats.measured}/${stats.n} entry points` + ); + } + lines.push( + ` ${"sensitive".padEnd(12)} ${bar(report.sensitiveCohort.mean)} ${String( + report.sensitiveCohort.mean + ).padStart(3)} ${report.sensitiveCohort.measured}/${report.sensitiveCohort.n} entry points` + ); + + lines.push(""); + const { sensitiveMutations, withAudit } = report.auditGap; + lines.push( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `No audit helper exists in the webapp.` + ); + + const worst = report.entries + .filter((e) => scoredFailures(e).length > 0) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + + lines.push(""); + lines.push("FIX FIRST"); + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + ` ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + lines.push(` ${e.fileName}`); + } + if (worst.length > 3) { + lines.push(""); + lines.push(`THEN ${worst.length - 3} more with gaps`); + } + + lines.push(""); + lines.push(`already solid: ${report.entries.length - worst.length}`); + if (report.parseFailures.length > 0) { + lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); + } + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts new file mode 100644 index 00000000000..be977d24d8f --- /dev/null +++ b/internal-packages/observability-map/test/report.test.ts @@ -0,0 +1,112 @@ +import { renderTerminal } from "../src/report/terminal.js"; +import { renderJson } from "../src/report/json.js"; +import { buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const report = () => + buildReport( + [ + scanFile( + "api.v1.a.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + )!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ], + ["broken.ts"] + ); + +describe("renderTerminal", () => { + it("shows the global score, the audit gap and the fix list", () => { + const out = renderTerminal(report()); + expect(out).toContain("COVERAGE"); + expect(out).toContain("FIX FIRST"); + expect(out).toContain("audit"); + expect(out).toContain("/api/v1/auth/tokens"); + }); + + it("surfaces parse failures so the denominator is not silently wrong", () => { + expect(renderTerminal(report())).toContain("broken.ts"); + }); + + it("shows the unmeasured count so 415 vs 427 is not silently confusing", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + expect(out).toContain("1 unmeasured"); + }); + + it("orders FIX FIRST by sensitivity first, then ascending score, and excludes audit-only gaps", () => { + // Sensitive, score 0: both applicable scored checks fail (auth-boundary, error-classification, + // request-context all fail because the catch swallows without naming who it happened to). + const sensitiveZero = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + + // Sensitive, score 67: only auth-boundary fails (no guard called), the other two pass because + // there is no try/catch to fumble. + const sensitiveSixtySeven = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!; + + // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both + // sensitive entries because sensitivity outranks raw score. + const notSensitiveZero = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + // Sensitive mutation, scored checks all pass (guarded, no try/catch): the only gap is + // audit-trail, which is a headline figure, not a per-route fix-list item. Must not appear. + const sensitiveAuditOnly = scanFile( + "api.v1.billing.ts", + `import { prisma } from "~/db.server"; + import { requireUserId } from "~/services/session.server"; + export async function action({ request }: { request: Request }) { + const userId = requireUserId(request); + return prisma.billing.update({ where: { userId }, data: {} }); + }` + )!; + + const out = renderTerminal( + buildReport([sensitiveSixtySeven, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + ); + + const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); + const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); + const idxSixtySeven = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); + + expect(idxZero).toBeGreaterThan(-1); + expect(idxSixtySeven).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxSixtySeven); + expect(fixFirst).not.toContain("api.v1.billing.ts"); + }); +}); + +describe("renderJson", () => { + it("round-trips to an object carrying the score and entries", () => { + const parsed = JSON.parse(renderJson(report())); + expect(typeof parsed.global).toBe("number"); + expect(Array.isArray(parsed.entries)).toBe(true); + }); +}); From 718397645e3777f0f01e884bb52a3651c9e7b3a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:19:54 +0100 Subject: [PATCH 014/117] fix(observability-map): stop auth-boundary accusing routes it cannot see into The rendered fix list opened with three auth-boundary findings and all three were wrong. Two delegate to clearImpersonation, which authenticates and writes an audit row in a file the scanner never opens, and the third is a redirect stub flagged only because its path contains billing. A fail here says the route does privileged work with no guard, which is only supportable when the body is where a guard would have to be. A trivial body cannot hold a visible privileged operation, by the triviality rule's own definition, so either nothing privileged happens or the work sits behind an import along with any guard. Those now report not-applicable with a detail saying the guard could not be verified, rather than failing. Signature checks also count as guards now, which clears the HMAC-authenticated waitpoint callback. Three findings remain and all three are genuinely unauthenticated. request-context stops treating a parse guard as the route taking over its failure path, through the same shared reading of catchesNarrowly that error-classification uses. Re-ran the incentive sweep after the change: 57 verdicts move when logging is removed, none for the better. --- .../src/checks/authBoundary.ts | 53 ++++++-- .../src/checks/errorClassification.ts | 12 +- .../src/checks/requestContext.ts | 16 ++- .../observability-map/test/checks.test.ts | 128 ++++++++++++++++++ 4 files changed, 195 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 11030d1d2d8..dfe38ed1fed 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -1,24 +1,54 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; +import { isTrivial } from "../triviality.js"; import { usesBuilder } from "./errorClassification.js"; const ID = "auth-boundary"; /** - * The two shapes the webapp's guards take: `requireUserId`, `requireAdminApiRequest`, - * `authenticateApiRequest`, `authenticateProjectApiKey`. Matched against `calleeNames`, which is + * What a guard looks like from the body. All three are matched against `calleeNames`, which is * scoped to the loader/action bodies and follows one hop into a same-file helper, so a guard the * route only imports and never calls does not count. */ -const GUARD = /^(require|authenticate)/; +const GUARDS = [ + /** `requireUserId`, `requireAdminApiRequest`, `authenticateApiRequest`. */ + /^(require|authenticate)/, + /** + * Proof of possession. A callback URL carrying an HMAC is authenticated by checking that HMAC: + * `verifyHttpCallbackHash`, `verifyWebhookSignature`. Narrowed to the signature words so that an + * unrelated `verifyEmail` does not read as an auth boundary. + */ + /^verify.*(Hash|Hmac|Signature|Webhook|Callback|Token)/, + /** `resolveAuthenticatedEnv`: the name says the identity was established. */ + /Authenticated/, +]; /** * Whether a route that handles credentials, tokens or money checks who is asking. * - * The design matched `importedNames` as well as `calleeNames`. Across the 67 sensitive entry points - * in the real tree that widening changes nothing: every route with a `require*` import calls it - * from the body too. So the file-wide half only ever stood to hand out a pass for a dead import, - * and it is gone. + * A fail here is an accusation, and it is only supportable when the body is the place a guard + * would have to be. That holds when the route does its privileged work in the open: reads the + * request, queries the datastore, mints the token. It does not hold for a trivial body, so those + * are reported not-applicable rather than failed. + * + * The reasoning is the triviality rule's own definition rather than a convenience. A trivial body + * has three statements or fewer, three calls or fewer, no try/catch, no builder, and no mention of + * prisma, redis, fetch or the engine anywhere in its source. It therefore cannot contain a visible + * privileged operation. Either it does nothing privileged at all, like the `/orgs/:slug/billing` + * redirect stub, or the privileged work sits behind an import, like `clearImpersonation`, which + * authenticates and writes an audit row in `app/models/admin.server.ts`. In the second case the + * guard is in the same unopened file as the work. Absence of evidence, and reporting it as a + * finding puts a wrong answer at the top of the fix list. + * + * This is not the rule `request-context` uses, deliberately. There the thing being looked for, a + * field on a log call inside a catch, would be in the body if it existed at all, because the catch + * is in the body. Absence of a log is evidence. Here the thing being looked for guards work that + * is not in the body either, so its absence proves nothing. The test that separates them: would + * this evidence necessarily be visible in the body if it existed? + * + * The design also matched `importedNames`. Across the 67 sensitive entry points that widening + * changes nothing, every route with a `require*` import calls it from the body too, so the + * file-wide half only ever stood to hand out a pass for a dead import. It is gone. */ export const authBoundary = { id: ID, @@ -30,9 +60,16 @@ export const authBoundary = { if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "authenticated by the builder" }; } - if (ep.calleeNames.some((n) => GUARD.test(n))) { + if (ep.calleeNames.some((n) => GUARDS.some((g) => g.test(n)))) { return { id: ID, status: "pass", detail: "guarded in the body" }; } + if (isTrivial(ep)) { + return { + id: ID, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", + }; + } return { id: ID, status: "fail", diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index caf18b3cded..7320e60d092 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -30,6 +30,16 @@ export const BUILDERS = new Set([ */ const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; +/** + * Whether every catch in the entry point guards a parse and nothing wider. Both checks read the + * field the same way: a guard around one parse is not the route taking charge of its failures, + * so `error-classification` does not call it a swallow and `request-context` does not ask it to + * name a tenant. + */ +export function guardsOnlyAParse(ep: EntryPoint): boolean { + return ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); +} + export function usesBuilder(ep: EntryPoint): boolean { return ( (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || @@ -66,7 +76,7 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const guardsAParse = ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); + const guardsAParse = guardsOnlyAParse(ep); if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { return { id: ID, diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index c8694b5c168..116df0cc193 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,5 +1,6 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; import { isTrivial } from "../triviality.js"; +import { guardsOnlyAParse } from "./errorClassification.js"; const ID = "request-context"; @@ -31,10 +32,13 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * thing the check exists to find and meant deleting a log line took a route out of the report. * Every non-trivial entry point is now judged: * - * - no catch at all: pass. The error reaches the central handler, which is the intended path in - * this codebase, and the tenant it does not name there is a platform-level gap reported once - * rather than against each of 222 routes. - * - a catch: the route decided the outcome itself, so it has to say whose failure it was. + * - no catch at all, or nothing but a parse guard: pass. The route's own work still throws past it + * to the central handler, which is the intended path in this codebase, and the tenant that + * handler does not name is a platform-level gap reported once rather than against each route. A + * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure + * path, and reading it as one put false positives at the top of the first rendered report. + * - a catch that covers the route's work: it decided the outcome itself, so it has to say whose + * failure it was. * * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch * without a report is a regression the check reports, which is the direction the incentive should @@ -47,7 +51,9 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (!ep.hasTryCatch) { + // A route whose only catch guards a parse still throws its own work past it to the central + // handler. Same reading error-classification gives the field. + if (!ep.hasTryCatch || guardsOnlyAParse(ep)) { return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; } const logs = failurePathLogs(ep); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 97fb21991e9..b04f7daa183 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -242,6 +242,73 @@ describe("auth-boundary", () => { expect(r.status).toBe("not-applicable"); }); + // False positive fixture for the delegated guard. `clearImpersonation` authenticates and writes + // an audit row, in `app/models/admin.server.ts`, which the scanner cannot open. The body shows no + // privileged work either, so there is nothing here to accuse: absence of evidence, not evidence + // of absence. + it("does not flag a sensitive route that hands its work to an imported helper", () => { + const r = run( + "auth-boundary", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + return clearImpersonation(request, "/admin"); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toMatch(/verif/i); + }); + + it("does not flag a sensitive redirect stub", () => { + const r = run( + "auth-boundary", + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The gate must not swallow the real thing: a body doing its own privileged work, unguarded. + it("still fails a sensitive route whose visible body does the work unguarded", () => { + const r = run( + "auth-boundary", + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function action({ request }) { + const body = await request.json(); + const code = await prisma.authorizationCode.findFirst({ where: { code: body.code } }); + if (!code) return json({ error: "Not found" }, { status: 404 }); + const token = await createPersonalAccessToken(code.userId); + return json({ token }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Possession of a valid signature is the auth boundary for a callback URL. + it("passes a sensitive callback guarded by a signature check", () => { + const r = run( + "auth-boundary", + "api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts", + `import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + const waitpoint = await prisma.waitpoint.findFirst({ where: { id: params.id } }); + if (!verifyHttpCallbackHash(params.hash, waitpoint)) { + return json({ error: "Invalid" }, { status: 401 }); + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + // False positive fixture: the guard sits one hop away, in a same-file helper. it("does not flag a sensitive route whose guard is in a same-file helper", () => { const r = run( @@ -397,6 +464,67 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); + // A guard around a parse is not the route taking over its failure path: whatever its real work + // throws still reaches the central handler. Same reading error-classification gives the field. + it("passes a route whose only catch guards a parse", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + return json(await prisma.thing.findMany({ where: body })); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Known false positive, named in the task 5 report. The only catch here guards `new URL()`, a + // constructor, which never reaches `calleeTexts`, so the parse cannot be seen and the route is + // judged as though it kept its failures. Fixing it needs the scanner, not the check. + it("fails a route whose only catch guards a constructor parse", () => { + const r = run( + "request-context", + "_app.@.orgs.$organizationSlug.$.tsx", + `import { prisma } from "~/db.server"; + function refererOrigin(request) { + const referer = request.headers.get("referer"); + try { return new URL(referer).origin; } + catch { return undefined; } + } + export async function loader({ request }) { + const origin = refererOrigin(request); + return typedjson({ origin, things: await prisma.thing.findMany() }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still judges a route with a handler-wide catch beside a narrow guard", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const rows = await prisma.thing.findMany({ where: body }); + const count = await prisma.thing.count(); + return json({ rows, count }); + } catch (error) { + logger.error("failed", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. // Deleting that line must never improve the verdict or drop the route out of the report. it("never improves a verdict when the log call is deleted", () => { From a8413e1840c79a45d3cdace392804d4085373572 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:26:34 +0100 Subject: [PATCH 015/117] feat(observability-map): record catch evidence per clause Whole-entry catch booleans collapse when a route has a narrow parse guard and a broad handler catch, so a check cannot reason about either. 17 route entry points are in that state. - catches: one CatchEvidence per catch clause in the bodies and the one-hop helpers, carrying narrow, rethrows, branches, guardsParse and the try block statement count - guardsParse reads constructors as well as parse calls, so new URL(referer) is visible without touching calleeTexts, which other checks read - a try/finally now yields an empty catches list. hasTryCatch keeps its meaning, a try appears, so ask catches.length whether anything is caught - catchRethrows, catchBranches and catchesNarrowly are now derived from the list and keep their values on all 427 route entry points 242 catch clauses over 189 entry points, 9 of them swallowing outright. Clears all three false positives at the top of the report. --- .../observability-map/src/scan.ts | 55 +++-- .../observability-map/src/types.ts | 33 ++- .../observability-map/test/scan.test.ts | 221 ++++++++++++++++++ 3 files changed, 294 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 1cb69bcdc27..0028d55bab8 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -1,7 +1,7 @@ import ts from "typescript"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import type { EntryPoint, LogCall } from "./types.js"; +import type { CatchEvidence, EntryPoint, LogCall } from "./types.js"; /** Thrown by `scanFile` when the source does not parse cleanly. */ export class ParseFailureError extends Error { @@ -113,6 +113,32 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields */ const NARROW_TRY_STATEMENTS = 2; +/** + * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover + * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is + * the remix response helper, which every route calls and which parses nothing. + */ +const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; + +/** + * Whether the guarded region parses or constructs something. A constructor counts: `new URL(x)` is + * the commonest parse guard in the tree, and constructors are absent from `calleeTexts`. + */ +function guardsParse(tryBlock: ts.Block): boolean { + let found = false; + const visit = (node: ts.Node) => { + if (found) return; + if (ts.isNewExpression(node)) found = true; + if (ts.isCallExpression(node)) { + const text = calleeText(node.expression) ?? calleeName(node.expression); + if (text !== null && PARSE_CALLEE.test(text)) found = true; + } + ts.forEachChild(node, visit); + }; + visit(tryBlock); + return found; +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -462,10 +488,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; - let catchRethrows = false; - let catchBranches = false; - let catchClauseCount = 0; - let broadCatch = false; + const catches: CatchEvidence[] = []; const calleeNames: string[] = []; const calleeTexts: string[] = []; const logCalls: LogCall[] = []; @@ -485,15 +508,19 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (ts.isTryStatement(node)) { hasTryCatch = true; if (node.catchClause) { - catchClauseCount += 1; - if (countStatements(node.tryBlock.statements) > NARROW_TRY_STATEMENTS) broadCatch = true; + const tryStatementCount = countStatements(node.tryBlock.statements); + const clause = catchClauseEvidence(node.catchClause); + catches.push({ + narrow: tryStatementCount <= NARROW_TRY_STATEMENTS, + rethrows: clause.rethrows, + branches: clause.branches, + guardsParse: guardsParse(node.tryBlock), + tryStatementCount, + }); } } if (ts.isCatchClause(node)) { - const evidence = catchClauseEvidence(node); - catchRethrows ||= evidence.rethrows; - catchBranches ||= evidence.branches; ts.forEachChild(node, (child) => visit(child, true)); return; } @@ -546,9 +573,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { calleeNames, calleeTexts, hasTryCatch, - catchRethrows, - catchBranches, - catchesNarrowly: catchClauseCount > 0 && !broadCatch, + catches, + // Kept as aggregates of `catches` so the checks can migrate one at a time. + catchRethrows: catches.some((c) => c.rethrows), + catchBranches: catches.some((c) => c.branches), + catchesNarrowly: catches.length > 0 && catches.every((c) => c.narrow), logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 5d7dc2292ae..2e2401c87a4 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -6,6 +6,28 @@ export type CheckResult = { detail?: string; }; +/** + * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause + * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays + * legible instead of collapsing into one boolean. + */ +export type CatchEvidence = { + /** The guarded try block holds at most two statements: one operation, not the handler. */ + narrow: boolean; + /** The clause contains a `throw`. */ + rethrows: boolean; + /** The clause branches on the error: an `if`, a `switch`, or an `instanceof`. */ + branches: boolean; + /** + * The guarded region parses or constructs something: `JSON.parse`, `request.json()`, a zod + * `parse`/`safeParse`, or any `new X(...)`. Constructors are counted here because `new URL(x)` is + * the commonest parse guard in the tree and constructors never appear in `calleeTexts`. + */ + guardsParse: boolean; + /** Statements in the guarded try block, counted as `statementCount` counts them. */ + tryStatementCount: number; +}; + /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ export type LogCall = { /** Full callee path, e.g. `logger.error`. */ @@ -36,11 +58,18 @@ export type EntryPoint = { * something unnameable (`new PromptService().createOverride`) falls back to the bare name. */ calleeTexts: string[]; - /** Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. */ + /** + * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note + * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and + * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. + */ hasTryCatch: boolean; + /** One entry per catch clause in those bodies, in source order. */ + catches: CatchEvidence[]; /** * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided - * the error is not its to answer, which is a different act from swallowing it. + * the error is not its to answer, which is a different act from swallowing it. Aggregate of + * `catches`, kept so existing consumers keep working. */ catchRethrows: boolean; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 943c4a56f3e..e4e2388fd35 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1025,3 +1025,224 @@ describe("scanFile: narrow catches", () => { expect(ep!.catchesNarrowly).toBe(false); }); }); + +describe("scanFile: per-catch evidence", () => { + it("records one entry per catch clause, keeping a narrow guard distinct from a broad catch", () => { + const ep = scanFile( + "two-catches.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches[0]).toEqual({ + narrow: true, + rethrows: false, + branches: false, + guardsParse: true, + tryStatementCount: 1, + }); + expect(ep!.catches[1]).toMatchObject({ + narrow: false, + guardsParse: false, + tryStatementCount: 4, + }); + // The aggregate still collapses, which is why the per-catch list exists. + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("leaves catches empty for a try/finally with no catch clause", () => { + const ep = scanFile( + "runs-replication.status.ts", + ` + export async function loader() { + const redis = createRedis(); + try { + for (const source of sources) { + const exists = await redis.exists(source.slotName); + leaders.set(source.id, exists === 1); + } + } finally { + await redis.quit(); + } + return json({}); + } + ` + ); + expect(ep!.catches).toEqual([]); + // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catchRethrows).toBe(false); + expect(ep!.catchBranches).toBe(false); + expect(ep!.catchesNarrowly).toBe(false); + }); + + it("sees a constructor as a guarded parse", () => { + const ep = scanFile( + "_app.@.orgs.$organizationSlug.$.tsx", + ` + function refererOrigin(request) { + const referer = request.headers.get("referer"); + if (!referer) return undefined; + try { + return new URL(referer).origin; + } catch { + return undefined; + } + } + export async function action({ request }) { + const origin = refererOrigin(request); + return json({ origin }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); + }); + + it("sees a parse in a try that grew past the narrowness threshold", () => { + const ep = scanFile( + "admin.api.v1.orgs.$organizationId.stream-basin.ts", + ` + export async function action({ request }) { + let parsed; + try { + const text = await request.text(); + const raw = text.length > 0 ? JSON.parse(text) : {}; + const result = BodySchema.safeParse(raw); + if (!result.success) { + return json({ ok: false }, { status: 400 }); + } + parsed = result.data; + } catch { + return json({ ok: false, error: "Invalid JSON body" }, { status: 400 }); + } + return json(parsed); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.narrow).toBe(false); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.tryStatementCount).toBe(6); + }); + + it("does not call a broad catch over database work a parse guard", () => { + const ep = scanFile( + "broad.ts", + ` + export async function loader({ params }) { + try { + const run = await prisma.run.findFirst({ where: { id: params.id } }); + const events = await prisma.event.findMany({ where: { runId: run.id } }); + await touch(run); + return json({ run, events }); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]).toEqual({ + narrow: false, + rethrows: false, + branches: false, + guardsParse: false, + tryStatementCount: 4, + }); + }); + + it("keeps rethrow and branch evidence per clause", () => { + const ep = scanFile( + "mixed-clauses.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) throw e; + return json({}, { status: 500 }); + } + } + export async function action({ request }) { + try { + return json(await save(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); + expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); + // Aggregates stay as they are: any clause sets them. + expect(ep!.catchRethrows).toBe(true); + expect(ep!.catchBranches).toBe(true); + }); + + it("includes a catch from a same-file helper and excludes one from the React component", () => { + const ep = scanFile( + "route.tsx", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + export default function Page() { + try { + render(); + } catch (e) { + throw e; + } + return null; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catchRethrows).toBe(false); + }); + + it("keeps the aggregates derivable from the per-catch list", () => { + const ep = scanFile( + "aggregate.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) return json({}, { status: 400 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catchRethrows).toBe(ep!.catches.some((c) => c.rethrows)); + expect(ep!.catchBranches).toBe(ep!.catches.some((c) => c.branches)); + expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); + }); +}); From 093d447d0dfd66a827ac45f5155cd770de092773 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:35:59 +0100 Subject: [PATCH 016/117] fix(observability-map): judge catch clauses one at a time Both checks now read EntryPoint.catches instead of the aggregate booleans, so an entry point is only as good as its worst catch. 39 routes have more than one catch and 17 mix a narrow guard with a broad handler, and a single well-behaved catch used to speak for the swallow beside it. Neither check reads hasTryCatch any more. A try/finally leaves it true with no catch clause at all, which is what put runs-replication.status at the top of the first rendered fix list; the question is now catches.length. A parse guard is recognised when it covers less than half the body, which keeps otel.v1.logs reported, where the catch covers 15 of 18 statements and merely contains a request.json. The narrow limb of the proposed rule is left out. A one-statement try around an awaited service call is as narrow as one around a parse. Taking it clears eleven more routes and reading all eleven says six are real, including a silent run cancellation and two credential paths that report a database failure to the browser as a 400 with the internal message in it. FIX FIRST now reads account.tokens, api.v1.authorization-code and api.v1.token, all three genuine. Global score 83. --- .../src/checks/errorClassification.ts | 70 ++++++------ .../src/checks/requestContext.ts | 11 +- .../observability-map/test/checks.test.ts | 101 +++++++++++++++++- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 7320e60d092..9b6e856d1ed 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -1,4 +1,4 @@ -import type { CheckResult, EntryPoint } from "../types.js"; +import type { CatchEvidence, CheckResult, EntryPoint } from "../types.js"; import { isTrivial } from "../triviality.js"; const ID = "error-classification"; @@ -25,19 +25,33 @@ export const BUILDERS = new Set([ ]); /** - * A parse: `await request.json()`, `JSON.parse(raw)`. The dot matters, it keeps Remix's `json({})` - * response helper out. Matched against `calleeTexts`, which carries the whole callee path. + * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and + * covers less than half the entry point's statements. Both halves matter. `guardsParse` alone lets + * `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and merely happens to contain a + * `request.json()`, and that is a real swallow. The coverage test is relative to the body rather + * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement + * one alike. */ -const PARSE_CALL = /(^|\.)JSON\.parse$|\.json$/; +export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; +} /** - * Whether every catch in the entry point guards a parse and nothing wider. Both checks read the - * field the same way: a guard around one parse is not the route taking charge of its failures, - * so `error-classification` does not call it a swallow and `request-context` does not ask it to - * name a tenant. + * Whether a clause has decided what the error means. Rethrowing is a decision, branching is a + * decision, and guarding a parse answers for the one thing the guard covers. + * + * `narrow` is deliberately not a fourth way to qualify, which is where this differs from the rule + * the scanner work proposed. A one-statement try around `await service.call(run)` is narrow and is + * still a swallow. Taking the narrow limb clears eleven more entry points, and reading all eleven + * says six are real: the silent cancel in `api.v2.runs.$runParam.cancel.ts`, the PAT revoke in + * `account.tokens/route.tsx` and the invite revoke, both of which report a database failure to the + * browser as a 400 with an internal message in it, a `.map` that drops a broken dashboard on the + * floor, and two more. The four it would rightly clear are all the same deliberate shape, best + * effort side work that logs and carries on, and all four are non-sensitive so they sort to the + * bottom of the fix list. See the task 5 report; switching is one limb in this function. */ -export function guardsOnlyAParse(ep: EntryPoint): boolean { - return ep.catchesNarrowly && ep.calleeTexts.some((t) => PARSE_CALL.test(t)); +export function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.rethrows || clause.branches || isParseGuard(clause, ep); } export function usesBuilder(ep: EntryPoint): boolean { @@ -55,20 +69,15 @@ export function usesBuilder(ep: EntryPoint): boolean { * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that * error never does. Crediting the wrapper would hide the one case in this family worth finding. * - * `catchRethrows` and `catchBranches` are OR-ed across every catch clause in the bodies, so both - * false means every catch in the entry point takes the same way out whatever was thrown. The - * asymmetry that buys: one good catch alongside one swallow reads as a pass. This check misses - * those rather than inventing them. + * Judged per catch clause, so an entry point is only as good as its worst one. That is the whole + * point of the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard + * with a broad handler, and under the old aggregate booleans a single well-behaved catch spoke for + * the swallow next to it. * - * `catchesNarrowly` excuses the guard that wraps one operation and answers for that operation: - * `try { body = await request.json() } catch { 400 }` neither branches nor rethrows and does not - * need to. On its own it excuses too much, because a one-statement try around an awaited service - * call is exactly as narrow as one around a parse: applied unencumbered it passes - * `try { await service.call(run) } catch { 500 }`, and it passes the design's own swallow fixture, - * `try { return await prisma.thing.findMany() } catch { return null }`. So the exemption also asks - * that the body parse something, which is the idiom the exemption was justified by. Over the real - * tree that combination clears the nine verbatim `request.json()` guards and holds back the four - * hand-read findings, see the task 5 report. + * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no + * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error + * propagates once the cleanup has run, and reading the old flag as a catch put + * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. */ export const errorClassification = { id: ID, @@ -76,19 +85,18 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const guardsAParse = guardsOnlyAParse(ep); - if (ep.hasTryCatch && !ep.catchRethrows && !ep.catchBranches && !guardsAParse) { + const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); + if (unaccounted.length > 0) { + const which = + ep.catches.length > 1 ? ` (${unaccounted.length} of ${ep.catches.length} catches)` : ""; return { id: ID, status: "fail", - detail: "catches its errors and takes one way out regardless of what was thrown", + detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (guardsAParse) { - return { id: ID, status: "pass", detail: "guards a parse, not the handler" }; - } - if (ep.catchRethrows || ep.catchBranches) { - return { id: ID, status: "pass", detail: "the catch distinguishes what it caught" }; + if (ep.catches.length > 0) { + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; } if (usesBuilder(ep)) { return { id: ID, status: "pass", detail: "classified by the builder" }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 116df0cc193..3ab19bc4695 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,6 +1,6 @@ import type { CheckResult, EntryPoint, LogCall } from "../types.js"; import { isTrivial } from "../triviality.js"; -import { guardsOnlyAParse } from "./errorClassification.js"; +import { isParseGuard } from "./errorClassification.js"; const ID = "request-context"; @@ -32,7 +32,7 @@ function failurePathLogs(ep: EntryPoint): LogCall[] { * thing the check exists to find and meant deleting a log line took a route out of the report. * Every non-trivial entry point is now judged: * - * - no catch at all, or nothing but a parse guard: pass. The route's own work still throws past it + * - no catch at all, or nothing but parse guards: pass. The route's own work still throws past it * to the central handler, which is the intended path in this codebase, and the tenant that * handler does not name is a platform-level gap reported once rather than against each route. A * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure @@ -51,9 +51,10 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - // A route whose only catch guards a parse still throws its own work past it to the central - // handler. Same reading error-classification gives the field. - if (!ep.hasTryCatch || guardsOnlyAParse(ep)) { + // `catches`, not `hasTryCatch`: a try/finally catches nothing, so its errors reach the central + // handler like any other. A route whose every clause guards a parse is in the same position, + // its own work still throws past those guards. Same reading error-classification gives them. + if (ep.catches.every((c) => isParseGuard(c, ep))) { return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; } const logs = failurePathLogs(ep); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index b04f7daa183..26dc4bc3f2c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -173,6 +173,79 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is + // `catches` that answers "does this route catch anything". Nothing is swallowed: the error + // propagates once the connection is closed. + it("passes a try/finally that catches nothing", () => { + const r = run( + "error-classification", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + const other = await redis.exists("other-key"); + return json({ exists, other }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse + // guard is a guard, the handler catch rethrows, so both are accounted for. + it("passes a parse guard sitting beside a handler catch that rethrows", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + const count = await prisma.thing.count(); + return json({ thing, audit, count }); + } catch (error) { + logger.error("create failed", { error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A parse guard that has grown to cover the handler is not a guard any more. `otel.v1.logs.ts` + // catches 15 of its 18 statements around a `request.json()` and answers 500 for all of them. + it("fails a parse guard that covers most of the body", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not flag a route whose try/catch is in the React component", () => { const r = run( @@ -481,10 +554,10 @@ describe("request-context", () => { expect(r.status).toBe("pass"); }); - // Known false positive, named in the task 5 report. The only catch here guards `new URL()`, a - // constructor, which never reaches `calleeTexts`, so the parse cannot be seen and the route is - // judged as though it kept its failures. Fixing it needs the scanner, not the check. - it("fails a route whose only catch guards a constructor parse", () => { + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the + // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard + // is legible now and the route is no longer judged as though it kept its failures. + it("passes a route whose only catch guards a constructor parse", () => { const r = run( "request-context", "_app.@.orgs.$organizationSlug.$.tsx", @@ -499,7 +572,25 @@ describe("request-context", () => { return typedjson({ origin, things: await prisma.thing.findMany() }); }` ); - expect(r.status).toBe("fail"); + expect(r.status).toBe("pass"); + }); + + it("passes a try/finally that catches nothing", () => { + const r = run( + "request-context", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + return json({ exists }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("pass"); }); it("still judges a route with a handler-wide catch beside a narrow guard", () => { From 1dbb49a7a5c5f88f1c540e9bfc6682263428f969 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 15:45:38 +0100 Subject: [PATCH 017/117] feat(observability-map): cli, single entry inspection and integration smoke Adds pnpm run map (repo root and package script), single-entry inspection mode, and index.ts exports. The routes directory now resolves against the repo root found by walking up to pnpm-workspace.yaml, not process.cwd(), so the CLI works from both the repo root and the package directory. Single-entry mode notes when an entry has no applicable scored checks rather than printing a bare 100/100. Gitignores the generated observability-map.json artifact. --- .gitignore | 3 + .../observability-map/package.json | 6 +- .../observability-map/src/cli.ts | 67 +++++++++++++++++++ .../observability-map/src/index.ts | 6 ++ .../test/integration.test.ts | 47 +++++++++++++ package.json | 1 + pnpm-lock.yaml | 3 + 7 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 internal-packages/observability-map/src/cli.ts create mode 100644 internal-packages/observability-map/src/index.ts create mode 100644 internal-packages/observability-map/test/integration.test.ts diff --git a/.gitignore b/.gitignore index 7d9dc169042..f540927e32b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ ailogger-output.log # local planning/design docs, not committed **/docs/superpowers/ + +# observability-map CLI output artifact, not committed +observability-map.json diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 3d6f695f67b..03321ad7d1f 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -9,13 +9,15 @@ }, "devDependencies": { "@types/node": "^24.13.3", - "rimraf": "6.0.1" + "rimraf": "6.0.1", + "tsx": "^4.19.2" }, "scripts": { "clean": "rimraf dist", "typecheck": "tsc --noEmit", "build": "pnpm run clean && tsc -p tsconfig.build.json", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "map": "tsx src/cli.ts" } } diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts new file mode 100644 index 00000000000..360f39a84f6 --- /dev/null +++ b/internal-packages/observability-map/src/cli.ts @@ -0,0 +1,67 @@ +import { existsSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { scanDirectory } from "./scan.js"; +import { buildReport, scoreEntry } from "./score.js"; +import { renderTerminal } from "./report/terminal.js"; +import { renderJson } from "./report/json.js"; + +const DEFAULT_ROUTES = "apps/webapp/app/routes"; + +/** + * Walks up from this file looking for `pnpm-workspace.yaml`, so the routes directory resolves + * correctly whether `map` is run from the repo root or from the package directory (where + * `pnpm --filter` puts you). Resolving `DEFAULT_ROUTES` against `process.cwd()` instead would only + * work from the repo root. + */ +function findRepoRoot(startDir: string): string { + let dir = startDir; + for (let i = 0; i < 10; i++) { + if (existsSync(resolve(dir, "pnpm-workspace.yaml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); +} + +export function main(argv: string[]): number { + const args = argv.slice(2); + const asJson = args.includes("--json"); + const noWrite = args.includes("--no-write"); + const target = args.find((a) => !a.startsWith("--")); + + const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); + const routesDir = resolve(repoRoot, DEFAULT_ROUTES); + const { entryPoints, parseFailures } = scanDirectory(routesDir); + + if (target) { + const match = entryPoints.find( + (e) => e.fileName === target || e.fileName.startsWith(target.replace(/^\//, "")) + ); + if (!match) { + process.stderr.write(`no entry point matching "${target}"\n`); + return 1; + } + const scored = scoreEntry(match); + const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; + process.stdout.write( + `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` + ); + for (const c of scored.checks) { + const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; + process.stdout.write(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + } + return 0; + } + + const report = buildReport(entryPoints, parseFailures); + process.stdout.write(asJson ? renderJson(report) : renderTerminal(report)); + process.stdout.write("\n"); + if (!noWrite) { + writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); + } + return 0; +} + +process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/index.ts b/internal-packages/observability-map/src/index.ts new file mode 100644 index 00000000000..1a8e9904931 --- /dev/null +++ b/internal-packages/observability-map/src/index.ts @@ -0,0 +1,6 @@ +export { scanDirectory, scanFile } from "./scan.js"; +export { buildReport, scoreEntry } from "./score.js"; +export { renderTerminal } from "./report/terminal.js"; +export { renderJson } from "./report/json.js"; +export type { MapReport, ScoredEntry } from "./score.js"; +export type { EntryPoint, CheckResult, CheckStatus } from "./types.js"; diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts new file mode 100644 index 00000000000..f628c5a1dfd --- /dev/null +++ b/internal-packages/observability-map/test/integration.test.ts @@ -0,0 +1,47 @@ +import { existsSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { scanDirectory } from "../src/scan.js"; +import { buildReport } from "../src/score.js"; + +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); + +/** + * Counts route module candidates the same way `scanDirectory` walks the tree, without scanning + * their contents: a flat `.ts`/`.tsx` file, or one `route.ts`/`route.tsx` per directory. This is a + * structural upper bound, not a golden number - not every candidate exports a loader or action, so + * `entryPoints.length` must stay below it, and the route set is free to grow or shrink over time + * without breaking this test. + */ +function countCandidates(dir: string): number { + let count = 0; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (child.isFile() && (child.name === "route.ts" || child.name === "route.tsx")) count++; + } + continue; + } + if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; + } + return count; +} + +describe("scanning the real webapp routes", () => { + it("parses every route file without crashing", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + // Invariants, not exact numbers: the route set changes constantly. + expect(entryPoints.length).toBeGreaterThan(200); + expect(entryPoints.length).toBeLessThan(countCandidates(ROUTES)); + expect(parseFailures).toEqual([]); + }); + + it("produces a report with a score in range", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const report = buildReport(entryPoints, parseFailures); + expect(report.global).toBeGreaterThanOrEqual(0); + expect(report.global).toBeLessThanOrEqual(100); + expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); + }); +}); diff --git a/package.json b/package.json index 2708b06be73..9fad94dd9d4 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "clean": "turbo run clean", "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", "typecheck": "turbo run typecheck", + "map": "pnpm --filter @internal/observability-map run map", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "test:e2e:dev": "turbo run test:e2e:dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2b0c4a17c4..d5fdb570112 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1151,6 +1151,9 @@ importers: rimraf: specifier: 6.0.1 version: 6.0.1 + tsx: + specifier: ^4.19.2 + version: 4.22.4 internal-packages/otlp-importer: dependencies: From ca8b9160e6b9f1caaee5e0b77b6accbdfee95115 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:25:16 +0100 Subject: [PATCH 018/117] fix(observability-map): stop guardsParse and branches firing on lookalikes Two catch-evidence fields matched shapes that resemble the thing they detect, which excused catches the checks exist to find. - guardsParse took any new X(), so new BranchesPresenter() or new Set() excused a catch over ordinary work. It now needs a parsing constructor, URL, URLSearchParams or RegExp, chosen from what the route tree actually constructs inside try blocks. 60 of 242 clauses change, 141 true to 81 - branches took an instanceof anywhere in the clause, including the error instanceof Error ? error.message : String(error) idiom, which words a message rather than picking a path. It now needs an if, a switch, or a conditional that is the whole return or throw. 29 of 242 clauses change, 134 true to 105. All 33 bare instanceof uses in the tree are the formatting idiom Clauses with no evidence at all go from 9 to 37. error-classification will need recalibrating: on this evidence it reports 64 routes rather than 28, and nothing it reported before stops being reported. --- .../observability-map/src/scan.ts | 46 ++++-- .../observability-map/src/types.ts | 14 +- .../observability-map/test/scan.test.ts | 149 +++++++++++++++++- 3 files changed, 194 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 0028d55bab8..6301f8ed6f8 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -121,14 +121,27 @@ const NARROW_TRY_STATEMENTS = 2; const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; /** - * Whether the guarded region parses or constructs something. A constructor counts: `new URL(x)` is - * the commonest parse guard in the tree, and constructors are absent from `calleeTexts`. + * Constructors that parse untrusted input and throw when it is malformed. Deliberately short: any + * constructor at all would mean `new BranchesPresenter()` or `new Set(...)` excuses a catch that + * guards ordinary work, which was true of 77 try blocks in the route tree. + */ +const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); + +/** + * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here + * because constructors are absent from `calleeTexts`. */ function guardsParse(tryBlock: ts.Block): boolean { let found = false; const visit = (node: ts.Node) => { if (found) return; - if (ts.isNewExpression(node)) found = true; + if ( + ts.isNewExpression(node) && + ts.isIdentifier(node.expression) && + PARSE_CONSTRUCTORS.has(node.expression.text) + ) { + found = true; + } if (ts.isCallExpression(node)) { const text = calleeText(node.expression) ?? calleeName(node.expression); if (text !== null && PARSE_CALLEE.test(text)) found = true; @@ -139,6 +152,26 @@ function guardsParse(tryBlock: ts.Block): boolean { return found; } +function containsInstanceOf(node: ts.Node): boolean { + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword) { + return true; + } + return ts.forEachChild(node, containsInstanceOf) === true; +} + +/** + * Whether a conditional expression tests the error to pick what the clause does, rather than to + * word what it says. It counts only when the whole `return`/`throw` is the conditional, so + * `return e instanceof Response ? e : json({}, { status: 500 })` counts and + * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. + * The second is message formatting: every error leaves by the same path. + */ +function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { + if (!containsInstanceOf(node.condition)) return false; + const parent = node.parent; + return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -147,12 +180,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc const visit = (node: ts.Node) => { if (ts.isThrowStatement(node)) rethrows = true; if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; - if ( - ts.isBinaryExpression(node) && - node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword - ) { - branches = true; - } + if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; ts.forEachChild(node, visit); }; visit(clause.block); diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 2e2401c87a4..4a9eec56b69 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -16,12 +16,18 @@ export type CatchEvidence = { narrow: boolean; /** The clause contains a `throw`. */ rethrows: boolean; - /** The clause branches on the error: an `if`, a `switch`, or an `instanceof`. */ + /** + * The clause picks what to do from what it caught: an `if`, a `switch`, or a conditional that is + * the whole `return`/`throw`. An `instanceof` used only to word a message, + * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count: every error + * still leaves by the same path. + */ branches: boolean; /** - * The guarded region parses or constructs something: `JSON.parse`, `request.json()`, a zod - * `parse`/`safeParse`, or any `new X(...)`. Constructors are counted here because `new URL(x)` is - * the commonest parse guard in the tree and constructors never appear in `calleeTexts`. + * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, + * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here + * because constructors never appear in `calleeTexts`; other constructors do not count, or every + * `new SomePresenter()` in a try would excuse its catch. */ guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index e4e2388fd35..6625adf9343 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -628,7 +628,7 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catchRethrows).toBe(false); }); - it("sets catchBranches for a bare instanceof with no `if`", () => { + it("sets catchBranches for an instanceof conditional that is the whole returned expression", () => { const ep = scanFile( "branch-instanceof.ts", ` @@ -1092,7 +1092,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catchesNarrowly).toBe(false); }); - it("sees a constructor as a guarded parse", () => { + it("sees a URL constructor as a guarded parse", () => { const ep = scanFile( "_app.@.orgs.$organizationSlug.$.tsx", ` @@ -1246,3 +1246,148 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); }); }); + +describe("scanFile: guardsParse is limited to parsing constructors", () => { + it("does not treat a presenter construction as a parse guard", () => { + const ep = scanFile( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.branches/route.tsx", + ` + export async function loader({ request, params }) { + try { + const presenter = new BranchesPresenter(); + const result = await presenter.call({ userId: 1, projectSlug: params.projectParam }); + return typedjson(result); + } catch (error) { + logger.error("Error loading preview branches page", { error }); + throw new Response(undefined, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("does not treat a collection construction as a parse guard", () => { + const ep = scanFile( + "account.tokens/route.tsx", + ` + export async function action({ request }) { + try { + const roles = await loadRoles(request); + const names = new Set(roles.map((r) => r.name)); + return json({ names: [...names] }); + } catch (error) { + return json({ error: "failed" }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("treats RegExp and URLSearchParams construction as a parse guard", () => { + const regexp = scanFile( + "regexp.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + new RegExp(pattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(regexp!.catches[0]!.guardsParse).toBe(true); + + const search = scanFile( + "search-params.ts", + ` + export async function loader({ request }) { + try { + return json(Object.fromEntries(new URLSearchParams(request.url))); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(search!.catches[0]!.guardsParse).toBe(true); + }); + + it("still sees a parse call in a try that also constructs something ordinary", () => { + const ep = scanFile( + "parse-and-construct.ts", + ` + export async function action({ request }) { + try { + const body = JSON.parse(await request.text()); + const service = new PromptService(); + return json(await service.create(body)); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: branches ignores the error-stringifying ternary", () => { + it("does not count an instanceof nested inside a returned call argument", () => { + const ep = scanFile( + "admin.api.v1.runs-replication.start.ts", + ` + export async function action({ request }) { + try { + return json(await start(request)); + } catch (error) { + return json({ error: error instanceof Error ? error.message : error }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.branches).toBe(false); + expect(ep!.catchBranches).toBe(false); + }); + + it("does not count an instanceof used to build a logged message", () => { + const ep = scanFile( + "admin.api.v1.feature-flags.ts", + ` + export async function action({ request }) { + try { + return json(await setFlag(request)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.error("flag update failed", { message }); + return json({ error: message }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still counts an instanceof in an `if`", () => { + const ep = scanFile( + "branch-if-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) return e; + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From e4eaa7caf511e47aeef11882e5600ab2fbd06ffd Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:29:13 +0100 Subject: [PATCH 019/117] chore(observability-map): declare vitest and scope the build The build config had no include and no rootDir, so tsc inferred the package root because vitest.config.ts happened to be inside the compilation. That is the only reason dist/src/index.js landed where the package's main points, and excluding the config would have silently moved the entry point. Scope the build to src and pin rootDir so the layout is intentional. vitest was resolving from the root workspace by hoisting despite being the test runner and supplying the global types. Declare it at the version the other internal packages use. --- internal-packages/observability-map/package.json | 3 ++- .../observability-map/tsconfig.build.json | 10 ++++++++-- pnpm-lock.yaml | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 03321ad7d1f..776ec458963 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -10,7 +10,8 @@ "devDependencies": { "@types/node": "^24.13.3", "rimraf": "6.0.1", - "tsx": "^4.19.2" + "tsx": "^4.19.2", + "vitest": "4.1.7" }, "scripts": { "clean": "rimraf dist", diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json index 56ade258e18..254fcfbca49 100644 --- a/internal-packages/observability-map/tsconfig.build.json +++ b/internal-packages/observability-map/tsconfig.build.json @@ -1,5 +1,11 @@ { "extends": "./tsconfig.json", - "compilerOptions": { "noEmit": false, "outDir": "dist", "declaration": true }, - "exclude": ["node_modules", "dist", "test"] + "include": ["src/**/*.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d5fdb570112..64f6fe98b18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1154,6 +1154,9 @@ importers: tsx: specifier: ^4.19.2 version: 4.22.4 + vitest: + specifier: 4.1.7 + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) internal-packages/otlp-importer: dependencies: From 8232d110b932a135a055daa90f438a2f1ab4180f Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 20:36:41 +0100 Subject: [PATCH 020/117] fix(observability-map): stop paying routes for having no error handling Emptying every catch clause in the tree scored it 100. Both scored checks passed on the single fact that a route has no catch: error-classification credited it as propagating to the global handler, request-context treated it as having handed its failures over. So the gradient rewarded deleting error handling, and 222 of 412 entries scored 100 on that one shared fact. error-classification now reports not-applicable for a route with no catch, since there is no classification decision to judge, and no longer credits a builder wrapper for error handling the route does not do. request-context fails it instead: the global handler carries requestId, path, host and method and no tenant, so such a route genuinely cannot name whose request broke. Excusing it would reinstate the perverse incentive. Score falls from 76 to 22, which is the honest reading. Deleting all error handling now takes it to 7 rather than 100. The two checks decorrelate: kappa on error-classification against request-context moves from +0.231 to -0.032, and the other two pairs stay near zero. --- .../src/checks/errorClassification.ts | 40 ++++++----- .../src/checks/requestContext.ts | 66 ++++++++----------- .../observability-map/test/checks.test.ts | 36 +++++----- .../observability-map/test/report.test.ts | 22 +++++-- .../observability-map/test/score.test.ts | 47 +++++++++---- 5 files changed, 118 insertions(+), 93 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 9b6e856d1ed..f4a8ce5e8b2 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -4,9 +4,10 @@ import { isTrivial } from "../triviality.js"; const ID = "error-classification"; /** - * The route builders that own the failure path: they authenticate, they catch, they pass a thrown - * `Response` through untouched and report anything else through `logBoundaryError` before - * answering 500. A route wrapped in one of these has its errors classified for it. + * The route builders that authenticate the request, which is what `auth-boundary` reads them for. + * They also catch and classify, passing a thrown `Response` through untouched and reporting + * anything else through `logBoundaryError`, but `error-classification` no longer credits that: a + * route with no catch of its own is judged on nothing, wrapper or not. * * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. @@ -64,15 +65,17 @@ export function usesBuilder(ep: EntryPoint): boolean { /** * Who decides what a failure means, and on what evidence. * - * The swallow is read before the builder is credited, which looks like the wrong order until you - * read `api.v2.runs.$runParam.cancel.ts`: a `createActionApiRoute` handler wrapping its service - * call in `try { ... } catch { return 500 }`. The builder classifies what reaches it, and that - * error never does. Crediting the wrapper would hide the one case in this family worth finding. + * Judged per catch clause, so an entry point is only as good as its worst one. That is the point of + * the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard with a + * broad handler, and under the old aggregate booleans a single well-behaved catch spoke for the + * swallow next to it. * - * Judged per catch clause, so an entry point is only as good as its worst one. That is the whole - * point of the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard - * with a broad handler, and under the old aggregate booleans a single well-behaved catch spoke for - * the swallow next to it. + * A route with no catch is not-applicable, not a pass. It makes no classification decision, so + * there is nothing here to judge and nothing to credit. Crediting it was worse than merely + * generous: with `request-context` also passing the same routes, emptying every catch clause in the + * tree scored it 100, so the metric paid you for deleting error handling. Out of the denominator + * is the honest place for it, and it takes the builder credit with it: a builder-wrapped route with + * no catch of its own now sits out too, rather than collecting a point for the wrapper. * * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error @@ -85,6 +88,13 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } + if (ep.catches.length === 0) { + return { + id: ID, + status: "not-applicable", + detail: "catches nothing, so it classifies nothing", + }; + } const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); if (unaccounted.length > 0) { const which = @@ -95,12 +105,6 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (ep.catches.length > 0) { - return { id: ID, status: "pass", detail: "every catch decides what it caught" }; - } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "classified by the builder" }; - } - return { id: ID, status: "pass", detail: "errors propagate to the global handler" }; + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; }, }; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 3ab19bc4695..041a9515257 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -1,6 +1,5 @@ -import type { CheckResult, EntryPoint, LogCall } from "../types.js"; +import type { CheckResult, EntryPoint } from "../types.js"; import { isTrivial } from "../triviality.js"; -import { isParseGuard } from "./errorClassification.js"; const ID = "request-context"; @@ -12,38 +11,27 @@ const ID = "request-context"; */ const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; -function failurePathLogs(ep: EntryPoint): LogCall[] { - return ep.logCalls.filter((l) => l.inCatch); -} - /** - * Whether a failure this route reports itself can be traced to whoever it happened to. + * Whether a failure here can be traced to whoever it happened to. * - * Everything the platform attaches centrally is already accounted for, which is what makes this - * worth asking. `logger` pushes the http context (requestId, path, host, method) onto every line + * Everything the platform attaches centrally is accounted for, which is what makes this worth + * asking. `logger` pushes the http context, `{ requestId, path, host, method }`, onto every line * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is - * `logBoundaryError(message, error, url)`, which is a url and an error. So a builder-wrapped route - * is not attributed either and gets no free pass here. An incident tells you which route and which - * request; whose environment it was is only ever in the fields the route passes itself. - * - * Applicability turns on whether the route keeps its own failures, never on whether it logs. The - * first version made a route not-applicable when it had no failure-path log, which excused the very - * thing the check exists to find and meant deleting a log line took a route out of the report. - * Every non-trivial entry point is now judged: + * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route + * and which request failed, and never whose environment it was, unless the route passed the field + * itself. 21 of 427 entry points do. * - * - no catch at all, or nothing but parse guards: pass. The route's own work still throws past it - * to the central handler, which is the intended path in this codebase, and the tenant that - * handler does not name is a platform-level gap reported once rather than against each route. A - * `try { body = await request.json() } catch { 400 }` is not a route taking over its failure - * path, and reading it as one put false positives at the top of the first rendered report. - * - a catch that covers the route's work: it decided the outcome itself, so it has to say whose - * failure it was. + * Every non-trivial entry point is judged, and a route that never catches fails like any other. + * That is the whole point rather than an oversight: its failures go to the global handler, which + * names no tenant, so it genuinely cannot say whose request broke. Passing those routes, as this + * check used to, meant deleting every catch clause in the tree scored it 100. Excusing them as + * not-applicable would be the same mistake in quieter clothes, since it would once again reward + * having no failure handling to inspect. * - * Deleting a log call can then only make a verdict worse or leave it alone, and adding a catch - * without a report is a regression the check reports, which is the direction the incentive should - * run in. The one move that still improves a verdict is deleting the try/catch outright, and that - * hands the error back to the central handler, which `error-classification` also treats as correct. + * The consequence is a check that fails 90% of what it looks at, which is an honest reading of a + * codebase where the fix is one platform change, tenant fields through `trace(...)` in the auth + * path, rather than 300 route edits. Weight it accordingly, but do not read the count as noise. */ export const requestContext = { id: ID, @@ -51,25 +39,27 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - // `catches`, not `hasTryCatch`: a try/finally catches nothing, so its errors reach the central - // handler like any other. A route whose every clause guards a parse is in the same position, - // its own work still throws past those guards. Same reading error-classification gives them. - if (ep.catches.every((c) => isParseGuard(c, ep))) { - return { id: ID, status: "pass", detail: "hands its failures to the central handler" }; - } - const logs = failurePathLogs(ep); - const named = logs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + const failurePathLogs = ep.logCalls.filter((l) => l.inCatch); + const named = failurePathLogs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); if (named) { const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } + if (failurePathLogs.length > 0) { + return { + id: ID, + status: "fail", + detail: + "logs its failure without naming an environment, project, organization, run or user", + }; + } return { id: ID, status: "fail", detail: - logs.length === 0 + ep.catches.length > 0 ? "keeps its failures and records nothing about whose they were" - : "logs its failure without naming an environment, project, organization, run or user", + : "leaves its failures to the central handler, which names no tenant", }; }, }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 26dc4bc3f2c..856a221c6b6 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -40,14 +40,16 @@ describe("registry", () => { }); describe("error-classification", () => { - it("passes a builder-wrapped route with no local try/catch", () => { + // C1. A route with no catch makes no classification decision, so there is nothing here to judge + // and nothing to credit. Crediting it made deleting error handling raise the score. + it("is not applicable to a builder-wrapped route with no local try/catch", () => { const r = run( "error-classification", "api.v1.x.ts", `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); it("fails a raw route whose catch swallows every error identically", () => { @@ -109,7 +111,7 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - it("passes a raw route that lets its errors propagate", () => { + it("is not applicable to a raw route that lets its errors propagate", () => { const r = run( "error-classification", "api.v1.w.ts", @@ -119,7 +121,7 @@ describe("error-classification", () => { return json({ rows }); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); it("is not applicable to a trivial redirect", () => { @@ -176,7 +178,7 @@ describe("error-classification", () => { // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is // `catches` that answers "does this route catch anything". Nothing is swallowed: the error // propagates once the connection is closed. - it("passes a try/finally that catches nothing", () => { + it("is not applicable to a try/finally that catches nothing", () => { const r = run( "error-classification", "admin.api.v1.runs-replication.status.ts", @@ -192,7 +194,7 @@ describe("error-classification", () => { } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse @@ -247,7 +249,7 @@ describe("error-classification", () => { }); // False positive fixture: the only try/catch in the file belongs to the component. - it("does not flag a route whose try/catch is in the React component", () => { + it("does not judge a route whose try/catch is in the React component", () => { const r = run( "error-classification", "_app.orgs.$organizationSlug.things/route.tsx", @@ -258,7 +260,7 @@ describe("error-classification", () => { } ${COMPONENT}` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); }); @@ -494,7 +496,9 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - it("passes a route that hands its failures to the central handler", () => { + // C1. The global handler carries requestId, path, host and method, and no tenant. A route that + // never catches cannot name one, so it fails rather than being credited or excused. + it("fails a route that leaves everything to the central handler", () => { const r = run( "request-context", "api.v1.q.ts", @@ -505,7 +509,7 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); it("fails a route that catches but only names an identifier outside the catch", () => { @@ -539,7 +543,7 @@ describe("request-context", () => { // A guard around a parse is not the route taking over its failure path: whatever its real work // throws still reaches the central handler. Same reading error-classification gives the field. - it("passes a route whose only catch guards a parse", () => { + it("fails a route whose only catch guards a parse", () => { const r = run( "request-context", "api.v1.q.ts", @@ -551,13 +555,13 @@ describe("request-context", () => { return json(await prisma.thing.findMany({ where: body })); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard // is legible now and the route is no longer judged as though it kept its failures. - it("passes a route whose only catch guards a constructor parse", () => { + it("fails a route whose only catch guards a constructor parse", () => { const r = run( "request-context", "_app.@.orgs.$organizationSlug.$.tsx", @@ -572,10 +576,10 @@ describe("request-context", () => { return typedjson({ origin, things: await prisma.thing.findMany() }); }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); - it("passes a try/finally that catches nothing", () => { + it("fails a try/finally that catches nothing", () => { const r = run( "request-context", "admin.api.v1.runs-replication.status.ts", @@ -590,7 +594,7 @@ describe("request-context", () => { } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("fail"); }); it("still judges a route with a handler-wide catch beside a narrow guard", () => { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index be977d24d8f..c41dbf6136a 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -57,12 +57,18 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 67: only auth-boundary fails (no guard called), the other two pass because - // there is no try/catch to fumble. + // Sensitive, score 67: guarded and classifies what it catches, but its failure log names + // nobody, so only request-context fails. const sensitiveSixtySeven = scanFile( "api.v1.auth.tokens.ts", - `import { prisma } from "~/db.server"; - export async function action() { return prisma.token.create({ data: {} }); }` + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("failed", { error }); throw error; } + }` )!; // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both @@ -80,10 +86,12 @@ describe("renderTerminal", () => { const sensitiveAuditOnly = scanFile( "api.v1.billing.ts", `import { prisma } from "~/db.server"; + import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; - export async function action({ request }: { request: Request }) { - const userId = requireUserId(request); - return prisma.billing.update({ where: { userId }, data: {} }); + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.billing.update({ where: { userId }, data: {} }); } + catch (error) { logger.error("billing update failed", { userId, error }); throw error; } }` )!; diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index ef27f5473c4..01f90ff72d6 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -17,14 +17,39 @@ export async function loader() { try { return await prisma.thing.findMany(); } catch (e) { return null; } }`; +/** Guarded, classifies what it catches, and names the tenant on the failure path. */ +const CLEAN = `import { requireUserId } from "~/services/session.server"; +import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function action({ request, params }) { + const userId = await requireUserId(request); + try { + return await prisma.token.create({ data: { userId } }); + } catch (error) { + logger.error("token create failed", { userId, environmentId: params.envId, error }); + throw error; + } +}`; + describe("scoreEntry", () => { - it("scores a builder route 100", () => { - expect(scoreEntry(scanFile("api.v1.a.ts", BUILDER)!).score).toBe(100); + it("scores an entry that passes every applicable check 100", () => { + expect(scoreEntry(scanFile("api.v1.auth.tokens.ts", CLEAN)!).score).toBe(100); + }); + + // A builder wrapper classifies errors for the route, but the route itself catches nothing and + // names nobody on its failure path, so there is one applicable check and it fails. + it("does not credit a builder route for the error handling it does not do", () => { + const scored = scoreEntry(scanFile("api.v1.a.ts", BUILDER)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.score).toBe(0); }); it("excludes audit-trail from the per-entry score", () => { - const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", BUILDER)!); - expect(scored.checks.some((c) => c.id === "audit-trail")).toBe(true); + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", CLEAN)!); + expect(scored.checks.find((c) => c.id === "audit-trail")!.status).toBe("fail"); expect(scored.score).toBe(100); }); @@ -63,18 +88,12 @@ ${BUSY_AND_FAILING}`; describe("buildReport", () => { it("reports the audit gap separately from the score", () => { - const eps = [ - scanFile("api.v1.a.ts", BUILDER)!, - scanFile( - "api.v1.auth.tokens.ts", - `import { prisma } from "~/db.server"; - export async function action() { return prisma.token.create({ data: {} }); }` - )!, - ]; - const report = buildReport(eps, []); + // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported + // as its own figure and must not pull the score down with it. + const report = buildReport([scanFile("api.v1.auth.tokens.ts", CLEAN)!], []); expect(report.auditGap.sensitiveMutations).toBe(1); expect(report.auditGap.withAudit).toBe(0); - expect(report.global).toBeGreaterThan(0); + expect(report.global).toBe(100); }); it("records parse failures", () => { From bacda64b3b8fbf54f548ef14073402267d7dfe8a Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:09:31 +0100 Subject: [PATCH 021/117] fix(observability-map): stop a suppression buying a better score score was passed over applicable, and a suppressed check left the denominator, so removing a failing check raised the entry: 33 became 50 became 100 as checks were suppressed. The comment above the code claimed the opposite of what it did. A suppression now takes its check out of the numerator and the denominator, and the result is capped by what the entry would score unsuppressed, so it can only ever hold the number still or lower it. What a suppression buys is removal from the worklist, with a reason on the record. An entry whose every scored check is suppressed no longer reads as measured, and the report prints a suppression count so laundering is visible rather than silent. Tests assert the score effect, not just the check status, which is why the original slipped through. --- .../observability-map/src/report/terminal.ts | 10 +++ .../observability-map/src/score.ts | 42 +++++++++--- .../observability-map/test/report.test.ts | 18 +++++ .../observability-map/test/score.test.ts | 66 +++++++++++++++++++ 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index c5d19e491f9..312b7b8a16c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -42,6 +42,16 @@ export function renderTerminal(report: MapReport): string { `No audit helper exists in the webapp.` ); + if (report.suppressions.checks > 0) { + const { entries, checks } = report.suppressions; + lines.push( + `SUPPRESSED ${checks} check${checks === 1 ? "" : "s"} across ${entries} entry point${ + entries === 1 ? "" : "s" + }, each with a reason on the record. A suppression removes a finding from this list, ` + + `it does not raise a score.` + ); + } + const worst = report.entries .filter((e) => scoredFailures(e).length > 0) .sort( diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 542b968177c..65782c79d05 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -17,6 +17,8 @@ export type ScoredEntry = { * default cannot inflate a figure nobody checked. */ measured: boolean; + /** Scored checks a comment in the source suppressed, in `SCORED_CHECK_IDS` order. */ + suppressed: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; }; @@ -27,6 +29,8 @@ export type MapReport = { measured: number; /** Entry points every scored check reported not-applicable for; excluded from `global`. */ unmeasured: number; + /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ + suppressions: { entries: number; checks: number }; byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number }; auditGap: { sensitiveMutations: number; withAudit: number }; @@ -36,19 +40,25 @@ export type MapReport = { export function scoreEntry(ep: EntryPoint): ScoredEntry { const suppressed = suppressedChecks(ep.source); - const checks = CHECKS.map((c) => { - const result = c.run(ep); - const reason = suppressed.get(c.id); - // A suppression always lands on not-applicable, never on pass: suppressing a check must remove - // it from the score, not launder it into a point in the entry's favor. + const raw = CHECKS.map((c) => c.run(ep)); + const checks = raw.map((result) => { + const reason = suppressed.get(result.id); return reason - ? { id: c.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + ? { id: result.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } : result; }); - const scored = checks.filter((c) => SCORED_CHECK_IDS.includes(c.id)); - const applicable = scored.filter((c) => c.status !== "not-applicable"); - const passed = applicable.filter((c) => c.status === "pass").length; + const scored = raw.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const ratio = (of: CheckResult[]) => { + const applicable = of.filter((c) => c.status !== "not-applicable"); + if (applicable.length === 0) return 100; + return Math.round( + (applicable.filter((c) => c.status === "pass").length / applicable.length) * 100 + ); + }; + + const visible = scored.filter((c) => !suppressed.has(c.id)); + const applicable = visible.filter((c) => c.status !== "not-applicable"); return { fileName: ep.fileName, @@ -56,8 +66,14 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { family: familyOf(ep.fileName), sensitive: classifySensitivity(ep).sensitive, checks, + suppressed: scored.filter((c) => suppressed.has(c.id)).map((c) => c.id), + // Suppressing a check takes it out of the numerator and the denominator, and the result is + // capped by what the entry would have scored unsuppressed. Otherwise removing a failing check + // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the + // suppression comment laundered the finding into a point. What a suppression buys is removal + // from the worklist, with a reason on the record. It cannot buy a better number. measured: applicable.length > 0, - score: applicable.length === 0 ? 100 : Math.round((passed / applicable.length) * 100), + score: Math.min(ratio(visible), ratio(scored)), }; } @@ -98,10 +114,16 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); + const suppressing = entries.filter((e) => e.suppressed.length > 0); + return { global: mean(measuredEntries.map((e) => e.score)), measured: measuredEntries.length, unmeasured: entries.length - measuredEntries.length, + suppressions: { + entries: suppressing.length, + checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), + }, byFamily, sensitiveCohort: groupStats(sensitive), auditGap: { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index c41dbf6136a..39fef9730e4 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -29,6 +29,24 @@ describe("renderTerminal", () => { expect(out).toContain("/api/v1/auth/tokens"); }); + it("surfaces suppressions so laundering is visible rather than silent", () => { + const suppressed = scanFile( + "api.v1.d.ts", + `// obs-map-disable-next-line error-classification -- deliberate, see ticket + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + const out = renderTerminal(buildReport([suppressed], [])); + expect(out).toMatch(/suppress/i); + expect(out).toContain("1"); + }); + + it("does not mention suppressions when there are none", () => { + expect(renderTerminal(report())).not.toMatch(/suppress/i); + }); + it("surfaces parse failures so the denominator is not silently wrong", () => { expect(renderTerminal(report())).toContain("broken.ts"); }); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 01f90ff72d6..f962aa2b8c8 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -73,6 +73,57 @@ ${BUSY_AND_FAILING}`; ); }); + // I1. `score = passed / applicable` meant removing a failing check from the denominator raised + // the entry's score, so suppression laundered findings into points. A suppression buys removal + // from the worklist, never a better number. + it("does not raise the score when a failing check is suppressed", () => { + const source = `import { requireUserId } from "~/services/session.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { return null; } +}`; + const plain = scoreEntry(scanFile("api.v1.auth.tokens.ts", source)!); + const suppressed = scoreEntry( + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable-next-line error-classification -- deliberate, see ticket +${source}` + )! + ); + + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(suppressed.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); + + it("records which scored checks were suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +// obs-map-disable-next-line request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); + }); + + it("does not let an entry whose every scored check is suppressed read as measured", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +// obs-map-disable-next-line request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.measured).toBe(false); + }); + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); @@ -87,6 +138,21 @@ ${BUSY_AND_FAILING}`; }); describe("buildReport", () => { + it("counts suppressions so laundering is visible in the report", () => { + const report = buildReport( + [ + scanFile( + "api.v1.b.ts", + `// obs-map-disable-next-line error-classification -- health probe +${BUSY_AND_FAILING}` + )!, + scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, + ], + [] + ); + expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + it("reports the audit gap separately from the score", () => { // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported // as its own figure and must not pull the score down with it. From 8631151b64d4d2dde9546f0b76c0b5dee0a20cba Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:12:11 +0100 Subject: [PATCH 022/117] fix(observability-map): break the circularity in sensitivity requireAdminApiRequest was on the sensitive-symbol list, so 34 of the 67 sensitive entry points were sensitive only because they call the admin guard, which auth-boundary then passed them for. A mitigation cannot be what makes a route risky, and this list is the fix list's primary sort key. The tokens path segment was matching waitpoint routes: seven of the eight matches were run coordination handles rather than credentials. A token segment directly under waitpoints no longer counts. The cohort falls from 67 to 26, and the two remaining token matches are the personal access token routes the segment was meant to find. --- .../observability-map/src/sensitivity.ts | 16 +++- .../observability-map/test/checks.test.ts | 18 +++-- .../test/sensitivity.test.ts | 73 +++++++++++++++++++ 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index 40f348afac7..b80beee65e5 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -1,10 +1,15 @@ import type { EntryPoint } from "./types.js"; import { routePathOf } from "./adapters/remix.js"; +/** + * Symbols whose presence says the route does something risky. Calling a guard is not one of them: + * `requireAdminApiRequest` was on this list and made 34 of the 67 sensitive entry points sensitive + * purely because they were guarded, which `auth-boundary` then passed them for. A mitigation + * cannot be the hazard, and this list feeds the fix list's primary sort key. + */ const SENSITIVE_SYMBOLS = [ "clearImpersonation", "setImpersonation", - "requireAdminApiRequest", "createPersonalAccessToken", "regenerateApiKey", "createJWT", @@ -47,8 +52,13 @@ export function classifySensitivity(ep: EntryPoint): Sensitivity { const segments = routePathOf(ep.fileName) .split("/") .filter((s) => s.length > 0); - for (const seg of segments) { - if (SENSITIVE_SEGMENTS.includes(seg)) reasons.push(`path segment "${seg}"`); + for (const [i, seg] of segments.entries()) { + if (!SENSITIVE_SEGMENTS.includes(seg)) continue; + // A waitpoint token is a handle for resuming a run, not a credential. Seven of the eight + // `tokens` matches in the tree were waitpoint routes, so the segment on its own was mostly + // finding the wrong thing. + if ((seg === "token" || seg === "tokens") && segments[i - 1] === "waitpoints") continue; + reasons.push(`path segment "${seg}"`); } return { sensitive: reasons.length > 0, reasons }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 856a221c6b6..081db1661ed 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -266,14 +266,16 @@ describe("error-classification", () => { describe("auth-boundary", () => { it("passes a sensitive route guarded by a require helper", () => { + // Sensitive on the impersonation call, not on the guard: calling a guard is not what makes a + // route sensitive, see sensitivity.test.ts. const r = run( "auth-boundary", - "admin.api.v1.gc.ts", + "admin.api.v1.impersonate.ts", `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; - import { prisma } from "~/db.server"; - export async function loader({ request }) { + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { await requireAdminApiRequest(request); - return prisma.thing.findMany(); + return setImpersonation(request, "user_1"); }` ); expect(r.status).toBe("pass"); @@ -370,12 +372,12 @@ describe("auth-boundary", () => { it("passes a sensitive callback guarded by a signature check", () => { const r = run( "auth-boundary", - "api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts", - `import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; + "webhooks.v1.billing.$hash.ts", + `import { verifyWebhookSignature } from "~/services/webhooks.server"; import { prisma } from "~/db.server"; export async function action({ request, params }) { - const waitpoint = await prisma.waitpoint.findFirst({ where: { id: params.id } }); - if (!verifyHttpCallbackHash(params.hash, waitpoint)) { + const invoice = await prisma.invoice.findFirst({ where: { id: params.id } }); + if (!verifyWebhookSignature(params.hash, invoice)) { return json({ error: "Invalid" }, { status: 401 }); } return json({ ok: true }); diff --git a/internal-packages/observability-map/test/sensitivity.test.ts b/internal-packages/observability-map/test/sensitivity.test.ts index 9a5c12e5f6a..d7ff63edde3 100644 --- a/internal-packages/observability-map/test/sensitivity.test.ts +++ b/internal-packages/observability-map/test/sensitivity.test.ts @@ -99,3 +99,76 @@ describe("classifySensitivity: calleeNames is body-scoped, importedNames is file expect(classifySensitivity(e).sensitive).toBe(false); }); }); + +describe("what sensitivity must not mean", () => { + const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + + // C4. Calling the admin guard cannot be what makes a route risky: it is the mitigation, not the + // hazard. Counting it made 34 of 67 sensitive entries sensitive only because they were guarded, + // and `auth-boundary` then passed every one of them on the same call. Circular, and it was the + // fix list's primary sort key. + it("does not treat calling the admin guard as what makes a route sensitive", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.queue-metrics.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return json(await prisma.queueMetric.findMany()); + }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags an admin route that does something sensitive on its own account", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return setImpersonation(request, "user_1"); + }` + ) + ); + expect(s.sensitive).toBe(true); + expect(s.reasons).toContain("calls setImpersonation"); + }); + + // A waitpoint token is a run coordination handle, not a credential. Seven of the eight routes + // matching the `tokens` segment were waitpoint routes. + it("does not treat a waitpoint token route as a credential route", () => { + const s = classifySensitivity( + ep( + "api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.waitpoint.update({ where: {}, data: {} }); }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags the personal access token routes", () => { + expect( + classifySensitivity( + ep( + "account.tokens/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.personalAccessToken.findMany(); }` + ) + ).reasons + ).toContain('path segment "tokens"'); + expect( + classifySensitivity( + ep( + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ) + ).reasons + ).toContain('path segment "token"'); + }); +}); From fe629644413df77024af7f6fcf381cd4a98b26a3 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:18:55 +0100 Subject: [PATCH 023/117] fix(observability-map): make the report and the CLI say what they mean An empty group meant mean 100, so a family with nothing measured rendered a full green bar; it renders as not measured now, and so does a global score with nothing behind it. The audit sentence claiming no helper exists is only printed when the figure in front of it says zero, and the whole line is skipped when no sensitive mutation was found. 'already solid' counted entries that pass because they do nothing alongside entries nothing applied to, so it now reports the two separately. The suppression directive is read from comments only, line by line. Matching the raw source meant a string literal quoting the directive switched a real check off. The CLI accepts the route paths the report prints, not just file names, so the identifier on screen can be pasted back in. An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns instead of silently taking the first. cli.ts had no tests and now has six, which needed the entry point guarded so importing the module does not scan the tree. Also unexports BUILDERS, isParseGuard and accountedFor, none of which had a caller outside their own module. --- .../src/checks/errorClassification.ts | 6 +- .../observability-map/src/checks/index.ts | 1 - .../observability-map/src/cli.ts | 64 +++++++++++++---- .../observability-map/src/report/terminal.ts | 43 ++++++++---- .../observability-map/src/score.ts | 21 ++++-- .../observability-map/src/suppression.ts | 42 +++++++++-- .../observability-map/test/checks.test.ts | 8 ++- .../observability-map/test/cli.test.ts | 60 ++++++++++++++++ .../observability-map/test/report.test.ts | 70 +++++++++++++++++++ 9 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 internal-packages/observability-map/test/cli.test.ts diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index f4a8ce5e8b2..c2d46b3af5b 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -13,7 +13,7 @@ const ID = "error-classification"; * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. */ -export const BUILDERS = new Set([ +const BUILDERS = new Set([ "createLoaderApiRoute", "createActionApiRoute", "createLoaderPATApiRoute", @@ -33,7 +33,7 @@ export const BUILDERS = new Set([ * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement * one alike. */ -export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { +function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; } @@ -51,7 +51,7 @@ export function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * effort side work that logs and carries on, and all four are non-sensitive so they sort to the * bottom of the fix list. See the task 5 report; switching is one limb in this function. */ -export function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { +function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.rethrows || clause.branches || isParseGuard(clause, ep); } diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts index b94c8ad4867..73025d0c5c8 100644 --- a/internal-packages/observability-map/src/checks/index.ts +++ b/internal-packages/observability-map/src/checks/index.ts @@ -9,4 +9,3 @@ export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; /** audit-trail is scored separately, see score.ts. */ export const CHECKS: Check[] = [errorClassification, authBoundary, requestContext, auditTrail]; export const SCORED_CHECK_IDS = ["error-classification", "auth-boundary", "request-context"]; -export { usesBuilder, BUILDERS } from "./errorClassification.js"; diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 360f39a84f6..6feef381e33 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; +import { routePathOf } from "./adapters/remix.js"; import { renderTerminal } from "./report/terminal.js"; import { renderJson } from "./report/json.js"; @@ -25,7 +26,36 @@ function findRepoRoot(startDir: string): string { throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); } -export function main(argv: string[]): number { +/** Where output goes. Injectable so the tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** + * Entry points matching what the user typed, by file name or by route path, exact first. + * + * Route paths matter because they are what the report prints: `map /api/v1/token` used to exit 1 + * because only the file name was matched, so the identifier on screen was not one you could paste + * back in. + */ +function findMatches(entryPoints: { fileName: string }[], target: string) { + const asPath = target.startsWith("/") ? target : `/${target}`; + const asFile = target.replace(/^\//, ""); + + const exact = entryPoints.filter( + (e) => e.fileName === target || routePathOf(e.fileName) === asPath + ); + if (exact.length > 0) return exact; + + return entryPoints.filter( + (e) => e.fileName.startsWith(asFile) || routePathOf(e.fileName).startsWith(asPath) + ); +} + +export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); const asJson = args.includes("--json"); const noWrite = args.includes("--no-write"); @@ -36,32 +66,42 @@ export function main(argv: string[]): number { const { entryPoints, parseFailures } = scanDirectory(routesDir); if (target) { - const match = entryPoints.find( - (e) => e.fileName === target || e.fileName.startsWith(target.replace(/^\//, "")) - ); - if (!match) { - process.stderr.write(`no entry point matching "${target}"\n`); + const matches = findMatches(entryPoints, target); + if (matches.length === 0) { + io.err(`no entry point matching "${target}"\n`); return 1; } - const scored = scoreEntry(match); + if (matches.length > 1) { + const others = matches.slice(1, 4).map((m) => routePathOf(m.fileName)); + const rest = matches.length - 1 - others.length; + io.err( + `"${target}" matches ${matches.length} entry points, showing the first. ` + + `Others: ${others.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}\n` + ); + } + const scored = scoreEntry(matches[0]!); const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; - process.stdout.write( + io.out( `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` ); for (const c of scored.checks) { const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; - process.stdout.write(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + io.out(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); } return 0; } const report = buildReport(entryPoints, parseFailures); - process.stdout.write(asJson ? renderJson(report) : renderTerminal(report)); - process.stdout.write("\n"); + io.out(asJson ? renderJson(report) : renderTerminal(report)); + io.out("\n"); if (!noWrite) { writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); } return 0; } -process.exitCode = main(process.argv); +// Only when run as a program. Importing the module, which the tests do, must not scan the tree or +// write a report. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 312b7b8a16c..21facadb21c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -1,9 +1,13 @@ import type { MapReport, ScoredEntry } from "../score.js"; import { SCORED_CHECK_IDS } from "../checks/index.js"; -const bar = (score: number) => { +const NOT_MEASURED = "not measured".padEnd(15); + +/** A bar and a figure, or a plain "not measured" where there is no figure to draw. */ +const gauge = (score: number | null) => { + if (score === null) return NOT_MEASURED; const filled = Math.round(score / 10); - return "▰".repeat(filled) + "▱".repeat(10 - filled); + return `${"▰".repeat(filled)}${"▱".repeat(10 - filled)} ${String(score).padStart(3)}`; }; /** @@ -19,28 +23,36 @@ const scoredFailures = (e: ScoredEntry) => export function renderTerminal(report: MapReport): string { const lines: string[] = []; + const headline = report.global === null ? "score not measured" : `score ${report.global}/100`; lines.push( - `score ${report.global}/100 ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` + `${headline} ${report.measured} measured, ${report.unmeasured} unmeasured of ${report.entries.length} entry points` ); lines.push(""); lines.push("COVERAGE"); for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { lines.push( - ` ${family.padEnd(12)} ${bar(stats.mean)} ${String(stats.mean).padStart(3)} ${stats.measured}/${stats.n} entry points` + ` ${family.padEnd(12)} ${gauge(stats.mean)} ${stats.measured}/${stats.n} entry points` ); } lines.push( - ` ${"sensitive".padEnd(12)} ${bar(report.sensitiveCohort.mean)} ${String( - report.sensitiveCohort.mean - ).padStart(3)} ${report.sensitiveCohort.measured}/${report.sensitiveCohort.n} entry points` + ` ${"sensitive".padEnd(12)} ${gauge(report.sensitiveCohort.mean)} ${ + report.sensitiveCohort.measured + }/${report.sensitiveCohort.n} entry points` ); - lines.push(""); const { sensitiveMutations, withAudit } = report.auditGap; - lines.push( - `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + - `No audit helper exists in the webapp.` - ); + if (sensitiveMutations > 0) { + lines.push(""); + // The closing sentence is a claim about the codebase, so it is only made when the figure in + // front of it supports it. It was printed unconditionally, including next to a non-zero count. + const gap = + withAudit === 0 + ? " No audit helper exists in the webapp." + : ` ${sensitiveMutations - withAudit} without one.`; + lines.push( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}` + ); + } if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; @@ -78,7 +90,12 @@ export function renderTerminal(report: MapReport): string { } lines.push(""); - lines.push(`already solid: ${report.entries.length - worst.length}`); + // Not one flattering number: an entry with nothing applicable is not the same as an entry that + // passed, and lumping them together counted routes as solid for doing nothing. + const clean = report.entries.filter((e) => e.measured && scoredFailures(e).length === 0).length; + lines.push( + `no findings: ${clean} passed every applicable check, ${report.unmeasured} had none to apply` + ); if (report.parseFailures.length > 0) { lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); } diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 65782c79d05..27aaa680950 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -24,15 +24,16 @@ export type ScoredEntry = { }; export type MapReport = { - global: number; + /** Null when no entry point had an applicable scored check: an absent figure, not a perfect one. */ + global: number | null; /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ measured: number; /** Entry points every scored check reported not-applicable for; excluded from `global`. */ unmeasured: number; /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ suppressions: { entries: number; checks: number }; - byFamily: Record; - sensitiveCohort: { n: number; measured: number; mean: number }; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; entries: ScoredEntry[]; parseFailures: string[]; @@ -77,8 +78,12 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; } -const mean = (xs: number[]) => - xs.length === 0 ? 100 : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); +/** + * Null for an empty group rather than 100. A family nothing was measured in has no score, and + * rendering the absence as a full green bar said the opposite of what the data said. + */ +const mean = (xs: number[]): number | null => + xs.length === 0 ? null : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); /** * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an @@ -86,7 +91,11 @@ const mean = (xs: number[]) => * is reported alongside so a reader can tell a family scoring high because it is clean apart from * a family scoring high because most of it was never measured. */ -function groupStats(entries: ScoredEntry[]): { n: number; measured: number; mean: number } { +function groupStats(entries: ScoredEntry[]): { + n: number; + measured: number; + mean: number | null; +} { const measuredEntries = entries.filter((e) => e.measured); return { n: entries.length, diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index bd807816254..8f7352f6a46 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,13 +1,43 @@ -// The reason runs to the end of the line: `.` does not match a newline, so a suppression on one -// line cannot pick up a reason from the next one. -const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/g; +/** + * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` + * does not match a newline, so a suppression on one line cannot pick up a reason from the next. + * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. + */ +const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/; -/** Check id to reason. A suppression without a reason is ignored. */ +/** + * The comment part of a line, or null if there is none. + * + * Line-scoped and comment-only, because the directive is a comment directive. Matching the raw + * source meant a string literal that merely quotes the directive, in a test fixture or an error + * message, silently switched a real check off. Handles line comments, block comments and the + * leading star of a jsdoc block; a line-comment marker inside a string on the same line can still + * be misread, which costs a suppression that was never written rather than hiding one that was. + */ +function commentPart(line: string): string | null { + const slashes = line.indexOf("//"); + if (slashes !== -1) return line.slice(slashes + 2); + + const block = line.indexOf("/*"); + if (block !== -1) return line.slice(block + 2).replace(/\*\/\s*$/, ""); + + const trimmed = line.trimStart(); + if (trimmed.startsWith("*")) return trimmed.slice(1); + + return null; +} + +/** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ export function suppressedChecks(source: string): Map { const out = new Map(); - for (const match of source.matchAll(PATTERN)) { + for (const line of source.split("\n")) { + const comment = commentPart(line); + if (comment === null) continue; + const match = PATTERN.exec(comment); + if (!match) continue; const [, id, reason] = match; - if (id && reason && reason.trim().length > 0) out.set(id, reason.trim()); + const trimmedReason = reason?.replace(/\*\/\s*$/, "").trim(); + if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); } return out; } diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 081db1661ed..6aeb75a7ecb 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -64,7 +64,7 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - // Restored from the brief: `catchBranches` sees the `instanceof` and the `if`. + // Restored from the brief: the clause branches, on the `instanceof` and the `if`. it("passes a raw route whose catch branches on the error", () => { const r = run( "error-classification", @@ -135,7 +135,8 @@ describe("error-classification", () => { }); // A narrow guard around one operation classifies an expected failure without needing to branch - // or rethrow. `catchesNarrowly` is what tells it apart from a handler-wide catch. + // or rethrow. Guarding a parse over a small part of the body is what tells it apart from a + // handler-wide catch. it("passes a narrow guard around a single parse", () => { const r = run( "error-classification", @@ -153,7 +154,8 @@ describe("error-classification", () => { }); // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad - // handler catch sitting next to it. `catchesNarrowly` is false when any guarded try is broad. + // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still + // counts against the entry point. it("still fails when a narrow guard sits beside a handler-wide swallow", () => { const r = run( "error-classification", diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts new file mode 100644 index 00000000000..2dec21ca9a8 --- /dev/null +++ b/internal-packages/observability-map/test/cli.test.ts @@ -0,0 +1,60 @@ +import { main, type Io } from "../src/cli.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "cli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("map ", () => { + // I8. The report prints route paths, so the identifier on screen has to be one you can paste + // back in. Matching file names only meant `map /api/v1/token` exited 1. + it("accepts the route path the report prints", () => { + const r = run("/api/v1/token"); + expect(r.code).toBe(0); + expect(r.out).toContain("/api/v1/token"); + expect(r.out).toContain("CHECKS"); + }); + + it("accepts a file name too", () => { + const r = run("api.v1.token.ts"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.token.ts"); + }); + + it("exits 1 with a message when nothing matches", () => { + const r = run("/api/v1/does-not-exist"); + expect(r.code).toBe(1); + expect(r.err).toContain("no entry point matching"); + }); + + it("warns when a prefix matches more than one route rather than silently taking the first", () => { + const r = run("/admin/api/v1/runs-replication"); + expect(r.code).toBe(0); + expect(r.err).toMatch(/matches \d+ entry points, showing the first/); + expect(r.err).toContain("Others:"); + }); + + // `/api/v1/runs` is a prefix of a dozen others, and also a route in its own right. + it("prefers an exact match over the routes it is a prefix of", () => { + const r = run("/api/v1/runs"); + expect(r.err).toBe(""); + expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); + }); +}); + +describe("map", () => { + it("renders the whole report without writing when asked not to", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.out).toContain("COVERAGE"); + expect(r.out).toContain("FIX FIRST"); + }); +}); diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 39fef9730e4..7a28c7a4b9c 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -136,3 +136,73 @@ describe("renderJson", () => { expect(Array.isArray(parsed.entries)).toBe(true); }); }); + +describe("rendering honestly when there is nothing to say", () => { + // I4. mean([]) returned 100, so a family with nothing measured rendered a full green bar. + it("renders a family with nothing measured as not measured, not as 100", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + const line = out.split("\n").find((l) => l.includes("resources"))!; + expect(line).not.toMatch(/100/); + expect(line).toMatch(/not measured/i); + }); + + it("renders the global score as not measured when nothing was measured", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + expect(renderTerminal(buildReport([trivial], []))).toMatch(/score not measured/i); + }); + + // I11. The audit sentence was printed unconditionally, including when the figure said otherwise. + it("does not claim no audit helper exists when one is in use", () => { + const audited = scanFile( + "api.v1.auth.tokens.ts", + `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }` + )!; + const out = renderTerminal(buildReport([audited], [])); + expect(out).toContain("1 of 1"); + expect(out).not.toContain("No audit helper exists"); + }); + + it("says nothing about audit when no sensitive mutation was found", () => { + const plain = scanFile( + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + expect(renderTerminal(buildReport([plain], []))).not.toMatch(/AUDIT/); + }); + + // I5. "already solid" counted entries that are clean because they do nothing, alongside entries + // nothing applied to, in one flattering number. + it("separates entries that passed from entries nothing applied to", () => { + const clean = scanFile( + "api.v1.clean.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([clean, trivial], [])); + expect(out).not.toMatch(/already solid/i); + expect(out).toMatch(/1 passed every applicable check/i); + expect(out).toMatch(/1 had none to apply/i); + }); +}); From 2f4823a7af12e36ed02810aec19601046208f1b7 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:19:17 +0100 Subject: [PATCH 024/117] test(observability-map): cover the comment-only suppression rule The directive inside a string literal must not suppress anything, and block comments and jsdoc lines must be read like line comments. --- .../test/suppression.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 8e8e88d5fa5..0bb6fcc5f9b 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -42,4 +42,40 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // I2. The directive is a comment directive. Matching it file-wide meant a string literal that + // merely quotes it, in a test fixture or an error message, silently suppressed a real check. + it("ignores the directive inside a string literal", () => { + const m = suppressedChecks( + `const example = "obs-map-disable-next-line error-classification -- not a real suppression"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("reads the directive from a block comment", () => { + const m = suppressedChecks( + `/* obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 */ + export async function loader() { return 1; }` + ); + expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); + }); + + it("reads the directive from a jsdoc line", () => { + const m = suppressedChecks( + `/** + * obs-map-disable-next-line request-context -- nothing tenant-scoped here + */ + export async function loader() { return 1; }` + ); + expect(m.get("request-context")).toBe("nothing tenant-scoped here"); + }); + + it("ignores code that happens to follow a comment on the same line", () => { + const m = suppressedChecks( + `const x = 1; // obs-map-disable-next-line error-classification -- fine + export async function loader() { return x; }` + ); + expect(m.get("error-classification")).toBe("fine"); + }); }); From ec5922a32b2c9f19c1f0e8c7a49c916d46fb14de Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:27:27 +0100 Subject: [PATCH 025/117] fix(observability-map): report the request-context gap as a figure, not a list request-context fails 391 of the 412 entry points it applies to, so listing each one turned the fix list into a single house-style finding repeated. That is the same reason audit-trail is kept out of the list, at 52 rather than 391. An entry whose only finding is request-context now collapses into a CONTEXT line reading how many entry points name a tenant on a failure path, and the line says how many appear only there so they are not mistaken for having disappeared. An entry that fails something else keeps its full set of findings, so /account/tokens still shows the request-context gap alongside the swallow. The check stays fully in the score. The gap is real and the score is meant to show it; only the presentation changes. --- .../observability-map/src/report/terminal.ts | 26 ++++++- .../observability-map/src/score.ts | 14 ++++ .../observability-map/test/report.test.ts | 68 +++++++++++++++---- .../observability-map/test/score.test.ts | 24 +++++++ 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 21facadb21c..868acf284d6 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -20,6 +20,18 @@ const gauge = (score: number | null) => { const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); +/** + * An entry whose only finding is `request-context`. 391 of 412 entry points fail that check, so + * listing each one turns the fix list into a single house-style finding repeated, which is the + * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. + * An entry that fails something else as well stays in the list with all of its findings, so a + * route like `/account/tokens` still shows the request-context gap alongside the rest. + */ +const contextOnly = (e: ScoredEntry) => { + const failures = scoredFailures(e); + return failures.length === 1 && failures[0]!.id === "request-context"; +}; + export function renderTerminal(report: MapReport): string { const lines: string[] = []; @@ -54,6 +66,18 @@ export function renderTerminal(report: MapReport): string { ); } + const { applicable, naming } = report.contextGap; + if (applicable > 0) { + const collapsed = report.entries.filter(contextOnly).length; + lines.push(""); + lines.push( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed > 0 + ? ` ${collapsed} appear${collapsed === 1 ? "s" : ""} only here, not in the list below.` + : "") + ); + } + if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; lines.push( @@ -65,7 +89,7 @@ export function renderTerminal(report: MapReport): string { } const worst = report.entries - .filter((e) => scoredFailures(e).length > 0) + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) .sort( (a, b) => Number(b.sensitive) - Number(a.sensitive) || diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 27aaa680950..d9493517864 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -35,6 +35,12 @@ export type MapReport = { byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; + /** + * `request-context` fails 391 of the 412 entry points it applies to, so it is reported as a + * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` + * gets. It stays fully in the score: the gap is real and the score is meant to show it. + */ + contextGap: { applicable: number; naming: number }; entries: ScoredEntry[]; parseFailures: string[]; }; @@ -119,6 +125,10 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // reported here as its own architectural figure instead: how many sensitive mutations have an // audit record, out of how many. Folding it into the score would tank every sensitive route on a // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + const contextChecks = entries + .map((e) => e.checks.find((c) => c.id === "request-context")) + .filter((c) => c !== undefined && c.status !== "not-applicable"); + const auditApplicable = entries.filter((e) => e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); @@ -141,6 +151,10 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") ).length, }, + contextGap: { + applicable: contextChecks.length, + naming: contextChecks.filter((c) => c.status === "pass").length, + }, entries, parseFailures, }; diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 7a28c7a4b9c..76680fb6072 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -75,21 +75,19 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 67: guarded and classifies what it catches, but its failure log names - // nobody, so only request-context fails. - const sensitiveSixtySeven = scanFile( + // Sensitive, score 33: classifies what it catches, but has no guard and names nobody. Fails + // more than request-context, so it stays in the list rather than collapsing into the figure. + const sensitiveThirtyThree = scanFile( "api.v1.auth.tokens.ts", - `import { requireUserId } from "~/services/session.server"; - import { logger } from "~/services/logger.server"; + `import { logger } from "~/services/logger.server"; import { prisma } from "~/db.server"; - export async function action({ request }) { - const userId = await requireUserId(request); - try { return await prisma.token.create({ data: { userId } }); } + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (error) { logger.error("failed", { error }); throw error; } }` )!; - // Not sensitive, score 0: worse score than sensitiveSixtySeven, but must still sort after both + // Not sensitive, score 0: worse score than sensitiveThirtyThree, but must still sort after both // sensitive entries because sensitivity outranks raw score. const notSensitiveZero = scanFile( "resources.busy.ts", @@ -114,17 +112,17 @@ describe("renderTerminal", () => { )!; const out = renderTerminal( - buildReport([sensitiveSixtySeven, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) ); const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); - const idxSixtySeven = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); expect(idxZero).toBeGreaterThan(-1); - expect(idxSixtySeven).toBeGreaterThan(idxZero); - expect(idxNotSensitive).toBeGreaterThan(idxSixtySeven); + expect(idxThirtyThree).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxThirtyThree); expect(fixFirst).not.toContain("api.v1.billing.ts"); }); }); @@ -206,3 +204,47 @@ describe("rendering honestly when there is nothing to say", () => { expect(out).toMatch(/1 had none to apply/i); }); }); + +describe("collapsing the house-style finding", () => { + const namesNobody = () => + scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + + const namesNobodyAndSwallows = () => + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }` + )!; + + // request-context fails 391 of 412 entry points, so listing each one turns the fix list into a + // single finding repeated. Same reasoning that keeps audit-trail out of the list. + it("keeps an entry whose only finding is request-context out of the fix list", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + const fixFirst = out.slice(out.indexOf("FIX FIRST")); + expect(fixFirst).not.toContain("api.v1.silent.ts"); + }); + + it("reports the gap as a headline figure instead", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); + }); + + it("says how many entries the collapse took out of the list", () => { + const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); + expect(out).toMatch(/1 appears? only here/i); + }); + + it("still lists request-context when the entry fails something else too", () => { + const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); + const fixFirst = out.slice(out.indexOf("FIX FIRST")); + expect(fixFirst).toContain("api.v1.auth.tokens.ts"); + expect(fixFirst).toContain("request-context"); + expect(fixFirst).toContain("error-classification"); + }); +}); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index f962aa2b8c8..85cee860676 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -138,6 +138,30 @@ ${BUSY_AND_FAILING}` }); describe("buildReport", () => { + it("reports the request-context gap as a figure, like the audit gap", () => { + const naming = scanFile( + "api.v1.named.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const silent = scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + + const report = buildReport([naming, silent, trivial], []); + expect(report.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + it("counts suppressions so laundering is visible in the report", () => { const report = buildReport( [ From 99e14ca1a9666f5eb32dd360678df0b52405b811 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:29:43 +0100 Subject: [PATCH 026/117] docs(observability-map): explain what the number means For a tool whose only value is being believed, there was nothing in the repo explaining the score, the checks, or where they are wrong. Covers both CLI modes, what 19 means and why it is deliberately unflattering, the four checks a line each, why the audit and context gaps are figures rather than list entries, why an unmeasured entry is excluded rather than scored, the visibility test every applicability decision follows, that a suppression needs a reason and cannot raise a score, and the known limits: one hop and same file only, loggers matched by spelling, only the first object-literal argument read, inline callbacks not descended into. --- internal-packages/observability-map/README.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 internal-packages/observability-map/README.md diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md new file mode 100644 index 00000000000..c4d2041868e --- /dev/null +++ b/internal-packages/observability-map/README.md @@ -0,0 +1,143 @@ +# @internal/observability-map + +Scores every webapp entry point on whether it could explain itself during an incident, and prints +the ones worth fixing. An entry point is a Remix `loader` or `action` under +`apps/webapp/app/routes`, 427 of them at the time of writing. + +The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly +about why you should believe it. + +## Running it + +```bash +pnpm --filter @internal/observability-map run map # the whole tree +pnpm --filter @internal/observability-map run map --json # same, as JSON on stdout +pnpm --filter @internal/observability-map run map /api/v1/token # one route, with its check results +``` + +The whole-tree run also writes `observability-map.json` at the repo root, which `--no-write` +suppresses. The single-route mode takes either the route path the report prints (`/api/v1/token`) or +the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an +ambiguous prefix warns and names the alternatives rather than silently picking one. + +## What 19 means + +It is the mean score of the 412 entry points that had at least one applicable check, where an +entry's score is the share of its applicable checks that passed. It is low because the webapp does +not attach tenant identity to its failures: **21 of 412 entry points name an environment, project, +organization, run or user on a failure path.** Everything else, when it breaks at 3am, tells you the +route and the request id and nothing about whose request it was. + +The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying +every catch clause in the tree used to score it 100, which meant the metric paid you for deleting +error handling. Now removing the catches takes 19 down to 8, and removing the logs as well takes it +to 2. If you change this package, keep that property: mutate the tree to remove error handling and +check the score falls. + +So the number is deliberately unflattering, and one platform change would move most of it. Nothing +central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line +through AsyncLocalStorage and forwards errors to Sentry, and the route builders log +`logBoundaryError(message, error, url)`. If the auth path ever pushed `environmentId` through +`trace(...)`, several hundred entry points would flip at once, and this check would want rethinking +rather than celebrating. + +## The four checks + +- **error-classification**: does every catch clause decide what it caught, by rethrowing, by + branching on the error, or by guarding a parse it can answer for. +- **auth-boundary**: does a route handling credentials, tokens, billing or impersonation check who + is asking. +- **request-context**: when this entry point's failure is reported, is the tenant named. +- **audit-trail**: does a sensitive mutation leave a record of who did it. Nothing in the webapp + writes one, so every applicable entry point fails. + +`audit-trail` is excluded from the score. `request-context` is in it. + +## Two findings are headlines, not list entries + +`audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route +would bury the route-specific findings under the same sentence repeated hundreds of times, so both +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 67 in the fix list. An entry that +fails `request-context` *and* something else keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. + +`request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score +is meant to show it. Only the presentation collapses. + +## Not applicable is not a pass + +An entry with no applicable scored check is `measured: false`, and it is left out of every mean the +report computes. Its `score` field reads 100, which is a placeholder for "nothing was measured +here", not a verdict, and nothing averages it. This matters because the alternative, letting +unmeasured entries into the mean at 100, would let the tool look better the less it understood. The +header prints both counts (`412 measured, 15 unmeasured`) so the denominator is never hidden, and a +family with nothing measured renders as `not measured` rather than as a full green bar. + +## When a check declines to judge + +The rule every applicability decision follows: **would this evidence necessarily be visible in the +body if it existed?** + +A log call inside a catch would be, because the catch is right there in the body being read. So its +absence is evidence of absence and `request-context` fails the route. A guard on work that happens +inside an imported helper would not be, because neither the work nor the guard is in the body. So +`auth-boundary` reports not-applicable with a detail saying it could not verify, rather than +accusing the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls +`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, +a file this tool never opens. + +The failure mode this rule exists to prevent is a fix list whose top three entries are all wrong. +That happened, twice, and both times the cause was a check asserting something the evidence did not +support. + +## Suppression + +```ts +// obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 +``` + +The reason is mandatory: a suppression without one is ignored. The directive is read from comments +only, line by line, so a string literal quoting it does not switch a check off. + +A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, +and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing +check holds the number still rather than improving it. What you buy is removal from the worklist +with a reason on the record. The report prints how many suppressions are in force so the practice +stays visible. + +## Known limits + +Read these before trusting a specific verdict. + +- **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's + statements, catches and calls count as the route's. A helper's own helpers do not, and nothing + imported from another module is ever opened. Most of what a route does is behind an import, which + is why `auth-boundary` applies to 26 entry points rather than 427. +- **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or + `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route + can be reported as recording nothing while it records plenty. +- **Only the first object-literal argument is read** for identifier fields, and only its property + names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither + does a second object. +- **Inline callbacks are not descended into** when counting statements, so a two-statement body can + hold a pile of work inside a `.map()`. The call count is what catches those cases, imperfectly. +- **Sensitivity is a heuristic**: a symbol list plus path segments. It was circular until recently, + counting `requireAdminApiRequest` as a hazard when it is a mitigation, which made 34 of 67 + sensitive routes sensitive purely for being guarded. Expect it to need pruning again as routes + move. +- **The score is a mean of means over a heuristic.** Read the fix list and the two headline figures. + Watching the single number for small movements will mislead you. + +## Layout + +`scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only +body-scoped evidence. `checks/` holds the four checks, each a pure function of an `EntryPoint`. +`score.ts` turns checks into an entry score and a report, `report/` renders it, `cli.ts` is the +entry point. `sensitivity.ts`, `triviality.ts` and `suppression.ts` are the three inputs the checks +share. + +Tests sit next to their subject in `test/`. Every check has a false-positive fixture, something it +must not flag, alongside the positive one. Keep that: most of the bugs this package has had were +checks that fired on the wrong thing, and a test that only proves the heuristic fires would have +caught none of them. From 3c748691efbc8843a4e2caa0b2aea5e0a6cc76e5 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:48:02 +0100 Subject: [PATCH 027/117] fix(observability-map): make the package typecheck and build again Three type errors from the last wave broke the root typecheck workflow and the internal-package build that gates the unit test job. findMatches was typed on a structural subset of EntryPoint and handed the result to scoreEntry, the local byFamily type had not followed groupStats to number or null, and the request-context filter was not a type predicate so the element stayed possibly undefined. Also sets noEmitOnError on the build config. tsc was exiting 2 and emitting anyway, so a broken build left a usable dist behind and only the exit code showed it. --- internal-packages/observability-map/src/cli.ts | 3 ++- internal-packages/observability-map/src/score.ts | 4 ++-- internal-packages/observability-map/tsconfig.build.json | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 6feef381e33..4be0ff59193 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -1,6 +1,7 @@ import { existsSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import type { EntryPoint } from "./types.js"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; import { routePathOf } from "./adapters/remix.js"; @@ -41,7 +42,7 @@ const processIo: Io = { * because only the file name was matched, so the identifier on screen was not one you could paste * back in. */ -function findMatches(entryPoints: { fileName: string }[], target: string) { +function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { const asPath = target.startsWith("/") ? target : `/${target}`; const asFile = target.replace(/^\//, ""); diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index d9493517864..1203f730158 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -114,7 +114,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo const entries = eps.map(scoreEntry); const measuredEntries = entries.filter((e) => e.measured); - const byFamily: Record = {}; + const byFamily: MapReport["byFamily"] = {}; for (const family of new Set(entries.map((e) => e.family))) { byFamily[family] = groupStats(entries.filter((e) => e.family === family)); } @@ -127,7 +127,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // gap that is the same everywhere, and bury the routes that have their own, fixable problems. const contextChecks = entries .map((e) => e.checks.find((c) => c.id === "request-context")) - .filter((c) => c !== undefined && c.status !== "not-applicable"); + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); const auditApplicable = entries.filter((e) => e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json index 254fcfbca49..6b6b2863f1f 100644 --- a/internal-packages/observability-map/tsconfig.build.json +++ b/internal-packages/observability-map/tsconfig.build.json @@ -3,6 +3,9 @@ "include": ["src/**/*.ts"], "compilerOptions": { "noEmit": false, + // A build that does not compile must not leave a dist behind: three type errors shipped in the + // last wave while `build` still emitted, so the failure was only visible in the exit code. + "noEmitOnError": true, "declaration": true, "outDir": "dist", "rootDir": ".", From 655b3caef0db14d57149222eb208a0b6c54a63d8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:52:13 +0100 Subject: [PATCH 028/117] fix(observability-map): stop paying for a catch that only rethrows Wrapping a body in try { ... } catch (e) { throw e } moved error-classification from not-applicable to pass, worth 50 points a route. Applied to the tree it took the global from 19 to 46, 223 entries up and none down, for a change that does nothing: the error propagates either way, so scoring the two differently paid for a semantic no-op. A clause now qualifies by branching or by guarding a parse. One that only rethrows is inert and reads exactly as no catch does, not-applicable. The same mutation is now flat at 18, no entry moves in either direction, while deleting error handling still costs, 18 down to 8 and to 2 with the logs as well. The cost is that a catch which logs and rethrows also reads as inert, since the clause evidence cannot say whether it does anything besides rethrow. That withholds credit rather than granting it, and crediting it would reopen the hole a single logger.error line wide. request-context still asks whether that log names a tenant. Both directions of the invariant are asserted in tests now, not just measured once: no-op error handling must not raise a score, deleting error handling must not raise a score. --- .../src/checks/errorClassification.ts | 65 ++++++++++++------ .../observability-map/test/checks.test.ts | 7 +- .../observability-map/test/report.test.ts | 13 ++-- .../observability-map/test/score.test.ts | 68 +++++++++++++++++++ 4 files changed, 124 insertions(+), 29 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index c2d46b3af5b..2c48f40dded 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -38,21 +38,39 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { } /** - * Whether a clause has decided what the error means. Rethrowing is a decision, branching is a - * decision, and guarding a parse answers for the one thing the guard covers. + * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, on + * an `if`, a `switch` or an `instanceof`, or it guards a parse it can answer for. * - * `narrow` is deliberately not a fourth way to qualify, which is where this differs from the rule - * the scanner work proposed. A one-statement try around `await service.call(run)` is narrow and is - * still a swallow. Taking the narrow limb clears eleven more entry points, and reading all eleven - * says six are real: the silent cancel in `api.v2.runs.$runParam.cancel.ts`, the PAT revoke in - * `account.tokens/route.tsx` and the invite revoke, both of which report a database failure to the - * browser as a 400 with an internal message in it, a `.map` that drops a broken dashboard on the - * floor, and two more. The four it would rightly clear are all the same deliberate shape, best - * effort side work that logs and carries on, and all four are non-sensitive so they sort to the - * bottom of the fix list. See the task 5 report; switching is one limb in this function. + * Rethrowing is not a third way, which is the correction from the last wave. A clause whose only + * effect is `throw e` leaves the error propagating exactly as it would with no catch at all, so + * treating that as a pass while no catch is not-applicable paid 50 points a route for wrapping a + * body in `try { ... } catch (e) { throw e }`, and 27 across the tree. The two are observationally + * identical and are now scored identically. + * + * The cost is real and worth stating: `catch (e) { logger.error(...); throw e }` also reads as + * inert, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. That + * withholds credit from a route that reports before propagating, which is the safe direction to be + * wrong in, since crediting it would reopen the hole a bare `logger.error` line wide. + * `request-context` still reads that log and asks whether it names a tenant, so the reporting is + * unrewarded here rather than unmeasured. + * + * `narrow` is not a way to qualify either. A one-statement try around `await service.call(run)` is + * narrow and is still a swallow: reading all eleven entry points that limb would clear said six + * were real, including a silent run cancellation and two credential paths that report a database + * failure to the browser as a 400 with an internal message in it. */ -function accountedFor(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.rethrows || clause.branches || isParseGuard(clause, ep); +function decides(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.branches || isParseGuard(clause, ep); +} + +/** Passes the error through unchanged, which is the same outcome as not catching it. */ +function inert(clause: CatchEvidence, ep: EntryPoint): boolean { + return clause.rethrows && !decides(clause, ep); +} + +/** The error stops here and nothing chose what it meant. */ +function swallows(clause: CatchEvidence, ep: EntryPoint): boolean { + return !decides(clause, ep) && !inert(clause, ep); } export function usesBuilder(ep: EntryPoint): boolean { @@ -88,21 +106,24 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - if (ep.catches.length === 0) { + const swallowed = ep.catches.filter((c) => swallows(c, ep)); + if (swallowed.length > 0) { + const which = + ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; return { id: ID, - status: "not-applicable", - detail: "catches nothing, so it classifies nothing", + status: "fail", + detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - const unaccounted = ep.catches.filter((c) => !accountedFor(c, ep)); - if (unaccounted.length > 0) { - const which = - ep.catches.length > 1 ? ` (${unaccounted.length} of ${ep.catches.length} catches)` : ""; + if (!ep.catches.some((c) => decides(c, ep))) { return { id: ID, - status: "fail", - detail: `catches its errors and takes one way out regardless of what was thrown${which}`, + status: "not-applicable", + detail: + ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "every catch rethrows and nothing else, so it classifies nothing", }; } return { id: ID, status: "pass", detail: "every catch decides what it caught" }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 6aeb75a7ecb..cbc0c73a00d 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -78,7 +78,10 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); - it("passes a raw route whose catch rethrows without branching", () => { + // A clause that only rethrows makes no classification decision: the error propagates exactly as + // it would with no catch at all, so it is read as no catch at all. Scoring the two differently + // paid 50 points a route for wrapping a body in `try { ... } catch (e) { throw e }`. + it("is not applicable to a raw route whose catch only rethrows", () => { const r = run( "error-classification", "api.v1.v.ts", @@ -89,7 +92,7 @@ describe("error-classification", () => { catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } }` ); - expect(r.status).toBe("pass"); + expect(r.status).toBe("not-applicable"); }); // The builder only classifies what reaches it. A swallow inside the handler never does, so the diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 76680fb6072..bb80ca1bb6c 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -75,15 +75,18 @@ describe("renderTerminal", () => { }` )!; - // Sensitive, score 33: classifies what it catches, but has no guard and names nobody. Fails - // more than request-context, so it stays in the list rather than collapsing into the figure. + // Sensitive, score 33: its catch decides something, telling a bad request apart from the rest, + // but it has no guard and names nobody. Fails more than request-context, so it stays in the + // list rather than collapsing into the figure. const sensitiveThirtyThree = scanFile( "api.v1.auth.tokens.ts", - `import { logger } from "~/services/logger.server"; - import { prisma } from "~/db.server"; + `import { prisma } from "~/db.server"; export async function action() { try { return await prisma.token.create({ data: {} }); } - catch (error) { logger.error("failed", { error }); throw error; } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } }` )!; diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 85cee860676..e677d95f3f1 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -216,3 +216,71 @@ ${BUSY_AND_FAILING}` expect(family.mean).toBe(scoreEntry(busy).score); }); }); + +/** + * The invariant, both ways round. The README states one direction, removing error handling must + * lower the score, and that alone could not see the free-points path: adding a catch that only + * rethrows changes nothing about how the route behaves, and used to move it from not-applicable to + * pass, worth 50 points a route and 27 points across the tree. + */ +describe("no-op error handling must not pay", () => { + const BODY = `const rows = await prisma.thing.findMany(); + return json({ rows });`; + + const plain = `import { prisma } from "~/db.server"; +export async function loader() { + ${BODY} +}`; + + const wrappedInARethrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + throw e; + } +}`; + + const handled = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { + ${BODY} + } catch (error) { + if (error instanceof NotFoundError) return json({ error: "not found" }, { status: 404 }); + logger.error("thing lookup failed", { environmentId: params.envId, error }); + throw error; + } +}`; + + it("does not pay for wrapping a body in a catch that only rethrows", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", plain)!); + const after = scoreEntry(scanFile("api.v1.x.ts", wrappedInARethrow)!); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { + const before = buildReport( + [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], + [] + ); + const after = buildReport( + [scanFile("api.v1.x.ts", wrappedInARethrow)!, scanFile("api.v1.y.ts", wrappedInARethrow)!], + [] + ); + expect(after.global!).toBeLessThanOrEqual(before.global!); + }); + + it("does not pay for deleting error handling either", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", handled)!); + const after = scoreEntry(scanFile("api.v1.x.ts", plain)!); + expect(after.score).toBeLessThanOrEqual(before.score); + // And the handled version is genuinely better, so the invariant is not holding by both being 0. + expect(before.score).toBeGreaterThan(after.score); + }); + + it("still credits a catch that decides something on its way through", () => { + const scored = scoreEntry(scanFile("api.v1.x.ts", handled)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + }); +}); From 7754cac6b503f07bc15864010e2fa3834e6e4523 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Fri, 31 Jul 2026 22:57:08 +0100 Subject: [PATCH 029/117] fix(observability-map): name the suppression directive honestly, and tighten the tests obs-map-disable-next-line applied to the whole entry point, so a directive on the last line of a file switched a check off for everything above it. Scoping it to a line is not available, since a finding is attached to an entry point and carries no line number to match against, and inventing a proximity rule would silently drop legitimate suppressions. So the name is now obs-map-disable, which is what it does. The old spelling is not honoured and a test says so. The CONTEXT line now says how many of the collapsed entries are sensitive, 18 of 333, so a reader knows to open the JSON rather than trusting the list. Three tests were passing by luck. The FIX FIRST ordering test sliced on a string the report no longer prints, so its assertions ran against the whole tail. The --no-write test never checked that no file was written, so it would have written into the repo root if the flag broke. A suppression test asserted toContain('1') against a report full of digits, and another never asserted the measured flag it was named for. README corrected: the score is 18, auth-boundary applies to 23 entry points rather than 26 and gates on sensitivity before the one-hop limit, and the invariant section now states both directions. --- internal-packages/observability-map/README.md | 48 ++++++++++++++----- .../observability-map/src/report/terminal.ts | 8 ++-- .../observability-map/src/suppression.ts | 10 +++- .../observability-map/test/cli.test.ts | 11 +++++ .../observability-map/test/report.test.ts | 27 +++++++++-- .../observability-map/test/score.test.ts | 20 ++++---- .../test/suppression.test.ts | 31 ++++++++---- 7 files changed, 118 insertions(+), 37 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index c4d2041868e..457c22ff390 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 18 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -20,7 +20,7 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. -## What 19 means +## What 18 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does @@ -30,9 +30,18 @@ route and the request id and nothing about whose request it was. The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying every catch clause in the tree used to score it 100, which meant the metric paid you for deleting -error handling. Now removing the catches takes 19 down to 8, and removing the logs as well takes it -to 2. If you change this package, keep that property: mutate the tree to remove error handling and -check the score falls. +error handling. + +Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: + +- **Removing error handling must not raise the score.** Deleting every catch clause takes 18 to 8, + and deleting the logs as well takes it to 2. +- **Adding error handling that does nothing must not raise the score.** Wrapping every body in + `try { ... } catch (e) { throw e }` leaves it at 18, with no entry moving in either direction. + That mutation used to be worth 27 points across the tree, because a rethrow-only clause counted + as a pass while no catch at all was not-applicable, and the two are observationally identical. + +If you change this package, check both directions still hold. So the number is deliberately unflattering, and one platform change would move most of it. Nothing central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line @@ -43,8 +52,9 @@ rather than celebrating. ## The four checks -- **error-classification**: does every catch clause decide what it caught, by rethrowing, by - branching on the error, or by guarding a parse it can answer for. +- **error-classification**: does every catch clause decide what it caught, by branching on the + error or by guarding a parse it can answer for. A clause that only rethrows decides nothing and + is read as though there were no catch, so it neither passes nor fails. - **auth-boundary**: does a route handling credentials, tokens, billing or impersonation check who is asking. - **request-context**: when this entry point's failure is reported, is the tenant named. @@ -62,6 +72,11 @@ are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail fails `request-context` *and* something else keeps both findings and stays in the list, so `/account/tokens` still shows the whole picture. +18 of those 333 are sensitive, including `/admin/impersonate`, the API-key regeneration route and +four envvars routes, so the `CONTEXT` line says how many. Read them out of +`observability-map.json`, where every entry keeps its full check results, rather than assuming the +list is the whole story. + `request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score is meant to show it. Only the presentation collapses. @@ -94,11 +109,17 @@ support. ## Suppression ```ts -// obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 +// obs-map-disable auth-boundary -- public by design, see ADR 12 ``` The reason is mandatory: a suppression without one is ignored. The directive is read from comments -only, line by line, so a string literal quoting it does not switch a check off. +only, so a string literal quoting it does not switch a check off. + +It applies to the whole entry point, not to the line under it. It was called +`obs-map-disable-next-line`, which was untrue in a way that mattered: a directive on the last line +of a file switched a check off for everything above it. Genuine line scoping is not available, +because a finding is attached to an entry point and carries no line number to match against, so the +name was corrected instead. The old spelling is not honoured, and there is a test saying so. A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing @@ -112,11 +133,16 @@ Read these before trusting a specific verdict. - **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's statements, catches and calls count as the route's. A helper's own helpers do not, and nothing - imported from another module is ever opened. Most of what a route does is behind an import, which - is why `auth-boundary` applies to 26 entry points rather than 427. + imported from another module is ever opened. `auth-boundary` applies to 23 entry points: it gates + on sensitivity first, which is 26 routes, and the one-hop limit accounts for the other 3, which + hand their work to an imported helper and are reported as unverified rather than unguarded. - **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route can be reported as recording nothing while it records plenty. +- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot + say whether a clause does anything besides rethrow, so `error-classification` withholds credit + rather than granting it. Crediting it would reopen the free-points path a single `logger.error` + line wide. - **Only the first object-literal argument is read** for identifier fields, and only its property names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither does a second object. diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 868acf284d6..eaf0bd5ded1 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -68,12 +68,14 @@ export function renderTerminal(report: MapReport): string { const { applicable, naming } = report.contextGap; if (applicable > 0) { - const collapsed = report.entries.filter(contextOnly).length; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; lines.push(""); lines.push( `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + - (collapsed > 0 - ? ` ${collapsed} appear${collapsed === 1 ? "s" : ""} only here, not in the list below.` + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the list below.` : "") ); } diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 8f7352f6a46..cdf2957d38a 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -2,8 +2,16 @@ * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` * does not match a newline, so a suppression on one line cannot pick up a reason from the next. * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. + * + * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so + * the directive did too, and one on the last line of a file switched a check off for everything + * above it. The honest options were to scope it to a line or to rename it, and scoping is not + * available: a `CheckResult` carries no line number, and neither does an `EntryPoint`, so there is + * nothing to match a line against. Scoping it would mean inventing a proximity rule that silently + * drops legitimate suppressions. So the name now says what it does. Real line scoping needs + * positions on the findings, which is scanner work. */ -const PATTERN = /obs-map-disable-next-line\s+([a-z-]+)\s+--\s+(.+)/; +const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** * The comment part of a line, or null if there is none. diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index 2dec21ca9a8..f97aa2269ed 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,5 +1,9 @@ +import { existsSync, rmSync } from "node:fs"; +import { resolve } from "node:path"; import { main, type Io } from "../src/cli.js"; +const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); + const capture = () => { const out: string[] = []; const err: string[] = []; @@ -51,10 +55,17 @@ describe("map ", () => { }); describe("map", () => { + // The flag is the only thing standing between a test run and a file written into the repo root, + // so the test has to check the file, not just the exit code. it("renders the whole report without writing when asked not to", () => { + const existedBefore = existsSync(REPORT_FILE); + if (existedBefore) rmSync(REPORT_FILE); + const r = run("--no-write"); + expect(r.code).toBe(0); expect(r.out).toContain("COVERAGE"); expect(r.out).toContain("FIX FIRST"); + expect(existsSync(REPORT_FILE)).toBe(false); }); }); diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index bb80ca1bb6c..d287e19b259 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -32,15 +32,14 @@ describe("renderTerminal", () => { it("surfaces suppressions so laundering is visible rather than silent", () => { const suppressed = scanFile( "api.v1.d.ts", - `// obs-map-disable-next-line error-classification -- deliberate, see ticket + `// obs-map-disable error-classification -- deliberate, see ticket import { prisma } from "~/db.server"; export async function loader() { try { return await prisma.thing.findMany(); } catch (e) { return null; } }` )!; const out = renderTerminal(buildReport([suppressed], [])); - expect(out).toMatch(/suppress/i); - expect(out).toContain("1"); + expect(out).toMatch(/SUPPRESSED\s+1 check across 1 entry point/); }); it("does not mention suppressions when there are none", () => { @@ -118,7 +117,11 @@ describe("renderTerminal", () => { buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) ); - const fixFirst = out.slice(out.indexOf("FIX FIRST"), out.indexOf("already solid")); + // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for + // "already solid" and the assertions were quietly running against the whole tail. + const listEnd = out.indexOf("no findings:"); + expect(listEnd).toBeGreaterThan(-1); + const fixFirst = out.slice(out.indexOf("FIX FIRST"), listEnd); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); @@ -238,6 +241,22 @@ describe("collapsing the house-style finding", () => { expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); }); + // NEW-3. 18 of the collapsed entries are sensitive, including /admin/impersonate and the envvars + // routes, so the line has to say a reader should go and look at them. + it("says how many of the collapsed entries are sensitive", () => { + const sensitiveAndSilent = scanFile( + "api.v1.auth.jwt.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + }` + )!; + const out = renderTerminal(buildReport([sensitiveAndSilent], [])); + expect(out).toMatch(/1 appears? only here[^\n]*1 of them sensitive/i); + }); + it("says how many entries the collapse took out of the list", () => { const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); expect(out).toMatch(/1 appears? only here/i); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index e677d95f3f1..37b8eafec47 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -54,7 +54,7 @@ describe("scoreEntry", () => { }); it("counts a suppressed check as not-applicable", () => { - const suppressed = `// obs-map-disable-next-line error-classification -- health probe + const suppressed = `// obs-map-disable error-classification -- health probe ${RAW}`; const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); const ec = scored.checks.find((c) => c.id === "error-classification")!; @@ -65,12 +65,16 @@ ${RAW}`; // error-classification would fail here; auth-boundary is not-applicable (not sensitive). // Suppressing the only applicable scored check must not be indistinguishable from an entry // point nothing applies to: it is still reported, just not scored on that axis. - const suppressed = `// obs-map-disable-next-line error-classification -- health probe + const suppressed = `// obs-map-disable error-classification -- health probe ${BUSY_AND_FAILING}`; const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( "not-applicable" ); + // The point of the test, which it did not previously assert: request-context still applies, so + // the entry is still measured and still counted in the mean. + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.measured).toBe(true); }); // I1. `score = passed / applicable` meant removing a failing check from the denominator raised @@ -88,7 +92,7 @@ export async function action({ request }) { const suppressed = scoreEntry( scanFile( "api.v1.auth.tokens.ts", - `// obs-map-disable-next-line error-classification -- deliberate, see ticket + `// obs-map-disable error-classification -- deliberate, see ticket ${source}` )! ); @@ -104,8 +108,8 @@ ${source}` const suppressed = scoreEntry( scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe -// obs-map-disable-next-line request-context -- nothing to name here + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here ${BUSY_AND_FAILING}` )! ); @@ -116,8 +120,8 @@ ${BUSY_AND_FAILING}` const suppressed = scoreEntry( scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe -// obs-map-disable-next-line request-context -- nothing to name here + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here ${BUSY_AND_FAILING}` )! ); @@ -167,7 +171,7 @@ describe("buildReport", () => { [ scanFile( "api.v1.b.ts", - `// obs-map-disable-next-line error-classification -- health probe + `// obs-map-disable error-classification -- health probe ${BUSY_AND_FAILING}` )!, scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 0bb6fcc5f9b..8a01a5953c1 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -3,26 +3,26 @@ import { suppressedChecks } from "../src/suppression.js"; describe("suppressedChecks", () => { it("reads a suppression with its reason", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification -- liveness probe, deliberately silent + `// obs-map-disable error-classification -- liveness probe, deliberately silent export async function loader() { return { ok: true }; }` ); expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); }); it("ignores a suppression with no reason", () => { - const m = suppressedChecks(`// obs-map-disable-next-line error-classification`); + const m = suppressedChecks(`// obs-map-disable error-classification`); expect(m.size).toBe(0); }); it("ignores a suppression whose reason is only whitespace", () => { - const m = suppressedChecks(`// obs-map-disable-next-line error-classification -- `); + const m = suppressedChecks(`// obs-map-disable error-classification -- `); expect(m.size).toBe(0); }); it("reads several suppressions in one file", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification -- liveness probe - // obs-map-disable-next-line request-context -- no identifiers exist here + `// obs-map-disable error-classification -- liveness probe + // obs-map-disable request-context -- no identifiers exist here export async function loader() { return { ok: true }; }` ); expect(m.size).toBe(2); @@ -36,7 +36,7 @@ describe("suppressedChecks", () => { it("does not carry a reason across lines", () => { const m = suppressedChecks( - `// obs-map-disable-next-line error-classification + `// obs-map-disable error-classification // some other comment -- with a dash export async function loader() { return 1; }` ); @@ -47,7 +47,7 @@ describe("suppressedChecks", () => { // merely quotes it, in a test fixture or an error message, silently suppressed a real check. it("ignores the directive inside a string literal", () => { const m = suppressedChecks( - `const example = "obs-map-disable-next-line error-classification -- not a real suppression"; + `const example = "obs-map-disable error-classification -- not a real suppression"; export async function loader() { return 1; }` ); expect(m.size).toBe(0); @@ -55,7 +55,7 @@ describe("suppressedChecks", () => { it("reads the directive from a block comment", () => { const m = suppressedChecks( - `/* obs-map-disable-next-line auth-boundary -- public by design, see ADR 12 */ + `/* obs-map-disable auth-boundary -- public by design, see ADR 12 */ export async function loader() { return 1; }` ); expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); @@ -64,7 +64,7 @@ describe("suppressedChecks", () => { it("reads the directive from a jsdoc line", () => { const m = suppressedChecks( `/** - * obs-map-disable-next-line request-context -- nothing tenant-scoped here + * obs-map-disable request-context -- nothing tenant-scoped here */ export async function loader() { return 1; }` ); @@ -73,9 +73,20 @@ describe("suppressedChecks", () => { it("ignores code that happens to follow a comment on the same line", () => { const m = suppressedChecks( - `const x = 1; // obs-map-disable-next-line error-classification -- fine + `const x = 1; // obs-map-disable error-classification -- fine export async function loader() { return x; }` ); expect(m.get("error-classification")).toBe("fine"); }); + + // The directive was called `-next-line` while applying to the whole entry point, so a comment on + // the last line of a file switched a check off for everything above it. Renamed rather than + // scoped, because a finding has no line number to scope it to. The old spelling is not honoured. + it("does not honour the old -next-line spelling", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- stale directive + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); }); From 2cb36aa90e8c65e15b861daf0d482146e0d31861 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:08:48 +0100 Subject: [PATCH 030/117] chore(observability-map): remove dead scanner surface EntryPoint.calleeTexts, catchRethrows, catchBranches, catchesNarrowly and LogCall.hasObjectArgument were written by the scanner and read by nothing: errorClassification reads the per-clause CatchEvidence fields, sensitivity and triviality read calleeNames, not calleeTexts. Removed the fields, their derivations, and the tests that asserted them directly; reworded doc comments that referenced them. --- .../observability-map/src/scan.ts | 28 ++- .../observability-map/src/types.ts | 44 +---- .../observability-map/test/checks.test.ts | 7 +- .../observability-map/test/scan.test.ts | 168 ++++-------------- 4 files changed, 56 insertions(+), 191 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 6301f8ed6f8..0f10138e627 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -69,9 +69,10 @@ function calleeName(expr: ts.Expression): string | null { } /** - * Callee as recorded in `calleeTexts`: the whole path, `prisma.organization.findFirst` rather than - * `findFirst`. Null when the path runs through something with no name of its own, e.g. - * `new PromptService().createOverride`, where the caller falls back to the bare name. + * The whole callee path of a call, `prisma.organization.findFirst` rather than `findFirst`. Used to + * match a call against `LOGGER_CALLEE` and `PARSE_CALLEE`. Null when the path runs through something + * with no name of its own, e.g. `new PromptService().createOverride`, where the caller falls back to + * the bare name. */ function calleeText(expr: ts.Expression): string | null { const target = unwrap(expr); @@ -92,7 +93,7 @@ function calleeText(expr: ts.Expression): string | null { const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; /** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ -function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields: string[] } { +function objectArgumentFields(call: ts.CallExpression): string[] { for (const arg of call.arguments) { const target = unwrap(arg); if (!ts.isObjectLiteralExpression(target)) continue; @@ -101,9 +102,9 @@ function objectArgumentFields(call: ts.CallExpression): { found: boolean; fields const name = propertyName(property); if (name) fields.push(name); } - return { found: true, fields }; + return fields; } - return { found: false, fields: [] }; + return []; } /** @@ -128,8 +129,8 @@ const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$| const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); /** - * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here - * because constructors are absent from `calleeTexts`. + * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here as a + * `ts.isNewExpression`, because the call-callee scan that builds `calleeNames` never sees it. */ function guardsParse(tryBlock: ts.Block): boolean { let found = false; @@ -518,7 +519,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let hasTryCatch = false; const catches: CatchEvidence[] = []; const calleeNames: string[] = []; - const calleeTexts: string[] = []; const logCalls: LogCall[] = []; const localFunctions = collectLocalFunctions(sf); @@ -558,14 +558,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (cn) { const text = calleeText(node.expression) ?? cn; calleeNames.push(cn); - calleeTexts.push(text); if (LOGGER_CALLEE.test(text)) { - const argument = objectArgumentFields(node); logCalls.push({ callee: text, - hasObjectArgument: argument.found, - fields: argument.fields, + fields: objectArgumentFields(node), inCatch, }); } @@ -599,13 +596,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { actionInitializerCallee: target.actionInitializerCallee, importedNames, calleeNames, - calleeTexts, hasTryCatch, catches, - // Kept as aggregates of `catches` so the checks can migrate one at a time. - catchRethrows: catches.some((c) => c.rethrows), - catchBranches: catches.some((c) => c.branches), - catchesNarrowly: catches.length > 0 && catches.every((c) => c.narrow), logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 4a9eec56b69..ff26c7174fe 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -17,17 +17,19 @@ export type CatchEvidence = { /** The clause contains a `throw`. */ rethrows: boolean; /** - * The clause picks what to do from what it caught: an `if`, a `switch`, or a conditional that is - * the whole `return`/`throw`. An `instanceof` used only to word a message, - * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count: every error - * still leaves by the same path. + * The clause picks what to do from what it caught: an `if` or `switch` whose condition references + * the caught error binding, or a conditional that is the whole `return`/`throw`. `if (retries > 0)` + * does not count, and a bindingless `catch { ... }` cannot count at all. An `instanceof` used only + * to word a message, `json({ error: e instanceof Error ? e.message : String(e) })`, does not + * count either: every error still leaves by the same path. */ branches: boolean; /** * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here - * because constructors never appear in `calleeTexts`; other constructors do not count, or every - * `new SomePresenter()` in a try would excuse its catch. + * because a `new` expression is not a call, so the call-callee scan that feeds this check never + * sees them; other constructors do not count, or every `new SomePresenter()` in a try would excuse + * its catch. */ guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ @@ -38,9 +40,7 @@ export type CatchEvidence = { export type LogCall = { /** Full callee path, e.g. `logger.error`. */ callee: string; - /** Whether an object literal was passed as an argument. */ - hasObjectArgument: boolean; - /** Property names on that object literal, e.g. `["environmentId", "error"]`. */ + /** Property names on the first object-literal argument, e.g. `["environmentId", "error"]`. */ fields: string[]; /** Whether the call sits inside a catch clause, i.e. on the failure path. */ inCatch: boolean; @@ -58,12 +58,6 @@ export type EntryPoint = { importedNames: string[]; /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ calleeNames: string[]; - /** - * The same calls as `calleeNames`, same order and same length, but as the whole callee path: - * `prisma.organization.findFirst` where `calleeNames` has `findFirst`. A path that runs through - * something unnameable (`new PromptService().createOverride`) falls back to the bare name. - */ - calleeTexts: string[]; /** * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and @@ -72,26 +66,6 @@ export type EntryPoint = { hasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; - /** - * Whether any catch clause in those bodies contains a `throw`. A catch that rethrows has decided - * the error is not its to answer, which is a different act from swallowing it. Aggregate of - * `catches`, kept so existing consumers keep working. - */ - catchRethrows: boolean; - /** - * Whether any catch clause in those bodies branches on the error: an `if`, a `switch`, or an - * `instanceof`. With `catchRethrows` both false while `hasTryCatch` is true, every catch in the - * entry point takes one path out regardless of what was thrown. - */ - catchBranches: boolean; - /** - * Whether every catch in those bodies guards a specific operation rather than the handler: the - * entry point has at least one catch clause, and no try block with a catch holds more than two - * statements. The `try { body = await request.json() } catch { 400 }` idiom, which takes one path - * out and is still deliberate. False when any catch wraps the bulk of a body, and false when - * there is no catch clause at all. - */ - catchesNarrowly: boolean; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index cbc0c73a00d..eff2617e755 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -565,9 +565,10 @@ describe("request-context", () => { expect(r.status).toBe("fail"); }); - // Was a known false positive: `new URL()` is a constructor, so the parse was invisible while the - // evidence came from `calleeTexts`. `CatchEvidence.guardsParse` covers constructors, so the guard - // is legible now and the route is no longer judged as though it kept its failures. + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible to the + // call-callee scan the evidence used to come from. `CatchEvidence.guardsParse` covers + // constructors, so the guard is legible now and the route is no longer judged as though it kept + // its failures. it("fails a route whose only catch guards a constructor parse", () => { const r = run( "request-context", diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 6625adf9343..7f764e98981 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -571,7 +571,7 @@ describe("scanDirectory", () => { }); describe("scanFile: catch clause evidence", () => { - it("sets catchRethrows when a catch rethrows", () => { + it("sets rethrows on the clause when a catch rethrows", () => { const ep = scanFile( "rethrow.ts", ` @@ -586,8 +586,8 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); }); it("leaves both flags false when the catch only returns", () => { @@ -604,11 +604,11 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); }); - it("sets catchBranches for an `if` on the error", () => { + it("sets branches for an `if` on the error", () => { const ep = scanFile( "branch-if.ts", ` @@ -624,11 +624,11 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); - expect(ep!.catchRethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); }); - it("sets catchBranches for an instanceof conditional that is the whole returned expression", () => { + it("sets branches for an instanceof conditional that is the whole returned expression", () => { const ep = scanFile( "branch-instanceof.ts", ` @@ -641,10 +641,10 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); - it("sets catchBranches for a switch in the catch", () => { + it("sets branches for a switch on the error", () => { const ep = scanFile( "branch-switch.ts", ` @@ -662,7 +662,7 @@ describe("scanFile: catch clause evidence", () => { } ` ); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); it("ignores a catch that lives in the React component", () => { @@ -683,8 +683,7 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); + expect(ep!.catches).toEqual([]); }); it("reads a catch inside a same-file helper the body delegates to", () => { @@ -705,11 +704,11 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(true); }); - it("leaves both flags false for a try with no catch", () => { + it("leaves catches empty for a try with no catch", () => { const ep = scanFile( "finally-only.ts", ` @@ -723,77 +722,7 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - }); -}); - -describe("scanFile: callee texts", () => { - it("keeps the full callee expression alongside the bare name", () => { - const ep = scanFile( - "api.v1.things.ts", - ` - export async function loader({ request }) { - const org = await prisma.organization.findFirst({ where: { id: 1 } }); - logger.error("nope", { organizationId: org.id }); - return json(org); - } - ` - ); - expect(ep!.calleeNames).toContain("findFirst"); - expect(ep!.calleeTexts).toContain("prisma.organization.findFirst"); - expect(ep!.calleeTexts).toContain("logger.error"); - expect(ep!.calleeTexts).toContain("json"); - // Index-aligned with calleeNames, so a consumer can read either. - expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); - }); - - it("does not leak calls made in the React component", () => { - const ep = scanFile( - "route.tsx", - ` - export async function loader() { - return json(await prisma.run.findMany()); - } - export default function Page() { - useFancyHook(); - analytics.track("viewed"); - return null; - } - ` - ); - expect(ep!.calleeTexts).toContain("prisma.run.findMany"); - expect(ep!.calleeTexts).not.toContain("analytics.track"); - expect(ep!.calleeTexts).not.toContain("useFancyHook"); - }); - - it("records callee texts from a same-file helper the body delegates to", () => { - const ep = scanFile( - "delegating.ts", - ` - async function load(id) { - return prisma.project.findUnique({ where: { id } }); - } - export async function loader({ params }) { - return json(await load(params.id)); - } - ` - ); - expect(ep!.calleeTexts).toContain("prisma.project.findUnique"); - }); - - it("falls back to the bare name for a callee it cannot render as a path", () => { - const ep = scanFile( - "new-expression.ts", - ` - export async function action({ request }) { - return json(await new PromptService().createOverride(request)); - } - ` - ); - expect(ep!.calleeNames).toContain("createOverride"); - expect(ep!.calleeTexts).toContain("createOverride"); - expect(ep!.calleeTexts).toHaveLength(ep!.calleeNames.length); + expect(ep!.catches).toEqual([]); }); }); @@ -815,7 +744,6 @@ describe("scanFile: log calls", () => { expect(ep!.logCalls).toHaveLength(1); expect(ep!.logCalls[0]).toEqual({ callee: "logger.error", - hasObjectArgument: true, fields: ["environmentId", "error"], inCatch: true, }); @@ -831,9 +759,7 @@ describe("scanFile: log calls", () => { } ` ); - expect(ep!.logCalls).toEqual([ - { callee: "log.info", hasObjectArgument: false, fields: [], inCatch: false }, - ]); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); }); it("ignores a non-logger call and a log call in the React component", () => { @@ -872,9 +798,9 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("does not flag a catch wrapping the whole handler", () => { @@ -896,10 +822,10 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(false); }); - it("does not flag a body that has both a narrow catch and a broad one", () => { + it("keeps a narrow catch and a broad one distinct", () => { const ep = scanFile( "mixed.ts", ` @@ -921,7 +847,8 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(true); + expect(ep!.catches[1]!.narrow).toBe(false); }); it("allows a guarded operation with its own local binding", () => { @@ -940,7 +867,7 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("does not flag a try of three statements", () => { @@ -958,16 +885,16 @@ describe("scanFile: narrow catches", () => { } ` ); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches[0]!.narrow).toBe(false); }); - it("is false when there is no try at all", () => { + it("is empty when there is no try at all", () => { const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); - it("is false for a try with a finally and no catch", () => { + it("is empty for a try with a finally and no catch", () => { const ep = scanFile( "finally-only.ts", ` @@ -981,7 +908,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); it("reads a narrow catch inside a same-file helper the body delegates to", () => { @@ -1001,7 +928,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchesNarrowly).toBe(true); + expect(ep!.catches[0]!.narrow).toBe(true); }); it("ignores a narrow catch that lives in the React component", () => { @@ -1022,7 +949,7 @@ describe("scanFile: narrow catches", () => { ` ); expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); + expect(ep!.catches).toEqual([]); }); }); @@ -1062,8 +989,6 @@ describe("scanFile: per-catch evidence", () => { guardsParse: false, tryStatementCount: 4, }); - // The aggregate still collapses, which is why the per-catch list exists. - expect(ep!.catchesNarrowly).toBe(false); }); it("leaves catches empty for a try/finally with no catch clause", () => { @@ -1087,9 +1012,6 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches).toEqual([]); // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catchRethrows).toBe(false); - expect(ep!.catchBranches).toBe(false); - expect(ep!.catchesNarrowly).toBe(false); }); it("sees a URL constructor as a guarded parse", () => { @@ -1193,9 +1115,6 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches).toHaveLength(2); expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); - // Aggregates stay as they are: any clause sets them. - expect(ep!.catchRethrows).toBe(true); - expect(ep!.catchBranches).toBe(true); }); it("includes a catch from a same-file helper and excludes one from the React component", () => { @@ -1224,26 +1143,6 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.guardsParse).toBe(true); - expect(ep!.catchRethrows).toBe(false); - }); - - it("keeps the aggregates derivable from the per-catch list", () => { - const ep = scanFile( - "aggregate.ts", - ` - export async function loader({ request }) { - try { - return json(await request.json()); - } catch (e) { - if (e instanceof SyntaxError) return json({}, { status: 400 }); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.catchRethrows).toBe(ep!.catches.some((c) => c.rethrows)); - expect(ep!.catchBranches).toBe(ep!.catches.some((c) => c.branches)); - expect(ep!.catchesNarrowly).toBe(ep!.catches.length > 0 && ep!.catches.every((c) => c.narrow)); }); }); @@ -1353,7 +1252,6 @@ describe("scanFile: branches ignores the error-stringifying ternary", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.branches).toBe(false); - expect(ep!.catchBranches).toBe(false); }); it("does not count an instanceof used to build a logged message", () => { From 751af3feb85e91bc4687ab5afe86a600cd4a3a18 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:20:07 +0100 Subject: [PATCH 031/117] fix(observability-map): stop crediting an if or switch that never looks at the error catchClauseEvidence counted any if/switch inside a catch as branching, whether or not its condition read the caught error. catch (_e) { if (organization) ... } and catch (error: any) { if (wantsJson) ... } both counted, crediting a route that always takes the same path regardless of what was thrown. The if condition (or switch discriminant) must now reference the caught error binding; a bindingless catch { ... } cannot qualify at all. Ternary handling is unchanged. Measured on the real tree: 5 of 242 catch clauses lose branches, 4 routes flip from pass to fail on error-classification, and the global score moves from 18 to 17. Hand-read all 4: each was a false positive, branching on an unrelated local rather than the error. --- .../observability-map/src/scan.ts | 34 ++++++- .../observability-map/test/scan.test.ts | 98 +++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 0f10138e627..9cb19407c01 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -153,11 +153,28 @@ function guardsParse(tryBlock: ts.Block): boolean { return found; } +/** Whether some node in the tree rooted at `node` matches `predicate`. */ +function someNode(node: ts.Node, predicate: (n: ts.Node) => boolean): boolean { + if (predicate(node)) return true; + return ts.forEachChild(node, (child) => someNode(child, predicate)) === true; +} + function containsInstanceOf(node: ts.Node): boolean { - if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword) { - return true; - } - return ts.forEachChild(node, containsInstanceOf) === true; + return someNode( + node, + (n) => ts.isBinaryExpression(n) && n.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ); +} + +/** Whether `node` reads the given catch binding anywhere, e.g. `e` in `e instanceof X` or `error.code`. */ +function referencesBinding(node: ts.Node, bindingName: string): boolean { + return someNode(node, (n) => ts.isIdentifier(n) && n.text === bindingName); +} + +/** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ +function catchBindingName(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; } /** @@ -177,10 +194,17 @@ function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; + const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { if (ts.isThrowStatement(node)) rethrows = true; - if (ts.isIfStatement(node) || ts.isSwitchStatement(node)) branches = true; + if ( + bindingName !== null && + ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || + (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) + ) { + branches = true; + } if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; ts.forEachChild(node, visit); }; diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 7f764e98981..db1ff081952 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1289,3 +1289,101 @@ describe("scanFile: branches ignores the error-stringifying ternary", () => { expect(ep!.catches[0]!.branches).toBe(true); }); }); + +describe("scanFile: branches requires the if/switch condition to examine the error", () => { + it("does not set branches for an `if` on an unrelated variable", () => { + const ep = scanFile( + "retry-count.ts", + ` + export async function action({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch (e) { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` whose condition references the caught error", () => { + const ep = scanFile( + "branch-if-e.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof ApiError) return json({}, { status: e.status }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a `switch` on a property of the caught error", () => { + const ep = scanFile( + "branch-switch-error-code.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + switch (error.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not set branches for a `switch` on an unrelated discriminant", () => { + const ep = scanFile( + "branch-switch-unrelated.ts", + ` + export async function action({ request }) { + const mode = "strict"; + try { + return json(await load(request)); + } catch (error) { + switch (mode) { + case "strict": + return json({}, { status: 400 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("cannot set branches for a bindingless catch, even with an `if` inside it", () => { + const ep = scanFile( + "bindingless-if.ts", + ` + export async function loader({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); From 11c590890ede45f21d0394a896f10d61a53d38b7 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:20:20 +0100 Subject: [PATCH 032/117] docs(observability-map): explain triviality's exclusion and the gaming boundary Two gaps: nothing said what makes a route trivial or why exclusion beats a vacuous 100, and nothing said the tool verifies presence, not meaning. Documents isTrivial's rule and adds a gaming-boundary section, measured on the real tree: a synthetic environmentId on every in-catch log call, with no other change, moves the global score from 17 to 27. Also updates the current-score figures for the branches fix in the prior commit. --- internal-packages/observability-map/README.md | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 457c22ff390..703404aef9f 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 18 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 17 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -20,7 +20,7 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. -## What 18 means +## What 17 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does @@ -34,12 +34,12 @@ error handling. Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: -- **Removing error handling must not raise the score.** Deleting every catch clause takes 18 to 8, - and deleting the logs as well takes it to 2. +- **Removing error handling must not raise the score.** Deleting every catch clause drops it to 8, + and deleting the logs as well drops it to 2. - **Adding error handling that does nothing must not raise the score.** Wrapping every body in - `try { ... } catch (e) { throw e }` leaves it at 18, with no entry moving in either direction. - That mutation used to be worth 27 points across the tree, because a rethrow-only clause counted - as a pass while no catch at all was not-applicable, and the two are observationally identical. + `try { ... } catch (e) { throw e }` leaves the score unchanged. That mutation used to be worth 27 + points across the tree, because a rethrow-only clause counted as a pass while no catch at all was + not-applicable, and the two are observationally identical. If you change this package, check both directions still hold. @@ -89,6 +89,16 @@ unmeasured entries into the mean at 100, would let the tool look better the less header prints both counts (`412 measured, 15 unmeasured`) so the denominator is never hidden, and a family with nothing measured renders as `not measured` rather than as a full green bar. +15 of those 427 routes are unmeasured because `isTrivial` (`src/triviality.ts`) rules them out before +any check runs. Trivial means a body of three statements or fewer, three or fewer calls, no +try/catch, no builder wrapping it, and nothing in the calls or the source naming a datastore or a +service (`prisma`, `logger`, `fetch`, `redis`, and the like). Parse the params, build a path, +redirect: nothing there for a check to find evidence in either way. Exclusion is a denominator exit, +not a credit: a trivial route's `score` is the same placeholder 100 that an unmeasured entry always +carries, and it is left out of every mean for the same reason. Scoring it a pass instead would say a +route earned a clean result by never doing anything a check could look at, which is the same vacuous +100 the header already refuses to average in. + ## When a check declines to judge The rule every applicability decision follows: **would this evidence necessarily be visible in the @@ -127,6 +137,21 @@ check holds the number still rather than improving it. What you buy is removal f with a reason on the record. The report prints how many suppressions are in force so the practice stays visible. +## The gaming boundary + +`request-context` checks that a failure-path log names a tenant field. It does not check that the +value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it +up to the wrong variable or a constant, would move the score exactly as far as one that wired it up +correctly. Measured on the real tree: adding a synthetic `environmentId` field to every in-catch log +call, with no other change, takes the global score from 17 to 27. + +That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads +syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a +hardcoded string with the right key. What a reviewer owns is whether the value behind the field is +real, the same way a Lighthouse accessibility score checks that an `alt` attribute exists and not +that its text describes the image. The number tells you where to look. It does not tell you what +you will find there. + ## Known limits Read these before trusting a specific verdict. From b8971419ff4cb37778e32031c1110a062a80b079 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:27:12 +0100 Subject: [PATCH 033/117] docs(observability-map): refresh the collapse figures and say audit-trail collapses too --- internal-packages/observability-map/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 703404aef9f..0c0fe56d2d8 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -67,12 +67,14 @@ rather than celebrating. `audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route would bury the route-specific findings under the same sentence repeated hundreds of times, so both -are reported as a figure: the `AUDIT` and `CONTEXT` lines. 333 entry points fail nothing except -`request-context` and appear only in that figure, which leaves 67 in the fix list. An entry that -fails `request-context` *and* something else keeps both findings and stays in the list, so -`/account/tokens` still shows the whole picture. - -18 of those 333 are sensitive, including `/admin/impersonate`, the API-key regeneration route and +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 329 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 71 in the fix list. An entry that +fails `request-context` *and* another scored check keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. `audit-trail` does not count as "another" for this +purpose: it is already a headline, so a route failing only `request-context` and `audit-trail` +collapses too (13 do today, all sensitive, which is most of the 18 below). + +18 of those 329 are sensitive, including `/admin/impersonate`, the API-key regeneration route and four envvars routes, so the `CONTEXT` line says how many. Read them out of `observability-map.json`, where every entry keeps its full check results, rather than assuming the list is the whole story. From dd3c2f6a567d7c909033605da6671f9e6e21d025 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 00:51:30 +0100 Subject: [PATCH 034/117] fix(observability-map): remove the last dead scanner field and pin the callee-path fallback CatchEvidence.narrow was the same species as the fields removed earlier: written by scan.ts, asserted in tests, read by nothing. error-classification computes narrowness itself inline through isParseGuard and never reads clause.narrow. Removed the field, its NARROW_TRY_STATEMENTS derivation, and every test asserting it; kept tryStatementCount, which isParseGuard reads. Also pins the two callee-path shapes the earlier callee-texts test deletion left unexercised: a three-level property chain (prisma.organization.findFirst arriving as findFirst in calleeNames) and the constructor-fallback shape (new PromptService().createOverride), plus a logger call in the same fixture to confirm calleeText's normal chain-building still feeds LogCall.callee. --- .../src/checks/errorClassification.ts | 8 +- .../observability-map/src/scan.ts | 8 - .../observability-map/src/types.ts | 2 - .../observability-map/test/scan.test.ts | 230 ++++-------------- 4 files changed, 56 insertions(+), 192 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 2c48f40dded..a78559eefba 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -54,10 +54,10 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * `request-context` still reads that log and asks whether it names a tenant, so the reporting is * unrewarded here rather than unmeasured. * - * `narrow` is not a way to qualify either. A one-statement try around `await service.call(run)` is - * narrow and is still a swallow: reading all eleven entry points that limb would clear said six - * were real, including a silent run cancellation and two credential paths that report a database - * failure to the browser as a 400 with an internal message in it. + * A narrow guard is not a way to qualify either. A one-statement try around `await + * service.call(run)` is narrow and is still a swallow: reading all eleven entry points that limb + * would clear said six were real, including a silent run cancellation and two credential paths + * that report a database failure to the browser as a 400 with an internal message in it. */ function decides(clause: CatchEvidence, ep: EntryPoint): boolean { return clause.branches || isParseGuard(clause, ep); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 9cb19407c01..3454d0ad4c5 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -107,13 +107,6 @@ function objectArgumentFields(call: ts.CallExpression): string[] { return []; } -/** - * How much a try block may guard and still count as narrow. Two, so that the guarded operation can - * bind its result (`const stripped = ...; new RegExp(stripped);`) but a third statement means the - * try has started to cover the handler rather than one operation. - */ -const NARROW_TRY_STATEMENTS = 2; - /** * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is @@ -563,7 +556,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ - narrow: tryStatementCount <= NARROW_TRY_STATEMENTS, rethrows: clause.rethrows, branches: clause.branches, guardsParse: guardsParse(node.tryBlock), diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index ff26c7174fe..a1d3be0b2db 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -12,8 +12,6 @@ export type CheckResult = { * legible instead of collapsing into one boolean. */ export type CatchEvidence = { - /** The guarded try block holds at most two statements: one operation, not the handler. */ - narrow: boolean; /** The clause contains a `throw`. */ rethrows: boolean; /** diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index db1ff081952..d9af8df904c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -460,6 +460,31 @@ describe("scanFile: callee resolution", () => { expect(ep!.hasLoader).toBe(true); expect(ep!.loaderInitializerCallee).toBeNull(); }); + + it("resolves a multi-level call and falls back to the bare name past an unnameable one", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + try { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + return json(await new PromptService().createOverride(org)); + } catch (e) { + logger.error("nope", { error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + // A three-level property chain still lands on its bare method name. + expect(ep!.calleeNames).toContain("findFirst"); + // A chain through a `new` expression has no name of its own, so this also falls back to the + // bare name rather than losing the call. + expect(ep!.calleeNames).toContain("createOverride"); + // The full path still builds where nothing unnameable sits in it, which is what `LogCall.callee` + // depends on. + expect(ep!.logCalls[0]!.callee).toBe("logger.error"); + }); }); describe("scanFile: parse failures", () => { @@ -724,62 +749,7 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.hasTryCatch).toBe(true); expect(ep!.catches).toEqual([]); }); -}); -describe("scanFile: log calls", () => { - it("records the fields of a log call's object argument", () => { - const ep = scanFile( - "logging.ts", - ` - export async function loader({ request }) { - try { - return json(await load(request)); - } catch (e) { - logger.error("load failed", { environmentId: env.id, error: e }); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.logCalls).toHaveLength(1); - expect(ep!.logCalls[0]).toEqual({ - callee: "logger.error", - fields: ["environmentId", "error"], - inCatch: true, - }); - }); - - it("records a log call with no object argument, outside a catch", () => { - const ep = scanFile( - "logging-plain.ts", - ` - export async function loader() { - log.info("starting"); - return json({}); - } - ` - ); - expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); - }); - - it("ignores a non-logger call and a log call in the React component", () => { - const ep = scanFile( - "route.tsx", - ` - export async function loader() { - return json(await load()); - } - export default function Page() { - logger.debug("rendered", { runId: 1 }); - return null; - } - ` - ); - expect(ep!.logCalls).toEqual([]); - }); -}); - -describe("scanFile: narrow catches", () => { it("flags a try that guards a single request.json()", () => { const ep = scanFile( "admin.api.v1.platform-notifications.ts", @@ -800,92 +770,6 @@ describe("scanFile: narrow catches", () => { expect(ep!.hasTryCatch).toBe(true); expect(ep!.catches[0]!.rethrows).toBe(false); expect(ep!.catches[0]!.branches).toBe(false); - expect(ep!.catches[0]!.narrow).toBe(true); - }); - - it("does not flag a catch wrapping the whole handler", () => { - const ep = scanFile( - "otel.v1.logs.ts", - ` - export async function action({ request }) { - try { - const exporter = await otlpExporter; - const contentType = request.headers.get("content-type") ?? ""; - const body = await request.json(); - await exporter.export(body); - return json({ ok: true }); - } catch (e) { - logger.error(e); - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(false); - }); - - it("keeps a narrow catch and a broad one distinct", () => { - const ep = scanFile( - "mixed.ts", - ` - export async function action({ request }) { - let body; - try { - body = await request.json(); - } catch { - return json({}, { status: 400 }); - } - try { - const run = await find(body.id); - const updated = await update(run); - await notify(updated); - return json(updated); - } catch (e) { - return json({}, { status: 500 }); - } - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(true); - expect(ep!.catches[1]!.narrow).toBe(false); - }); - - it("allows a guarded operation with its own local binding", () => { - const ep = scanFile( - "regex.ts", - ` - export async function action({ request }) { - const pattern = await patternFrom(request); - try { - const stripped = pattern.startsWith("(?i)") ? pattern.slice(4) : pattern; - new RegExp(stripped); - } catch { - return json({ error: "Invalid regex" }, { status: 400 }); - } - return json({ ok: true }); - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(true); - }); - - it("does not flag a try of three statements", () => { - const ep = scanFile( - "three.ts", - ` - export async function loader({ request }) { - try { - const raw = await request.json(); - const parsed = Schema.parse(raw); - return json(parsed); - } catch { - return json({}, { status: 400 }); - } - } - ` - ); - expect(ep!.catches[0]!.narrow).toBe(false); }); it("is empty when there is no try at all", () => { @@ -893,68 +777,63 @@ describe("scanFile: narrow catches", () => { expect(ep!.hasTryCatch).toBe(false); expect(ep!.catches).toEqual([]); }); +}); - it("is empty for a try with a finally and no catch", () => { +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { const ep = scanFile( - "finally-only.ts", + "logging.ts", ` - export async function loader() { + export async function loader({ request }) { try { - return json(await load()); - } finally { - release(); + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); } } ` ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches).toEqual([]); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + fields: ["environmentId", "error"], + inCatch: true, + }); }); - it("reads a narrow catch inside a same-file helper the body delegates to", () => { + it("records a log call with no object argument, outside a catch", () => { const ep = scanFile( - "helper-narrow.ts", + "logging-plain.ts", ` - function parseTags(payload) { - try { - return JSON.parse(payload); - } catch { - return null; - } - } - export async function loader({ params }) { - return json(parseTags(params.payload)); + export async function loader() { + log.info("starting"); + return json({}); } ` ); - expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(true); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); }); - it("ignores a narrow catch that lives in the React component", () => { + it("ignores a non-logger call and a log call in the React component", () => { const ep = scanFile( "route.tsx", ` export async function loader() { - return json({}); + return json(await load()); } export default function Page() { - try { - JSON.parse(raw); - } catch { - return null; - } + logger.debug("rendered", { runId: 1 }); return null; } ` ); - expect(ep!.hasTryCatch).toBe(false); - expect(ep!.catches).toEqual([]); + expect(ep!.logCalls).toEqual([]); }); }); describe("scanFile: per-catch evidence", () => { - it("records one entry per catch clause, keeping a narrow guard distinct from a broad catch", () => { + it("records one entry per catch clause, keeping a parse guard distinct from a broad catch", () => { const ep = scanFile( "two-catches.ts", ` @@ -978,14 +857,12 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(2); expect(ep!.catches[0]).toEqual({ - narrow: true, rethrows: false, branches: false, guardsParse: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ - narrow: false, guardsParse: false, tryStatementCount: 4, }); @@ -1035,10 +912,9 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]!.guardsParse).toBe(true); - expect(ep!.catches[0]!.narrow).toBe(true); }); - it("sees a parse in a try that grew past the narrowness threshold", () => { + it("sees a parse in a try that outgrows a single statement", () => { const ep = scanFile( "admin.api.v1.orgs.$organizationId.stream-basin.ts", ` @@ -1060,7 +936,6 @@ describe("scanFile: per-catch evidence", () => { ` ); expect(ep!.catches).toHaveLength(1); - expect(ep!.catches[0]!.narrow).toBe(false); expect(ep!.catches[0]!.guardsParse).toBe(true); expect(ep!.catches[0]!.tryStatementCount).toBe(6); }); @@ -1083,7 +958,6 @@ describe("scanFile: per-catch evidence", () => { ); expect(ep!.catches).toHaveLength(1); expect(ep!.catches[0]).toEqual({ - narrow: false, rethrows: false, branches: false, guardsParse: false, From 1df57d4d7c1840b3db685349c546ccf0cd0ceed2 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:24:46 +0100 Subject: [PATCH 035/117] feat(observability-map): add --routes flag and a PR-comment renderer for CI --- internal-packages/observability-map/README.md | 7 + .../observability-map/src/cli.ts | 22 ++- .../observability-map/src/report/prComment.ts | 179 ++++++++++++++++++ .../src/report/prCommentCli.ts | 39 ++++ .../observability-map/src/report/terminal.ts | 61 +++--- .../observability-map/test/cli.test.ts | 33 +++- .../observability-map/test/prComment.test.ts | 119 ++++++++++++ .../test/prCommentCli.test.ts | 57 ++++++ 8 files changed, 489 insertions(+), 28 deletions(-) create mode 100644 internal-packages/observability-map/src/report/prComment.ts create mode 100644 internal-packages/observability-map/src/report/prCommentCli.ts create mode 100644 internal-packages/observability-map/test/prComment.test.ts create mode 100644 internal-packages/observability-map/test/prCommentCli.test.ts diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 0c0fe56d2d8..76638b7e7ab 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -20,6 +20,13 @@ suppresses. The single-route mode takes either the route path the report prints the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an ambiguous prefix warns and names the alternatives rather than silently picking one. +## CI + +A PR that touches `apps/webapp/app/routes` or this package gets a sticky comment scanning head +against the PR's merge base, with the score, what changed, and the current fix list. It is +report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a +later phase decides to add one. See `.github/workflows/observability-map.yml`. + ## What 17 means It is the mean score of the 412 entry points that had at least one applicable check, where an diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 4be0ff59193..cb9c9b6017a 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -1,4 +1,4 @@ -import { existsSync, writeFileSync } from "node:fs"; +import { existsSync, statSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { EntryPoint } from "./types.js"; @@ -63,7 +63,25 @@ export function main(argv: string[], io: Io = processIo): number { const target = args.find((a) => !a.startsWith("--")); const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); - const routesDir = resolve(repoRoot, DEFAULT_ROUTES); + const routesFlag = args.find((a) => a.startsWith("--routes=")); + + let routesDir: string; + if (routesFlag) { + routesDir = resolve(process.cwd(), routesFlag.slice("--routes=".length)); + let isDir = false; + try { + isDir = statSync(routesDir).isDirectory(); + } catch { + isDir = false; + } + if (!isDir) { + io.err(`--routes: not a readable directory: ${routesDir}\n`); + return 1; + } + } else { + routesDir = resolve(repoRoot, DEFAULT_ROUTES); + } + const { entryPoints, parseFailures } = scanDirectory(routesDir); if (target) { diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts new file mode 100644 index 00000000000..ed4480959a5 --- /dev/null +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -0,0 +1,179 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; + +/** First line of every comment this job posts, so the upsert step can find its own comment again. */ +export const MARKER = ""; + +const MAX_CHANGED_ROWS = 15; + +const failingIds = (e: ScoredEntry) => e.checks.filter((c) => c.status === "fail").map((c) => c.id); + +function scoreLine(head: MapReport, base: MapReport | null): string { + const headline = + head.global === null + ? `not measured over ${head.measured} measured of ${head.entries.length} entry points` + : `**${head.global}/100** over ${head.measured} measured of ${head.entries.length} entry points`; + + if (!base) return headline; + if (base.global === null || head.global === null) return `${headline} (base not measured)`; + + const diff = head.global - base.global; + const comparison = + diff === 0 + ? `(base ${base.global}, no change)` + : diff > 0 + ? `(base ${base.global}, up ${diff})` + : `(base ${base.global}, down ${-diff})`; + return `${headline} ${comparison}`; +} + +type ChangedRow = { + routePath: string; + sensitive: boolean; + baseScore: number | "new"; + headScore: number; + nowFailing: string[]; + /** + * How much the entry got worse, used to sort the table. A new entry has no base score to + * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the + * same as an existing one that dropped 40 points, which is the ordering "what needs fixing + * first" implies. + */ + drop: number; +}; + +function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; removed: number } { + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + + const rows: ChangedRow[] = []; + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) { + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: "new", + headScore: h.score, + nowFailing: failingIds(h), + drop: 100 - h.score, + }); + continue; + } + if (b.score === h.score) continue; + const baseFailing = new Set(failingIds(b)); + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: b.score, + headScore: h.score, + nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + drop: b.score - h.score, + }); + } + + rows.sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + b.drop - a.drop || + a.routePath.localeCompare(b.routePath) + ); + + const removed = base.entries.filter((e) => !headFiles.has(e.fileName)).length; + return { rows, removed }; +} + +function whatChangedSection(head: MapReport, base: MapReport | null): string[] { + const lines = ["**What this PR changed**"]; + + if (!base) { + lines.push("Base comparison unavailable."); + return lines; + } + + const { rows, removed } = changedRows(head, base); + + if (rows.length === 0 && removed === 0) { + lines.push("No entry point this PR touches changed its score."); + return lines; + } + + if (rows.length > 0) { + lines.push(""); + lines.push("| route | base | head | now failing |"); + lines.push("| --- | --- | --- | --- |"); + for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + lines.push( + `| ${row.routePath} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + ); + } + if (rows.length > MAX_CHANGED_ROWS) { + lines.push(""); + lines.push(`and ${rows.length - MAX_CHANGED_ROWS} more`); + } + } + + if (removed > 0) { + lines.push(""); + lines.push(`${removed} entries removed`); + } + + return lines; +} + +function fixFirstSection(head: MapReport): string[] { + const lines = ["FIX FIRST"]; + const worst = head.entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + `- ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + } + return lines; +} + +/** + * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the + * two by `fileName`, the same identifier `renderJson` carries. + */ +export function renderPrComment(head: MapReport, base: MapReport | null): string { + const lines = [MARKER, "", "## Observability map", "", scoreLine(head, base), ""]; + + lines.push(...whatChangedSection(head, base), ""); + lines.push(...fixFirstSection(head), ""); + + const audit = auditLine(head); + if (audit) lines.push(audit); + const context = contextLine(head); + if (context) lines.push(context); + if (audit || context) lines.push(""); + + lines.push( + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md." + ); + + const headFailures = head.parseFailures.length; + const baseFailures = base?.parseFailures.length ?? 0; + if (headFailures > 0 || baseFailures > 0) { + const parts: string[] = []; + if (headFailures > 0) parts.push(`${headFailures} at head`); + if (baseFailures > 0) parts.push(`${baseFailures} at base`); + lines.push( + `Warning: parse failures (${parts.join(", ")}) are excluded from the score, shrinking the denominator.` + ); + } + + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts new file mode 100644 index 00000000000..0fa1d7786e7 --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -0,0 +1,39 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { MapReport } from "../score.js"; +import { renderPrComment } from "./prComment.js"; + +/** Where output goes. Injectable so tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** `-` or a missing second arg means no base: the CI job falls back to this when the base scan + * itself failed, so the comment still renders rather than the job going red. */ +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const headPath = args[0]; + const basePath = args[1]; + + if (!headPath) { + io.err("usage: prCommentCli.ts [base.json|-]\n"); + return 1; + } + + const head = JSON.parse(readFileSync(headPath, "utf8")) as MapReport; + const base: MapReport | null = + !basePath || basePath === "-" ? null : JSON.parse(readFileSync(basePath, "utf8")); + + io.out(renderPrComment(head, base)); + io.out("\n"); + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not read a file. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index eaf0bd5ded1..6175a93a88c 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -17,7 +17,7 @@ const gauge = (score: number | null) => { * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as * `AUDIT`, below. */ -const scoredFailures = (e: ScoredEntry) => +export const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); /** @@ -27,11 +27,40 @@ const scoredFailures = (e: ScoredEntry) => * An entry that fails something else as well stays in the list with all of its findings, so a * route like `/account/tokens` still shows the request-context gap alongside the rest. */ -const contextOnly = (e: ScoredEntry) => { +export const contextOnly = (e: ScoredEntry) => { const failures = scoredFailures(e); return failures.length === 1 && failures[0]!.id === "request-context"; }; +/** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when + * there is nothing to report, i.e. no sensitive mutation exists. */ +export function auditLine(report: MapReport): string | null { + const { sensitiveMutations, withAudit } = report.auditGap; + if (sensitiveMutations === 0) return null; + // The closing sentence is a claim about the codebase, so it is only made when the figure in + // front of it supports it. It was printed unconditionally, including next to a non-zero count. + const gap = + withAudit === 0 + ? " No audit helper exists in the webapp." + : ` ${sensitiveMutations - withAudit} without one.`; + return `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}`; +} + +/** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ +export function contextLine(report: MapReport): string | null { + const { applicable, naming } = report.contextGap; + if (applicable === 0) return null; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; + return ( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the list below.` + : "") + ); +} + export function renderTerminal(report: MapReport): string { const lines: string[] = []; @@ -52,32 +81,16 @@ export function renderTerminal(report: MapReport): string { }/${report.sensitiveCohort.n} entry points` ); - const { sensitiveMutations, withAudit } = report.auditGap; - if (sensitiveMutations > 0) { + const audit = auditLine(report); + if (audit) { lines.push(""); - // The closing sentence is a claim about the codebase, so it is only made when the figure in - // front of it supports it. It was printed unconditionally, including next to a non-zero count. - const gap = - withAudit === 0 - ? " No audit helper exists in the webapp." - : ` ${sensitiveMutations - withAudit} without one.`; - lines.push( - `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}` - ); + lines.push(audit); } - const { applicable, naming } = report.contextGap; - if (applicable > 0) { - const collapsed = report.entries.filter(contextOnly); - const sensitive = collapsed.filter((e) => e.sensitive).length; + const context = contextLine(report); + if (context) { lines.push(""); - lines.push( - `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + - (collapsed.length > 0 - ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + - `${sensitive} of them sensitive, in the JSON rather than the list below.` - : "") - ); + lines.push(context); } if (report.suppressions.checks > 0) { diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index f97aa2269ed..2fd28955d85 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,5 +1,6 @@ -import { existsSync, rmSync } from "node:fs"; -import { resolve } from "node:path"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { main, type Io } from "../src/cli.js"; const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); @@ -69,3 +70,31 @@ describe("map", () => { expect(existsSync(REPORT_FILE)).toBe(false); }); }); + +describe("map --routes=", () => { + it("scans the directory it names instead of the repo's routes tree", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-routes-")); + writeFileSync( + join(dir, "resources.only.ts"), + `export const loader = () => new Response("ok");` + ); + + const r = run("--routes=" + dir, "--json", "--no-write"); + + expect(r.code).toBe(0); + const parsed = JSON.parse(r.out); + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0].fileName).toBe("resources.only.ts"); + + rmSync(dir, { recursive: true }); + }); + + it("exits 1 with a message when the directory does not exist", () => { + const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); + const r = run("--routes=" + dir); + + expect(r.code).toBe(1); + expect(r.err).toContain("not a readable directory"); + expect(r.out).toBe(""); + }); +}); diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts new file mode 100644 index 00000000000..9a115fc79d7 --- /dev/null +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -0,0 +1,119 @@ +import { renderPrComment } from "../src/report/prComment.js"; +import { buildReport } from "../src/score.js"; +import { scanFile } from "../src/scan.js"; + +const cleanSource = ` + import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + +const brokenSource = ` + import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +describe("renderPrComment", () => { + it("puts the upsert marker on the first line, always", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + expect(renderPrComment(head, null).split("\n")[0]).toBe(""); + expect(renderPrComment(head, head).split("\n")[0]).toBe(""); + }); + + it("says the comparison is unavailable when base is null", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Base comparison unavailable."); + expect(out).not.toContain("no change"); + }); + + it("reports a score drop and the newly failing checks", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\(base \d+, down \d+\)/); + expect(out).toContain("| /api/v1/auth/tokens |"); + // request-context and error-classification regress; auth-boundary is not applicable here + // (no sensitivity signal on this route), so it must not show up as newly failing. + expect(out).toMatch(/\| \/api\/v1\/auth\/tokens \| \d+ \| \d+ \|[^|]*error-classification/); + }); + + it("reports a score improvement the other way round", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + expect(out).toMatch(/\(base \d+, up \d+\)/); + }); + + it("says nothing changed when every score matches", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + expect(out).toContain("No entry point this PR touches changed its score."); + expect(out).toContain("(base 100, no change)"); + }); + + it("shows a new entry with its base column as 'new' and lists its failing checks", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", brokenSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); + }); + + it("reports a removed entry as a count line, not a row", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.gone.ts", brokenSource)!], + [] + ); + const out = renderPrComment(head, base); + + expect(out).toContain("1 entries removed"); + expect(out).not.toContain("/api/v1/gone"); + }); + + it("caps the changed-entries table at 15 rows and says how many more", () => { + const headEntries = []; + const baseEntries = []; + for (let i = 0; i < 20; i++) { + headEntries.push(scanFile(`api.v1.route${i}.ts`, brokenSource)!); + baseEntries.push(scanFile(`api.v1.route${i}.ts`, cleanSource)!); + } + const head = buildReport(headEntries, []); + const base = buildReport(baseEntries, []); + const out = renderPrComment(head, base); + + const rows = out.split("\n").filter((l) => l.startsWith("| /api/v1/route")); + expect(rows).toHaveLength(15); + expect(out).toContain("and 5 more"); + }); + + it("warns about parse failures in either report, since they shrink the denominator", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-head.ts"]); + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-base.ts"]); + const out = renderPrComment(head, base); + expect(out).toMatch(/Warning: parse failures \(1 at head, 1 at base\)/); + }); + + it("does not warn about parse failures when there are none", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + expect(renderPrComment(head, null)).not.toContain("Warning: parse failures"); + }); + + it("footer names the report-only rule and the readme", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Report only, nothing here gates the merge."); + expect(out).toContain("internal-packages/observability-map/README.md"); + }); +}); diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts new file mode 100644 index 00000000000..8e687bbf936 --- /dev/null +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -0,0 +1,57 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "../src/report/prCommentCli.js"; +import { buildReport } from "../src/score.js"; +import { renderJson } from "../src/report/json.js"; +import { scanFile } from "../src/scan.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "prCommentCli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("prCommentCli", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); + const headPath = join(dir, "head.json"); + const basePath = join(dir, "base.json"); + + const source = `export const loader = () => new Response("ok");`; + const report = buildReport([scanFile("resources.a.ts", source)!], []); + writeFileSync(headPath, renderJson(report)); + writeFileSync(basePath, renderJson(report)); + + afterAll(() => rmSync(dir, { recursive: true })); + + it("renders the markdown comment for head and base file arguments", () => { + const r = run(headPath, basePath); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + }); + + it("treats '-' as no base", () => { + const r = run(headPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("treats a missing second argument as no base", () => { + const r = run(headPath); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("exits 1 with a usage message when head.json is missing", () => { + const r = run(); + expect(r.code).toBe(1); + expect(r.err).toContain("usage:"); + }); +}); From 902c0136c20debae0d70fa97584815a64aba0fcc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:24:56 +0100 Subject: [PATCH 036/117] ci(observability-map): scan head vs merge base and upsert a sticky PR comment --- .github/workflows/observability-map.yml | 78 +++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/observability-map.yml diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml new file mode 100644 index 00000000000..460d0d5f884 --- /dev/null +++ b/.github/workflows/observability-map.yml @@ -0,0 +1,78 @@ +name: 🗺️ Observability Map + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "apps/webapp/app/routes/**" + - "internal-packages/observability-map/**" + +concurrency: + group: observability-map-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + report: + runs-on: warp-ubuntu-latest-x64-4x + # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. + if: github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🔎 Scan head + run: | + pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write > /tmp/head.json + + - name: 🔎 Scan base with the head's scanner + run: | + if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ + && pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ + --routes=/tmp/base-tree/apps/webapp/app/routes > /tmp/base.json; then + : + else + echo "-" > /tmp/base.json || true + echo "base scan failed or the worktree could not be added; falling back to no base" >&2 + fi + + - name: 📝 Render comment + run: | + if [ "$(cat /tmp/base.json)" = "-" ]; then + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md + else + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md + fi + + - name: 💬 Upsert PR comment + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + existing=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty') + if [ -n "$existing" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md + fi From df0333eb581ab6192f4f6371aae0562129699419 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:36:57 +0100 Subject: [PATCH 037/117] fix(observability-map): stop audit-trail leaking into the PR-comment fix list and never fail the CI job on a render or upsert error --- .github/workflows/observability-map.yml | 7 ++ .../observability-map/src/report/prComment.ts | 7 +- .../src/report/prCommentCli.ts | 28 +++++++- .../observability-map/test/prComment.test.ts | 68 +++++++++++++++++++ .../test/prCommentCli.test.ts | 26 +++++++ 5 files changed, 132 insertions(+), 4 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 460d0d5f884..f63409adb96 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -56,7 +56,11 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi + # continue-on-error: this job must never block a PR. A malformed head.json or a rendering + # bug would otherwise turn the job red the same way a base-scan failure would not, since that + # step already falls back to "-" instead of failing. - name: 📝 Render comment + continue-on-error: true run: | if [ "$(cat /tmp/base.json)" = "-" ]; then pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md @@ -64,7 +68,10 @@ jobs: pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md fi + # continue-on-error for the same reason: a transient gh api failure (rate limit, network) + # must not fail the job either. Worst case, the PR gets no comment this run. - name: 💬 Upsert PR comment + continue-on-error: true env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index ed4480959a5..8e857b8c6d9 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,4 +1,5 @@ import type { MapReport, ScoredEntry } from "../score.js"; +import { SCORED_CHECK_IDS } from "../checks/index.js"; import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; /** First line of every comment this job posts, so the upsert step can find its own comment again. */ @@ -6,7 +7,11 @@ export const MARKER = ""; const MAX_CHANGED_ROWS = 15; -const failingIds = (e: ScoredEntry) => e.checks.filter((c) => c.status === "fail").map((c) => c.id); +// Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost +// every sensitive mutation today, so listing it per route would nag with something unfixable +// instead of surfacing the route-specific gaps this column exists for. +const failingIds = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail").map((c) => c.id); function scoreLine(head: MapReport, base: MapReport | null): string { const headline = diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index 0fa1d7786e7..0283eb96591 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -12,6 +12,22 @@ const processIo: Io = { err: (s) => process.stderr.write(s), }; +/** Reads and parses one report file, raising a message naming the file rather than letting an + * unreadable path or malformed JSON surface as a stack trace. */ +function readReport(path: string, label: string): MapReport { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new Error(`cannot read ${label}: ${path}`); + } + try { + return JSON.parse(raw) as MapReport; + } catch { + throw new Error(`${label} is not valid JSON: ${path}`); + } +} + /** `-` or a missing second arg means no base: the CI job falls back to this when the base scan * itself failed, so the comment still renders rather than the job going red. */ export function main(argv: string[], io: Io = processIo): number { @@ -24,9 +40,15 @@ export function main(argv: string[], io: Io = processIo): number { return 1; } - const head = JSON.parse(readFileSync(headPath, "utf8")) as MapReport; - const base: MapReport | null = - !basePath || basePath === "-" ? null : JSON.parse(readFileSync(basePath, "utf8")); + let head: MapReport; + let base: MapReport | null; + try { + head = readReport(headPath, "head report"); + base = !basePath || basePath === "-" ? null : readReport(basePath, "base report"); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } io.out(renderPrComment(head, base)); io.out("\n"); diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 9a115fc79d7..7f651d57dc2 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -70,6 +70,74 @@ describe("renderPrComment", () => { expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); }); + // Mirrors the guard report.test.ts has for the terminal renderer: audit-trail fails almost + // every sensitive mutation today (no audit helper exists), so it is a headline figure, not a + // per-route nag. A regression here previously let it leak into the "now failing" column. + it("does not list audit-trail among a new sensitive entry's failing checks", () => { + const sensitiveMutation = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + const head = buildReport([sensitiveMutation], []); + const base = buildReport([], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/envvars |"))!; + expect(row).toBeDefined(); + expect(row).toContain("new"); + expect(row).not.toContain("audit-trail"); + expect(row).toMatch(/error-classification|auth-boundary|request-context/); + }); + + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { + const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; + const sensitiveSmallDropHead = scanFile( + "api.v1.auth.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { error }); throw error; } + }` + )!; + + const notSensitiveLargeDropBase = scanFile( + "resources.busy.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const notSensitiveLargeDropHead = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + const head = buildReport([sensitiveSmallDropHead, notSensitiveLargeDropHead], []); + const base = buildReport([sensitiveSmallDropBase, notSensitiveLargeDropBase], []); + const out = renderPrComment(head, base); + + const sensitiveIndex = out.indexOf("/api/v1/auth/tokens"); + const notSensitiveIndex = out.indexOf("/resources/busy"); + expect(sensitiveIndex).toBeGreaterThan(-1); + expect(notSensitiveIndex).toBeGreaterThan(-1); + expect(sensitiveIndex).toBeLessThan(notSensitiveIndex); + }); + it("reports a removed entry as a count line, not a row", () => { const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); const base = buildReport( diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts index 8e687bbf936..b464725ea77 100644 --- a/internal-packages/observability-map/test/prCommentCli.test.ts +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -54,4 +54,30 @@ describe("prCommentCli", () => { expect(r.code).toBe(1); expect(r.err).toContain("usage:"); }); + + it("exits 1 with a one-line message, not a stack trace, when head.json does not exist", () => { + const r = run(join(dir, "does-not-exist.json")); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("cannot read head report"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json is malformed", () => { + const malformedPath = join(dir, "malformed.json"); + writeFileSync(malformedPath, "{ not json"); + const r = run(malformedPath); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("head report is not valid JSON"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message when base.json is malformed", () => { + const malformedBasePath = join(dir, "malformed-base.json"); + writeFileSync(malformedBasePath, "not json at all"); + const r = run(headPath, malformedBasePath); + expect(r.code).toBe(1); + expect(r.err).toContain("base report is not valid JSON"); + }); }); From 960a568cdbaa7ddc17561ee1fd5a852cd49aaa91 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 14:44:33 +0100 Subject: [PATCH 038/117] fix(observability-map): make the CONTEXT line true in both renderers --- internal-packages/observability-map/src/report/terminal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 6175a93a88c..05cc7d6a248 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -56,7 +56,7 @@ export function contextLine(report: MapReport): string | null { `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + (collapsed.length > 0 ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + - `${sensitive} of them sensitive, in the JSON rather than the list below.` + `${sensitive} of them sensitive, in the JSON rather than the fix list.` : "") ); } From 08f6be57eb3fb4fe3aa64023c372a3c5362a3252 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:07:03 +0100 Subject: [PATCH 039/117] fix(observability-map): derive measured from pre-suppression applicability score.ts computed measured from visible (post-suppression) applicability, so suppressing an entry's only applicable check set measured: false and dropped it, still failing, out of the global mean, every family mean and the sensitive cohort. Tree-wide, prepending a zero-behaviour suppression comment moved the global from 17 to 33 and measured from 412 to 176. measured now reads scored (pre-suppression) applicability instead, so a fully-suppressed entry stays in the denominator at its capped score. unmeasured still means no applicable check at all. The per-entry Math.min cap was already correct and is untouched. --- .../observability-map/src/score.ts | 12 +++- .../test/integration.test.ts | 23 +++++++- .../observability-map/test/score.test.ts | 55 ++++++++++++++++++- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 1203f730158..a39a1ab7c82 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -11,7 +11,9 @@ export type ScoredEntry = { sensitive: boolean; checks: CheckResult[]; /** - * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable. + * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable + * before suppression. A fully-suppressed entry stays measured, at its capped score, so a + * suppression cannot buy removal from every mean by way of removal from this one. * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that * default cannot inflate a figure nobody checked. @@ -65,7 +67,11 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; const visible = scored.filter((c) => !suppressed.has(c.id)); - const applicable = visible.filter((c) => c.status !== "not-applicable"); + // Pre-suppression: an entry whose only applicable check gets suppressed still had something to + // measure, and must stay in the denominator at its capped score rather than vanish as if nothing + // ever applied. `unmeasured` is reserved for entries with no applicable check at all, suppression + // or no suppression. + const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); return { fileName: ep.fileName, @@ -79,7 +85,7 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the // suppression comment laundered the finding into a point. What a suppression buys is removal // from the worklist, with a reason on the record. It cannot buy a better number. - measured: applicable.length > 0, + measured: scoredApplicable.length > 0, score: Math.min(ratio(visible), ratio(scored)), }; } diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index f628c5a1dfd..032f960fb5e 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -1,7 +1,8 @@ import { existsSync, readdirSync } from "node:fs"; import { join, resolve } from "node:path"; -import { scanDirectory } from "../src/scan.js"; +import { scanDirectory, scanFile } from "../src/scan.js"; import { buildReport } from "../src/score.js"; +import { SCORED_CHECK_IDS } from "../src/checks/index.js"; const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); @@ -44,4 +45,24 @@ describe("scanning the real webapp routes", () => { expect(report.global).toBeLessThanOrEqual(100); expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); }); + + // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. + // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to + // 176, because every entry whose only applicable checks were suppressed dropped out of the + // mean. Measured must not move: every entry point that had something applicable still does. + it("suppressing every scored check on every real route does not raise the global", () => { + if (!existsSync(ROUTES)) return; + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const before = buildReport(entryPoints, parseFailures); + + const directive = SCORED_CHECK_IDS.map( + (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` + ).join(""); + const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); + const after = buildReport(suppressed, parseFailures); + + expect(after.measured).toBe(before.measured); + expect(after.unmeasured).toBe(before.unmeasured); + expect(after.global).not.toBeGreaterThan(before.global!); + }); }); diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 37b8eafec47..ccbc24b973e 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -116,7 +116,11 @@ ${BUSY_AND_FAILING}` expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); }); - it("does not let an entry whose every scored check is suppressed read as measured", () => { + // A1. `measured` reads pre-suppression applicability. Before the fix it read `visible` + // (post-suppression) applicability, so suppressing an entry's only applicable checks flipped + // `measured` to false and dropped the entry, still failing, out of the global mean, every family + // mean and the sensitive cohort. `unmeasured` stays for entries nothing was ever applicable to. + it("still reads as measured when every applicable scored check is suppressed", () => { const suppressed = scoreEntry( scanFile( "api.v1.b.ts", @@ -125,7 +129,8 @@ ${BUSY_AND_FAILING}` ${BUSY_AND_FAILING}` )! ); - expect(suppressed.measured).toBe(false); + expect(suppressed.measured).toBe(true); + expect(suppressed.score).toBe(0); }); it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { @@ -221,6 +226,52 @@ ${BUSY_AND_FAILING}` }); }); +// A1. `measured` used to read post-suppression (`visible`) applicability, so suppressing an +// entry's only applicable checks removed it from the global mean, every family mean and the +// sensitive cohort, rather than keeping it at its capped score. That is how a suppression with no +// behavioural change moved the global from 17 to 33 tree-wide. +describe("A1: a suppression cannot raise the global", () => { + it("scoring 100 and 0, suppressing every check on the failing entry leaves the global at 50", () => { + const passing = scanFile("api.v1.auth.tokens.ts", CLEAN)!; + const failing = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const before = buildReport([passing, failing], []); + expect(before.global).toBe(50); + + const failingSuppressed = scanFile( + "api.v1.busy.ts", + `// obs-map-disable error-classification -- silence +// obs-map-disable request-context -- silence +${BUSY_AND_FAILING}` + )!; + const after = buildReport([passing, failingSuppressed], []); + + expect(after.global).toBe(50); + expect(after.measured).toBe(2); + }); + + it("suppressing a passing check does not raise the score either", () => { + // error-classification branches on the error (pass); request-context never names a tenant + // (fail). Not sensitive, so auth-boundary sits out. + const source = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof Error) return null; throw e; } +}`; + const plain = scoreEntry(scanFile("api.v1.mixed.ts", source)!); + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + + const suppressed = scoreEntry( + scanFile( + "api.v1.mixed.ts", + `// obs-map-disable error-classification -- silence a pass +${source}` + )! + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); +}); + /** * The invariant, both ways round. The README states one direction, removing error handling must * lower the score, and that alone could not see the free-points path: adding a catch that only From e88f04d01bae7655dabb287942cf07ab634536d4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:11:08 +0100 Subject: [PATCH 040/117] fix(observability-map): read suppression comments from real comment ranges commentPart used line.indexOf("//") and indexOf("/*") against the raw text, so a string or template literal that merely quoted the directive granted a suppression nobody wrote and silenced a real check: "see // obs-map-disable error-classification -- because reasons" and a URL string containing "obs-map-disable" both suppressed for real. Replaced the hand-rolled text scan with the TypeScript scanner's own token stream. A string or template literal is one token there, consumed in a single step, and the scanner never emits comment trivia for what is inside it, so there is no substring left to fool. Line comments, block comments and jsdoc lines are read the same as before; no textual path remains. Also corrects the docstring's claimed failure direction: the old comment said misreading a string cost a suppression that was never written, when the real effect was the opposite, granting one nobody wrote. --- .../observability-map/src/suppression.ts | 55 ++++++++++++------- .../test/suppression.test.ts | 29 ++++++++++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index cdf2957d38a..6012ee09d4e 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,7 +1,8 @@ +import ts from "typescript"; + /** * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` * does not match a newline, so a suppression on one line cannot pick up a reason from the next. - * A trailing block-comment terminator is trimmed off so it does not end up inside the reason. * * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so * the directive did too, and one on the last line of a file switched a check off for everything @@ -14,37 +15,51 @@ const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** - * The comment part of a line, or null if there is none. + * Every physical line of genuine comment content in the source, line comments and block comments + * alike, one entry per line, with the `//`, `/*`, `*​/` and a jsdoc `*` prefix stripped. * - * Line-scoped and comment-only, because the directive is a comment directive. Matching the raw - * source meant a string literal that merely quotes the directive, in a test fixture or an error - * message, silently switched a real check off. Handles line comments, block comments and the - * leading star of a jsdoc block; a line-comment marker inside a string on the same line can still - * be misread, which costs a suppression that was never written rather than hiding one that was. + * Read from the TypeScript scanner's own token stream rather than `indexOf("//")` against the raw + * text. The old text-matching read the directive out of a string or template literal that merely + * quoted it, so `"see // obs-map-disable auth-boundary -- nope"` granted a suppression nobody + * wrote, silencing a real check. The scanner already knows the difference: a string or template + * literal is one token, consumed in a single step, and never yields comment trivia for what is + * inside it, so there is no substring rule left to fool. */ -function commentPart(line: string): string | null { - const slashes = line.indexOf("//"); - if (slashes !== -1) return line.slice(slashes + 2); +function commentLines(source: string): string[] { + const scanner = ts.createScanner( + ts.ScriptTarget.Latest, + /* skipTrivia */ false, + ts.LanguageVariant.Standard, + source + ); + const lines: string[] = []; - const block = line.indexOf("/*"); - if (block !== -1) return line.slice(block + 2).replace(/\*\/\s*$/, ""); + for (let kind = scanner.scan(); kind !== ts.SyntaxKind.EndOfFileToken; kind = scanner.scan()) { + if (kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(scanner.getTokenText().slice(2)); + continue; + } + if (kind !== ts.SyntaxKind.MultiLineCommentTrivia) continue; - const trimmed = line.trimStart(); - if (trimmed.startsWith("*")) return trimmed.slice(1); + const text = scanner.getTokenText(); + const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ + for (const rawLine of body.split("\n")) { + const trimmed = rawLine.trimStart(); + lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); + } + } - return null; + return lines; } /** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ export function suppressedChecks(source: string): Map { const out = new Map(); - for (const line of source.split("\n")) { - const comment = commentPart(line); - if (comment === null) continue; - const match = PATTERN.exec(comment); + for (const line of commentLines(source)) { + const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; - const trimmedReason = reason?.replace(/\*\/\s*$/, "").trim(); + const trimmedReason = reason?.trim(); if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); } return out; diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 8a01a5953c1..3259278a594 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -89,4 +89,33 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a + // string that merely quotes the directive granted a suppression nobody wrote and silenced a real + // check. Reading genuine comment ranges from the TypeScript scanner closes both shapes: the + // scanner consumes a string or template literal as one token and never emits comment trivia for + // what is inside it. + it("does not suppress from a directive quoted inside a string literal", () => { + const m = suppressedChecks( + `const msg = "see // obs-map-disable error-classification -- because reasons"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive quoted inside a string with no real comment marker", () => { + const m = suppressedChecks( + `const u = "https://example.com obs-map-disable auth-boundary -- nope"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a template literal", () => { + const m = suppressedChecks( + "const msg = `see // obs-map-disable error-classification -- template literal`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); }); From b3200c1e7d1e561dca2d4daee94b1defdfc21aef Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:20:17 +0100 Subject: [PATCH 041/117] fix(observability-map): require a genuine read of the catch binding referencesBinding matched any ts.Identifier whose text equalled the catch binding, so a property name in a member expression (fallback.error), an object literal key ({ error: false }), and a name re-declared in a nested scope all counted as reading the error. A clause whose only if tests fallback.error passed error-classification without ever inspecting what it caught. An identifier only counts now when it is a real reference: excluded are the name side of a property access, a property assignment name, and any position inside a scope (function or catch parameter, or a var/let/const/function/ class declared in a block) that re-declares the same name. Ternaries route through the same predicate instead of accepting any instanceof in the condition, so the fix lands once at the root rather than twice. Measured on the real routes tree by re-scanning every file with the old and new logic and diffing catches[].branches directly: zero clauses change and the global stays at 17. The bug is real and reproduces synthetically, it just has no live occurrence in the current tree to hand-read. --- .../observability-map/src/scan.ts | 62 +++++++++++- .../observability-map/test/checks.test.ts | 19 ++++ .../observability-map/test/scan.test.ts | 99 +++++++++++++++++++ 3 files changed, 176 insertions(+), 4 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 3454d0ad4c5..54b58d0aa23 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -159,9 +159,57 @@ function containsInstanceOf(node: ts.Node): boolean { ); } -/** Whether `node` reads the given catch binding anywhere, e.g. `e` in `e instanceof X` or `error.code`. */ +/** Whether `name` is declared by a var/let/const, function or class statement directly in this + * statement list. Not recursive: a nested block's own declarations are handled when the walk + * reaches that block. */ +function declaresInScope(statements: readonly ts.Statement[], name: string): boolean { + for (const statement of statements) { + if (ts.isFunctionDeclaration(statement) && statement.name?.text === name) return true; + if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; + if ( + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some( + (d) => ts.isIdentifier(d.name) && d.name.text === name + ) + ) { + return true; + } + } + return false; +} + +/** + * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` + * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the + * binding's text without reading it: the property side of a member expression (`fallback.error`) + * and an object literal key (`{ error: true }`), both excluded by checking which side of the + * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch + * parameter or as a var/let/const/function/class in a block, refers to that declaration instead, + * so the walk stops at the boundary that re-declares it rather than crediting the outer binding. + */ function referencesBinding(node: ts.Node, bindingName: string): boolean { - return someNode(node, (n) => ts.isIdentifier(n) && n.text === bindingName); + if (ts.isIdentifier(node) && node.text === bindingName) { + const parent = node.parent; + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false; + if (ts.isPropertyAssignment(parent) && parent.name === node) return false; + return true; + } + + if ( + ts.isFunctionLike(node) && + node.parameters.some((p) => ts.isIdentifier(p.name) && p.name.text === bindingName) + ) { + return false; + } + + if (ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + if (decl && ts.isIdentifier(decl.name) && decl.name.text === bindingName) return false; + } + + if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; + + return ts.forEachChild(node, (child) => referencesBinding(child, bindingName)) === true; } /** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ @@ -176,9 +224,15 @@ function catchBindingName(clause: ts.CatchClause): string | null { * `return e instanceof Response ? e : json({}, { status: 500 })` counts and * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. * The second is message formatting: every error leaves by the same path. + * + * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than + * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding + * is not a decision made on the error, and a bindingless catch has nothing here to reference. */ -function selectsAnErrorPath(node: ts.ConditionalExpression): boolean { +function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { + if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; + if (!referencesBinding(node.condition, bindingName)) return false; const parent = node.parent; return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); } @@ -198,7 +252,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc ) { branches = true; } - if (ts.isConditionalExpression(node) && selectsAnErrorPath(node)) branches = true; + if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; ts.forEachChild(node, visit); }; visit(clause.block); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index eff2617e755..e69d53b7f60 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -253,6 +253,25 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a + // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the + // caught binding itself, was credited with classifying an error it never inspected. + it("fails a catch whose only if tests a same-named property, not the caught error", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture: the only try/catch in the file belongs to the component. it("does not judge a route whose try/catch is in the React component", () => { const r = run( diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index d9af8df904c..5fea2292f81 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1261,3 +1261,102 @@ describe("scanFile: branches requires the if/switch condition to examine the err expect(ep!.catches[0]!.branches).toBe(false); }); }); + +// A3. `referencesBinding` matched any identifier with the binding's text, including a property +// name, an object literal key and a name re-declared in a nested scope. So a clause that never +// really inspects the error still counted as deciding on it. +describe("scanFile: branches requires a genuine read of the binding, not a lookalike", () => { + it("does not set branches for an `if` that only reads a same-named property", () => { + const ep = scanFile( + "branch-property-name.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` that only reads a same-named object literal key", () => { + const ep = scanFile( + "branch-object-key.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (buildOptions({ error: false }).ok) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a callback that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-param.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (items.some(function (error) { return error.code === 1; })) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a block that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-block.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if ( + (() => { + const error = 1; + return error > 0; + })() + ) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still sets branches for an `if` that genuinely reads the binding beside a lookalike", () => { + const ep = scanFile( + "branch-genuine-and-lookalike.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error || error instanceof NotFound) return json({}, { status: 404 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From 46c5ad011a2a94dc4f26eed65d7fc5ca09587897 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:25:06 +0100 Subject: [PATCH 042/117] fix(observability-map): stop a dead throw counting as a rethrow rethrows was set by any ThrowStatement anywhere in a catch clause, reachable or not, so appending throw e; after a return flipped a swallowing catch from a fail (swallows, no decision made) to not-applicable (read as inert, rethrow-only), with zero behavioural change. Worth about 3 points tree-wide on the real routes tree, confirmed by simulating the mutation against every catch clause's real evidence. catchClauseEvidence now walks each Block, CaseClause and DefaultClause's own statement list only up to the first statement that unconditionally exits (return, throw, continue or break); anything after that is dead and is never visited. This is not full flow analysis, an if/else where both branches return is not itself recognised as an exit, but it covers the mutation this check was measured against. --- .../observability-map/src/scan.ts | 27 +++++++++++++++++ .../observability-map/test/scan.test.ts | 21 +++++++++++++ .../observability-map/test/score.test.ts | 30 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 54b58d0aa23..aeb58f232be 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -237,6 +237,29 @@ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); } +/** A statement that unconditionally leaves the statement list it sits in, so anything after it in + * the same list never runs. */ +function isDefiniteExit(statement: ts.Statement): boolean { + return ( + ts.isReturnStatement(statement) || + ts.isThrowStatement(statement) || + ts.isContinueStatement(statement) || + ts.isBreakStatement(statement) + ); +} + +/** + * `statements` up to and including the first one that definitely exits. Not full flow analysis: + * an `if`/`else` where both branches return is not itself recognised as an exit, only a bare + * `return`, `throw`, `continue` or `break` is. That is enough to make a `throw e;` appended after + * a `return` dead code rather than evidence the clause rethrows, which is the one shape a mutation + * testing this check actually produced. + */ +function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { + const index = statements.findIndex(isDefiniteExit); + return index === -1 ? statements : statements.slice(0, index + 1); +} + /** What a catch clause does with the error, beyond the fact that it caught one. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; @@ -244,6 +267,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { + for (const statement of reachableStatements(node.statements)) visit(statement); + return; + } if (ts.isThrowStatement(node)) rethrows = true; if ( bindingName !== null && diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 5fea2292f81..ae67d598bc7 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -615,6 +615,27 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); + // A4. `rethrows` used to be set by any `ThrowStatement` in the clause, reachable or not, so a + // `throw e;` appended after a `return` flipped a swallowing catch from rethrows: false to true + // with no behavioural change, which read as inert instead of a swallow. + it("does not set rethrows for a throw that is dead code after a return", () => { + const ep = scanFile( + "dead-throw.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + throw e; + } + } + ` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index ccbc24b973e..05386622993 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -314,6 +314,36 @@ export async function loader({ params }) { expect(after.score).toBeLessThanOrEqual(before.score); }); + // A4. rethrows used to be set by any ThrowStatement anywhere in the clause, dead code included, + // so appending `throw e;` after a `return` in a swallowing catch flipped error-classification + // from fail to not-applicable: a mutation with no behavioural effect that hid the swallow. + it("does not improve the verdict when a dead throw follows a return in a swallowing catch", () => { + const swallows = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + } +}`; + const swallowsWithDeadThrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + throw e; + } +}`; + + const before = scoreEntry(scanFile("api.v1.x.ts", swallows)!); + const after = scoreEntry(scanFile("api.v1.x.ts", swallowsWithDeadThrow)!); + + expect(before.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { const before = buildReport( [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], From 433ebec31209f19532ddb3f5274a80c04edb82cc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:33:27 +0100 Subject: [PATCH 043/117] fix(observability-map): require request-context to name a tenant, at a level worth reading request-context filtered failure-path logs on inCatch only, never on level, and IDENTIFIER_FIELD matched any suffix shape (Id/Ids/Slug/Ref/Param/ Identifier) regardless of the root word. So logger.debug("cache miss", { id: 1 }) passed inside a catch, and 10 of the 21 passes named a resource (batchId, notificationId, chatId, spanParam, runFriendlyId, taskIdentifier, runId, waitpointId, sessionId) rather than a tenant. TENANT_FIELD now anchors on the root word: environment, organization, project or user, in the full and abbreviated forms the webapp actually writes (environmentId/envId, organizationId/organizationSlug/orgId, projectId/ projectParam, userId), derived by grepping the routes tree rather than invented. QUALIFYING_LEVELS restricts the log itself to error, warn and fatal; debug and trace are routinely dropped before an incident is read, and info is excluded too since it is not reserved for failure reporting. Measured on the real tree: CONTEXT moves from 21 of 412 to 11 of 412, all ten lost to the field-name fix (every flip was already at error or warn level, so the level restriction changes nothing on the current tree; it still closes the exact logger.debug shape in the brief). Global moves from 17 to 16. The 11 that still pass all name a genuine environment, organization or project field. --- .../src/checks/requestContext.ts | 42 +++++-- .../observability-map/src/report/terminal.ts | 2 +- .../observability-map/test/checks.test.ts | 110 ++++++++++++++++++ .../observability-map/test/report.test.ts | 2 +- 4 files changed, 143 insertions(+), 13 deletions(-) diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index 041a9515257..f2d37aa600e 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -4,12 +4,30 @@ import { isTrivial } from "../triviality.js"; const ID = "request-context"; /** - * A field name that says which tenant, request or resource the failure belongs to. Matched on the - * suffix, in the camelCase the webapp writes: `environmentId`, `organizationSlug`, `runFriendlyId`, - * `projectParam`, `taskIdentifier`. Lowercase `id` inside a word is not a suffix, so `valid` and - * `paid` do not qualify. + * A field name that plausibly names a TENANT: environment, organization, project or user, the four + * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, + * in the full and abbreviated camelCase the webapp actually writes for each: + * `environmentId`/`envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, + * `userId`. A bare `id`, and a resource id that happens to share the same `Id`/`Param` suffix, + * `batchId`, `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not + * qualify: those name a resource the failure touched, not who it happened to. */ -const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identifier)$/; +const TENANT_FIELD = + /^(environment|env|organization|org|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$/; + +/** + * error, warn and fatal are levels an incident is actually read at; debug and trace are routinely + * dropped or sampled out before anyone looks, so a tenant field logged only at that level is not + * really recorded on the failure path. info is deliberately excluded too: it is not reserved for + * failure reporting, so a route can log an info line inside a catch that says nothing about the + * catch actually handling anything, and crediting it would launder the same gap `debug` closes. + */ +const QUALIFYING_LEVELS = new Set(["error", "warn", "fatal"]); + +/** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ +function logLevel(callee: string): string { + return callee.slice(callee.lastIndexOf(".") + 1); +} /** * Whether a failure here can be traced to whoever it happened to. @@ -20,7 +38,8 @@ const IDENTIFIER_FIELD = /^(id|ids|slug|ref)$|[a-z](Id|Ids|Slug|Ref|Param|Identi * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route * and which request failed, and never whose environment it was, unless the route passed the field - * itself. 21 of 427 entry points do. + * itself. 11 of 427 entry points do, naming an environment, organization, project or user; the + * other 10 that used to be counted here only named a resource the failure touched, not a tenant. * * Every non-trivial entry point is judged, and a route that never catches fails like any other. * That is the whole point rather than an oversight: its failures go to the global handler, which @@ -39,18 +58,19 @@ export const requestContext = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const failurePathLogs = ep.logCalls.filter((l) => l.inCatch); - const named = failurePathLogs.find((l) => l.fields.some((f) => IDENTIFIER_FIELD.test(f))); + const failurePathLogs = ep.logCalls.filter( + (l) => l.inCatch && QUALIFYING_LEVELS.has(logLevel(l.callee)) + ); + const named = failurePathLogs.find((l) => l.fields.some((f) => TENANT_FIELD.test(f))); if (named) { - const fields = named.fields.filter((f) => IDENTIFIER_FIELD.test(f)); + const fields = named.fields.filter((f) => TENANT_FIELD.test(f)); return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; } if (failurePathLogs.length > 0) { return { id: ID, status: "fail", - detail: - "logs its failure without naming an environment, project, organization, run or user", + detail: "logs its failure without naming an environment, organization, project or user", }; } return { diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 05cc7d6a248..e85c629bb2d 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -21,7 +21,7 @@ export const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); /** - * An entry whose only finding is `request-context`. 391 of 412 entry points fail that check, so + * An entry whose only finding is `request-context`. 401 of 412 entry points fail that check, so * listing each one turns the fix list into a single house-style finding repeated, which is the * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. * An entry that fails something else as well stays in the list with all of its findings, so a diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index e69d53b7f60..f4377e930ab 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -709,6 +709,116 @@ describe("request-context", () => { ); expect(r.status).toBe("not-applicable"); }); + + // A5. IDENTIFIER_FIELD matched any suffix, so a resource id (a run, a batch, a notification, a + // chat, a span) that shares the same `Id`/`Param` shape as a tenant field passed, and a bare `id` + // passed too. TENANT_FIELD requires the root word itself to be environment, organization, project + // or user. + it("does not pass a failure log that only names a resource, not a tenant", () => { + const r = run( + "request-context", + "api.v3.batches.$batchId.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("batch lookup failed", { batchId: params.batchId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a failure log that only names a bare id", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("cache miss", { id: 1 }); + logger.error("lookup failed", { id: 1, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes on the abbreviated envId/orgId forms the webapp also writes", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, orgId: params.orgId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A5. request-context filtered failure-path logs on inCatch only, not on level, so a debug log + // naming a tenant field passed the check even though debug lines are routinely dropped or + // sampled out before an incident is read. + it("does not pass a debug-level failure log, even one that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass an info-level failure log either", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.info("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes a warn-level failure log that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.warn("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); }); describe("audit-trail", () => { diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index d287e19b259..7beed431da1 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -228,7 +228,7 @@ describe("collapsing the house-style finding", () => { }` )!; - // request-context fails 391 of 412 entry points, so listing each one turns the fix list into a + // request-context fails 401 of 412 entry points, so listing each one turns the fix list into a // single finding repeated. Same reasoning that keeps audit-trail out of the list. it("keeps an entry whose only finding is request-context out of the fix list", () => { const out = renderTerminal(buildReport([namesNobody()], [])); From 2061d999a53b8dbacf8e534050dab0e4a057c53e Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:41:42 +0100 Subject: [PATCH 044/117] fix(observability-map): compare a parse guard against its own body, not the entry point isParseGuard divided tryStatementCount by ep.statementCount, which sums the loader, the action and every one-hop helper together. Adding an unrelated sibling handler or a fat helper to the same file, with zero change to the clause itself, diluted that denominator and relabelled a broad swallow as a narrow parse guard. CatchEvidence now carries enclosingStatementCount, the statement count of the specific loader, action or helper body the clause is directly inside, set once per walkBody call in scan.ts. isParseGuard compares against that instead, and no longer needs the EntryPoint threaded through decides/inert/ swallows just to reach ep.statementCount. Measured on the real tree: two routes flip from pass to fail, both genuine swallows that a same-file sibling or a followed helper had been diluting into an apparent narrow guard. Global stays at 16 (rounds the same either way). A byte-identical swallowing action test proves the verdict no longer depends on what else shares the file. --- .../src/checks/errorClassification.ts | 35 ++++++------ .../observability-map/src/scan.ts | 4 +- .../observability-map/src/types.ts | 8 +++ .../observability-map/test/checks.test.ts | 54 +++++++++++++++++++ .../observability-map/test/scan.test.ts | 2 + 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index a78559eefba..7d5f24f5dc0 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -27,14 +27,19 @@ const BUILDERS = new Set([ /** * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * covers less than half the entry point's statements. Both halves matter. `guardsParse` alone lets - * `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and merely happens to contain a - * `request.json()`, and that is a real swallow. The coverage test is relative to the body rather - * than a second absolute threshold, so it holds for a three-statement route and a fifty-statement - * one alike. + * covers less than half the statements of the body it is actually in. Both halves matter. + * `guardsParse` alone lets `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and + * merely happens to contain a `request.json()`, and that is a real swallow. The coverage test is + * relative to the enclosing body rather than a second absolute threshold, so it holds for a + * three-statement route and a fifty-statement one alike. + * + * Compared against `clause.enclosingStatementCount`, never `ep.statementCount`: the entry point's + * total sums the loader, the action and every one-hop helper together, so an unrelated sibling + * handler or a fat helper in the same file diluted the denominator and relabelled a broad swallow + * as a narrow parse guard, with no change to the clause itself. */ -function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.guardsParse && clause.tryStatementCount * 2 < ep.statementCount; +function isParseGuard(clause: CatchEvidence): boolean { + return clause.guardsParse && clause.tryStatementCount * 2 < clause.enclosingStatementCount; } /** @@ -59,18 +64,18 @@ function isParseGuard(clause: CatchEvidence, ep: EntryPoint): boolean { * would clear said six were real, including a silent run cancellation and two credential paths * that report a database failure to the browser as a 400 with an internal message in it. */ -function decides(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.branches || isParseGuard(clause, ep); +function decides(clause: CatchEvidence): boolean { + return clause.branches || isParseGuard(clause); } /** Passes the error through unchanged, which is the same outcome as not catching it. */ -function inert(clause: CatchEvidence, ep: EntryPoint): boolean { - return clause.rethrows && !decides(clause, ep); +function inert(clause: CatchEvidence): boolean { + return clause.rethrows && !decides(clause); } /** The error stops here and nothing chose what it meant. */ -function swallows(clause: CatchEvidence, ep: EntryPoint): boolean { - return !decides(clause, ep) && !inert(clause, ep); +function swallows(clause: CatchEvidence): boolean { + return !decides(clause) && !inert(clause); } export function usesBuilder(ep: EntryPoint): boolean { @@ -106,7 +111,7 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const swallowed = ep.catches.filter((c) => swallows(c, ep)); + const swallowed = ep.catches.filter(swallows); if (swallowed.length > 0) { const which = ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; @@ -116,7 +121,7 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (!ep.catches.some((c) => decides(c, ep))) { + if (!ep.catches.some(decides)) { return { id: ID, status: "not-applicable", diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index aeb58f232be..4f43e70745d 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -627,7 +627,8 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const helpers: EntryFunction[] = []; const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - statementCount += countFunctionStatements(fn); + const enclosingStatementCount = countFunctionStatements(fn); + statementCount += enclosingStatementCount; if (!fn.body) return; const visit = (node: ts.Node, inCatch: boolean) => { @@ -641,6 +642,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { branches: clause.branches, guardsParse: guardsParse(node.tryBlock), tryStatementCount, + enclosingStatementCount, }); } } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index a1d3be0b2db..bf6e6a79022 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -32,6 +32,14 @@ export type CatchEvidence = { guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; + /** + * Statements in the body the clause actually sits in: the loader, the action, or the one-hop + * helper it delegates to, whichever function this `try` is directly inside. Never the entry + * point's combined total, which sums the loader, the action and every helper together: comparing + * a clause against that whole-entry-point figure let an unrelated sibling handler or a fat helper + * in the same file relabel a broad swallow as a narrow parse guard. + */ + enclosingStatementCount: number; }; /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index f4377e930ab..b958cd7a487 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -253,6 +253,60 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); + // A6. isParseGuard compared the clause against ep.statementCount, the loader and the action and + // every one-hop helper summed together, rather than the statements of the body the clause is + // actually in. So an unrelated sibling handler or a fat helper in the same file diluted the + // denominator and relabelled the same broad swallow as a narrow parse guard. Byte-identical + // action, verdict must not move. + it("gives the same verdict to a byte-identical swallow whether or not an unrelated sibling and helper share the file", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const withUnrelatedSiblingAndHelper = `${action} + function unrelatedHelper() { + let total = 0; + total += 1; + total += 2; + total += 3; + total += 4; + total += 5; + total += 6; + total += 7; + total += 8; + total += 9; + total += 10; + total += 11; + return total; + } + export async function loader() { + const helperTotal = unrelatedHelper(); + return new Response(String(helperTotal)); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const withSiblingAndHelper = run( + "error-classification", + "otel.v1.logs.ts", + withUnrelatedSiblingAndHelper + ); + + expect(alone.status).toBe("fail"); + expect(withSiblingAndHelper.status).toBe("fail"); + }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the // caught binding itself, was credited with classifying an error it never inspected. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index ae67d598bc7..af022e34974 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -882,6 +882,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, tryStatementCount: 1, + enclosingStatementCount: 10, }); expect(ep!.catches[1]).toMatchObject({ guardsParse: false, @@ -983,6 +984,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, tryStatementCount: 4, + enclosingStatementCount: 6, }); }); From 9e3b21a65a30feb1de9af05248b9bf35e0bec2a8 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 16:48:47 +0100 Subject: [PATCH 045/117] fix(observability-map): stop attributing an inline callback's catch to the route The catch collector descended into inline callbacks (Promise.all(items.map(async i => { try { ... } catch { ... } }))) while countStatement, by design, does not, so the two disagreed about what the entry point even is: tryStatementCount could exceed the entry's own statementCount, and a per-item error boundary was judged as though it were the route's own catch. Chose not to descend into an inline callback for catch evidence either, the smaller change: it matches the existing statement-counting decision rather than reopening the over-counting a callback walk was fixed for earlier. calleeNames and logCalls still descend into callbacks, unchanged, since isTrivial deliberately relies on calleeNames seeing work a short statement count hides. hasTryCatch also stays true for a callback-nested try, since it is only a "some try appeared" signal for triviality, not an attribution claim. Measured on the real tree: five routes move, four from fail to not-applicable and one from pass to not-applicable, all genuinely catching nothing at the route's own level once a callback's catch is no longer credited to it. Global moves from 16 to 15. --- .../observability-map/src/scan.ts | 21 ++++++++++--- .../observability-map/test/checks.test.ts | 25 +++++++++++++++ .../observability-map/test/scan.test.ts | 31 +++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 4f43e70745d..72b6b35ba16 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -631,10 +631,21 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += enclosingStatementCount; if (!fn.body) return; - const visit = (node: ts.Node, inCatch: boolean) => { + // `inCallback` is true once the walk has entered an inline function (`.map((item) => { ... })`), + // never reset back to false: a callback nested inside another callback is still a callback. + // `calleeNames` and `logCalls` keep descending in there regardless, which is what lets + // `isTrivial` see work a short statement count hides. A try/catch does not: it is not part of + // this body's own statement list, and `countStatement` already stops at a nested function + // boundary, so counting it here let `tryStatementCount` exceed the entry point's whole + // `statementCount` and judged a per-item error boundary as though it were the route's own. + const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { + if (ts.isFunctionLike(node)) { + ts.forEachChild(node, (child) => visit(child, inCatch, true)); + return; + } if (ts.isTryStatement(node)) { hasTryCatch = true; - if (node.catchClause) { + if (node.catchClause && !inCallback) { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ @@ -648,7 +659,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } if (ts.isCatchClause(node)) { - ts.forEachChild(node, (child) => visit(child, true)); + ts.forEachChild(node, (child) => visit(child, true, inCallback)); return; } @@ -678,9 +689,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { } } } - ts.forEachChild(node, (child) => visit(child, inCatch)); + ts.forEachChild(node, (child) => visit(child, inCatch, inCallback)); }; - visit(fn.body, false); + visit(fn.body, false, false); }; for (const fn of target.functions) walkBody(fn, true); diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index b958cd7a487..11c606639e2 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -340,6 +340,31 @@ describe("error-classification", () => { ); expect(r.status).toBe("not-applicable"); }); + + // A7. A per-item error boundary inside a `.map()` callback used to be judged as the route's own + // catch. The route's own visible body catches nothing, so it is not-applicable, not a pass or a + // fail on the strength of a swallow one level of nesting away. + it("is not applicable to a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); }); describe("auth-boundary", () => { diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index af022e34974..0f0151df9d7 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -798,6 +798,37 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.hasTryCatch).toBe(false); expect(ep!.catches).toEqual([]); }); + + // A7. The catch collector descended into inline callbacks while `countStatement`, by design, + // does not, so a per-item error boundary inside a `.map()` was judged as the route's own catch + // and `tryStatementCount` could exceed the entry point's whole `statementCount`. Fixed by not + // descending into inline callbacks for catch evidence either, the smaller change: it matches the + // existing statement-counting decision rather than reopening the over-counting a callback walk + // was fixed for earlier. `calleeNames` and `logCalls` still descend, unchanged, which is what lets + // `isTrivial` see work a short statement count hides. + it("does not attribute a catch inside an inline callback (Promise.all(items.map(...))) to the route", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); }); describe("scanFile: log calls", () => { From 470422889355aeaa5b8002a0315288d1fe54b7d9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:28:14 +0100 Subject: [PATCH 046/117] fix(observability-map): root the callback-boundary rule for catch attribution C1: the A7 fix stopped at ANY function-like node, purely lexical, so wrapping a whole route body in a single-shot callback (trace(async () => {...}), mutateWithFallback({ pgMutation }), new ReadableStream({ start })) deleted its catches and error-classification went not-applicable, worth up to 50 points for a syntactic no-op. That is the A1 defect class, reintroduced. Confirmed wrong on three real routes: api.v2.runs.$runParam.cancel.ts (the main mutation path, inside pgMutation), the presence.tsx SSE tick handler, and the ai-generate.tsx streaming body inside ReadableStream.start. The real distinction is per-item iteration versus everything else. walkBody now only treats a callback passed to map/forEach/filter/reduce/reduceRight/ flatMap/some/every as a boundary; every other function-like node, including the three wrapper shapes above, is descended into and its catch attributed. Verified against all five real routes the previous fix moved: the three wrongly-affected ones are fail again (their own genuine swallow), the two genuine .map() per-item boundaries stay not-applicable. C2: catchClauseEvidence still descended into every function body unconditionally, so a throw or an error test merely constructed inside a callback the clause registers (queue.push(() => { throw e }), setTimeout) was credited to the clause, and reachableStatements only handled dead code from statement ordering, not a statically-false condition (if (false), while (false)). Both closed: the clause-level walk stops at every function-like node (broader than walkBody's boundary on purpose, since here we are already inside a catch the route owns, and constructing a further callback is deferred work, not the clause's own decision), and isFalsyLiteral recognises false/null/undefined/0 so a dead branch is never visited. A do/while, which runs its body once regardless of the trailing condition, still registers, proving the fix is not over-broad. Real-tree effect: global unchanged at 15, measured unchanged at 412. Three routes move from not-applicable back to fail (their own catch, correctly attributed again); two more show additional catches now discovered (previously wrongly excluded) with no change to their overall verdict. --- .../observability-map/src/scan.ts | 93 +++++++- .../observability-map/test/scan.test.ts | 215 ++++++++++++++++-- 2 files changed, 275 insertions(+), 33 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 72b6b35ba16..a3bf3456363 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -260,17 +260,45 @@ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.S return index === -1 ? statements : statements.slice(0, index + 1); } -/** What a catch clause does with the error, beyond the fact that it caught one. */ +/** + * A literal that is always falsy: `false`, `null`, `undefined`, `0`. Not general constant folding, + * on purpose: `!true`, `1 === 2` and a reference to a `const` declared elsewhere are not covered, so + * `if (false) { throw e; }` and `while (false) { throw e; }` are recognised as dead and nothing more + * elaborate is claimed to be. + */ +function isFalsyLiteral(expr: ts.Expression): boolean { + if (expr.kind === ts.SyntaxKind.FalseKeyword || expr.kind === ts.SyntaxKind.NullKeyword) { + return true; + } + if (ts.isIdentifier(expr) && expr.text === "undefined") return true; + return ts.isNumericLiteral(expr) && expr.text === "0"; +} + +/** + * What a catch clause does with the error, beyond the fact that it caught one. + * + * Stops at every function-like node, not only an iteration callback (contrast `walkBody`'s + * boundary, which lets a route's own single-shot wrapper, `trace(async () => {...})`, + * `mutateWithFallback({ pgMutation })`, `new ReadableStream({ start })`, through so the route's own + * catch is found at all). Here the walk is already inside a catch clause that `walkBody` decided + * belongs to the route; anything the clause does by constructing a further callback, + * `queue.push(() => { throw e; })`, a `.then`, a `setTimeout`, is deferred work the clause merely + * registers, not a decision it makes on its own execution. Both walks refuse a per-item iteration + * callback; this one refuses every other kind of callback too, for that reason. + */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; const bindingName = catchBindingName(clause); const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node)) return; + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { for (const statement of reachableStatements(node.statements)) visit(statement); return; } + if (ts.isThrowStatement(node)) rethrows = true; if ( bindingName !== null && @@ -280,6 +308,20 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc branches = true; } if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; + + // A statically-false condition makes the guarded statement dead code: it can set `branches` + // above (the clause still decided to test the error, even if the arm never runs), but nothing + // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. + if (ts.isIfStatement(node)) { + if (!isFalsyLiteral(node.expression)) visit(node.thenStatement); + if (node.elseStatement) visit(node.elseStatement); + return; + } + if (ts.isWhileStatement(node)) { + if (!isFalsyLiteral(node.expression)) visit(node.statement); + return; + } + ts.forEachChild(node, visit); }; visit(clause.block); @@ -287,6 +329,35 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc return { rethrows, branches }; } +/** + * Array methods that invoke their callback once per element, never once as a whole. The one + * structural signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a + * fresh catch for every element) from a route's own body expressed through one more layer of + * function nesting (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => + * {...} })`, `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their + * callback exactly once, as the route's own continuation. + */ +const ITERATION_METHODS = new Set([ + "map", + "forEach", + "filter", + "reduce", + "reduceRight", + "flatMap", + "some", + "every", +]); + +/** Whether the function-like `node` is the callback argument of a call to one of + * `ITERATION_METHODS`, e.g. the arrow function in `items.map((item) => ...)`. */ +function isIterationCallback(node: ts.Node): boolean { + const parent = node.parent; + if (!parent || !ts.isCallExpression(parent)) return false; + if (!parent.arguments.includes(node as ts.Expression)) return false; + const callee = unwrap(parent.expression); + return ts.isPropertyAccessExpression(callee) && ITERATION_METHODS.has(callee.name.text); +} + const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); function propertyName(property: ts.ObjectLiteralElementLike): string | null { @@ -631,16 +702,22 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { statementCount += enclosingStatementCount; if (!fn.body) return; - // `inCallback` is true once the walk has entered an inline function (`.map((item) => { ... })`), - // never reset back to false: a callback nested inside another callback is still a callback. - // `calleeNames` and `logCalls` keep descending in there regardless, which is what lets - // `isTrivial` see work a short statement count hides. A try/catch does not: it is not part of - // this body's own statement list, and `countStatement` already stops at a nested function - // boundary, so counting it here let `tryStatementCount` exceed the entry point's whole + // `inCallback` is true once the walk has entered a per-item iteration callback + // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is + // still inside it. `calleeNames` and `logCalls` keep descending regardless, which is what lets + // `isTrivial` see work a short statement count hides. A try/catch does not: a per-item catch is + // not part of this body's own statement list, and `countStatement` already stops at a nested + // function boundary, so counting it here let `tryStatementCount` exceed the entry point's whole // `statementCount` and judged a per-item error boundary as though it were the route's own. + // + // Only an iteration callback is a boundary, not every function-like node: a route's own body + // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` + // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's + // own continuation one layer of nesting away, and its catch is the route's own error handling. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { - ts.forEachChild(node, (child) => visit(child, inCatch, true)); + const entersIterationCallback = inCallback || isIterationCallback(node); + ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 0f0151df9d7..aafc72438db 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -636,6 +636,67 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); + // C2. `reachableStatements` only handled dead code from statement ordering (A4), not a + // statically-false condition, and `catchClauseEvidence`'s walk descended into every function + // body unconditionally, so a throw or an error test merely REGISTERED in a callback the clause + // constructs (never executed as part of the clause's own synchronous handling) was credited to + // it. All four shapes below must leave a plain swallow (`catch (e) { return null; }`) inert. + describe("dead and deferred code inside a catch does not count as evidence", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + return null; + } + } + `; + + it("is inert as a baseline with no mutation", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw inside a statically-false if", () => { + const ep = scanFile("x.ts", swallow("if (false) { throw e; }")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw inside a statically-false while", () => { + const ep = scanFile("x.ts", swallow("while (false) { throw e; }")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set rethrows for a throw merely registered in a constructed callback", () => { + const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set branches for an error test merely registered in a constructed callback", () => { + const ep = scanFile( + "x.ts", + swallow("queue.push(() => { if (e instanceof Error) { doThing(); } });") + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Extra input beyond the brief's four, exercising the same "merely registered" mechanism with + // a different callback-taking call, to check the fix is not scoped to `.push` specifically. + it("does not set rethrows for a throw registered in a setTimeout callback", () => { + const ep = scanFile("x.ts", swallow("setTimeout(() => { throw e; }, 0);")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Positive control: a do/while runs its body at least once regardless of the trailing + // condition, so a throw in one is genuinely unconditional and must still register. This checks + // the while(false) fix was not implemented broadly enough to swallow a real rethrow too. + it("still sets rethrows for a throw in a do/while, which runs its body once regardless", () => { + const ep = scanFile("x.ts", swallow("do { throw e; } while (false);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", @@ -799,35 +860,139 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches).toEqual([]); }); - // A7. The catch collector descended into inline callbacks while `countStatement`, by design, - // does not, so a per-item error boundary inside a `.map()` was judged as the route's own catch - // and `tryStatementCount` could exceed the entry point's whole `statementCount`. Fixed by not - // descending into inline callbacks for catch evidence either, the smaller change: it matches the - // existing statement-counting decision rather than reopening the over-counting a callback walk - // was fixed for earlier. `calleeNames` and `logCalls` still descend, unchanged, which is what lets - // `isTrivial` see work a short statement count hides. - it("does not attribute a catch inside an inline callback (Promise.all(items.map(...))) to the route", () => { - const ep = scanFile( - "batch.process.ts", - ` - export async function action({ request }) { - const items = await loadItems(request); - await Promise.all( - items.map(async (item) => { + // A7 / C1. The first fix stopped at ANY function-like node, purely lexical, which correctly + // excludes a per-item `.map()` boundary but also deletes the route's own catch when the whole + // body is wrapped in a single-shot callback: `trace(async () => { ...whole body... })`, + // `mutateWithFallback({ pgMutation: async (t) => {...} })`, `new ReadableStream({ start: async (c) + // => {...} })`. All three invoke their callback exactly once, as the route's own continuation. + // The real distinction is per-item iteration versus everything else, so only a callback passed to + // `map`/`forEach`/`filter`/`reduce`/`reduceRight`/`flatMap`/`some`/`every` is a boundary now. + describe("inline single-shot wrappers are attributed to the route", () => { + it("attributes a catch wrapped in trace(async () => {...})", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + return trace("update", async () => { + try { + await doWork(request); + return json({ ok: true }); + } catch { + return json({ error: "failed" }, { status: 500 }); + } + }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside a pgMutation callback passed as an object property", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const outcome = await mutateWithFallback({ + pgMutation: async (taskRun) => { + try { + await doWork(taskRun); + } catch { + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }, + }); + return outcome; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside new ReadableStream({ start })", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const stream = new ReadableStream({ + async start(controller) { + try { + await doWork(controller); + } catch { + controller.error("failed"); + } + }, + }); + return new Response(stream); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + }); + + describe("per-item iteration callbacks are not attributed to the route", () => { + it("does not attribute a catch inside items.map(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map((item) => { try { - await processItem(item); + return process(item); } catch { return null; } - }) - ); - return json({ ok: true }); - } - ` - ); - expect(ep!.catches).toEqual([]); - // A try/catch appeared somewhere in the body, which is still a real signal for triviality. - expect(ep!.hasTryCatch).toBe(true); + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + it("does not attribute a catch inside Promise.all(items.map(...))", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("does not attribute a catch inside items.forEach(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + items.forEach((item) => { + try { + process(item); + } catch { + return; + } + }); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); }); }); From 255accbc4fa7e945c0d7500251f612f238d23ba0 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:36:21 +0100 Subject: [PATCH 047/117] fix(observability-map): read suppression comments off the parsed source, not a bare scanner I3: a standalone ts.createScanner has no parser state, so it still granted suppressions nobody wrote in two shapes, both reproduced end to end: a template literal WITH a substitution is never rescanned as a continuation after the `}`, so text following it lexes as ordinary code and a `//` in it is a real comment; and a scanner created in LanguageVariant.Standard has no JSX context, so `//` inside JSX text (e.g. a URL in prose) lexes as a line comment. The prior docstring claimed a template literal was one token with "no substring rule left to fool", which was false for both shapes; this is the second docstring in a row on this function to overclaim a safety property, so this one describes the mechanism and names the residual instead. Replaced the scanner with comment ranges read off the actual parsed ts.SourceFile: getLeadingCommentRanges and getTrailingCommentRanges at every leaf token's boundary (walking .getChildren(), which reaches bare punctuation and keyword tokens that ts.forEachChild silently skips). A template's literal segments and a JSX text node are real nodes with their own span there, never trivia, so a directive inside either is content the parser already claimed. suppressedChecks now takes the fileName too, to pick ScriptKind.TSX vs TS: parsing every file as TSX would misparse a generic arrow function ((x) => x) as JSX in a plain .ts file. Verified against the two brief shapes plus four extra inputs: a second substitution, a string literal nested in a JSX expression container, a genuine directive in a .tsx file, and a generic arrow function in a .ts file with a real trailing comment, to check the script-kind change did not break ordinary parsing. Zero real-tree effect (no live suppressions exist today). --- .../observability-map/src/score.ts | 2 +- .../observability-map/src/suppression.ts | 97 ++++++++++++++----- .../test/suppression.test.ts | 72 +++++++++++++- 3 files changed, 141 insertions(+), 30 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index a39a1ab7c82..66ed594e2d7 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -48,7 +48,7 @@ export type MapReport = { }; export function scoreEntry(ep: EntryPoint): ScoredEntry { - const suppressed = suppressedChecks(ep.source); + const suppressed = suppressedChecks(ep.source, ep.fileName); const raw = CHECKS.map((c) => c.run(ep)); const checks = raw.map((result) => { const reason = suppressed.get(result.id); diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 6012ee09d4e..4740594746d 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -15,47 +15,92 @@ import ts from "typescript"; const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** - * Every physical line of genuine comment content in the source, line comments and block comments - * alike, one entry per line, with the `//`, `/*`, `*​/` and a jsdoc `*` prefix stripped. + * Every leaf token in the parsed source: keeps descending through `.getChildren()` rather than + * `ts.forEachChild`, which only returns the child nodes a statement or expression models as its + * own properties and silently skips a bare punctuation or keyword token (a closing brace, a + * semicolon). A comment can sit directly before one of those with nothing else following it, the + * last line inside a block, and `.getChildren()` still reaches it because the token itself is + * still a node with a position. + */ +function leafTokens(node: ts.Node): ts.Node[] { + const children = node.getChildren(); + return children.length === 0 ? [node] : children.flatMap(leafTokens); +} + +/** + * Every genuine comment range in the source, read off a real parsed `ts.SourceFile` rather than a + * standalone `ts.createScanner`. A bare scanner has no parser state behind it, and that is exactly + * what let two shapes through: + * + * - a template literal WITH a substitution never gets rescanned as a template continuation by a + * scanner running on its own, so the text after `${x}` reads as ordinary code and a `//` in it is + * a real comment to the scanner, though it never leaves the template literal to the parser. + * - JSX text has no comment syntax at all, but a scanner created in `LanguageVariant.Standard` + * does not know it is looking at JSX text, so a `//` inside `

see https://x

` reads as a + * line comment starting mid-URL. * - * Read from the TypeScript scanner's own token stream rather than `indexOf("//")` against the raw - * text. The old text-matching read the directive out of a string or template literal that merely - * quoted it, so `"see // obs-map-disable auth-boundary -- nope"` granted a suppression nobody - * wrote, silencing a real check. The scanner already knows the difference: a string or template - * literal is one token, consumed in a single step, and never yields comment trivia for what is - * inside it, so there is no substring rule left to fool. + * The actual parser closes both: a template's literal segments and a JSX text node are real nodes + * with their own span here, never trivia, so a directive inside either is content the parser + * already claimed, not a comment. `getLeadingCommentRanges` and `getTrailingCommentRanges` are both + * needed at every token boundary, because which one returns a given comment depends on whether it + * shares a line with the token before it (trailing) or comes after a line break (leading), not on + * which directive it happens to be. */ -function commentLines(source: string): string[] { - const scanner = ts.createScanner( - ts.ScriptTarget.Latest, - /* skipTrivia */ false, - ts.LanguageVariant.Standard, - source - ); - const lines: string[] = []; +function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const seen = new Set(); + const ranges: ts.CommentRange[] = []; + const add = (found: ts.CommentRange[] | undefined) => { + for (const range of found ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + ranges.push(range); + } + }; + for (const token of leafTokens(sf)) { + add(ts.getLeadingCommentRanges(source, token.getFullStart())); + add(ts.getTrailingCommentRanges(source, token.getEnd())); + } + return ranges; +} - for (let kind = scanner.scan(); kind !== ts.SyntaxKind.EndOfFileToken; kind = scanner.scan()) { - if (kind === ts.SyntaxKind.SingleLineCommentTrivia) { - lines.push(scanner.getTokenText().slice(2)); +/** One physical line of comment content per range, the `//`, `/*`, `*​/` and a jsdoc `*` prefix + * stripped, so a multi-line block comment still matches the directive one line at a time. */ +function commentLines(source: string, sf: ts.SourceFile): string[] { + const lines: string[] = []; + for (const range of commentRanges(source, sf)) { + const text = source.slice(range.pos, range.end); + if (range.kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(text.slice(2)); continue; } - if (kind !== ts.SyntaxKind.MultiLineCommentTrivia) continue; - - const text = scanner.getTokenText(); const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ for (const rawLine of body.split("\n")) { const trimmed = rawLine.trimStart(); lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); } } - return lines; } -/** Check id to reason. A suppression without a reason, or outside a comment, is ignored. */ -export function suppressedChecks(source: string): Map { +/** + * Check id to reason. A suppression without a reason, or outside a comment, is ignored. + * + * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly + * distinguished from a generic type argument list (`(x) => x`), when the file is really a + * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. + */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sf = ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + scriptKind + ); + const out = new Map(); - for (const line of commentLines(source)) { + for (const line of commentLines(source, sf)) { const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 3259278a594..7ef32f9f94e 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -92,9 +92,8 @@ describe("suppressedChecks", () => { // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a // string that merely quotes the directive granted a suppression nobody wrote and silenced a real - // check. Reading genuine comment ranges from the TypeScript scanner closes both shapes: the - // scanner consumes a string or template literal as one token and never emits comment trivia for - // what is inside it. + // check. Reading comment ranges off the parsed source closes this: a string literal is one node + // with its own span, never trivia, so a directive inside it is content, not a comment. it("does not suppress from a directive quoted inside a string literal", () => { const m = suppressedChecks( `const msg = "see // obs-map-disable error-classification -- because reasons"; @@ -118,4 +117,71 @@ describe("suppressedChecks", () => { ); expect(m.size).toBe(0); }); + + // I3. A standalone `ts.createScanner` has no parser state, so it still granted two suppressions + // nobody wrote: it never rescans a template as a continuation after a `${...}` substitution, so + // text after the `}` reads as ordinary code and a `//` in it is a real comment to the scanner; + // and a scanner created in `LanguageVariant.Standard` has no JSX context, so a `//` inside JSX + // text reads as a line comment mid-URL. Reading comments off the actual parsed tree closes both: + // a template's literal segments and a JSX text node are real nodes, never trivia. + it("does not suppress from a directive after a template substitution", () => { + const m = suppressedChecks( + "const msg = `${name} // obs-map-disable error-classification -- via substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside JSX text", () => { + const m = suppressedChecks( + `export default function Page() { + return

docs at https://example.com obs-map-disable error-classification -- x

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism + // differently: a second substitution, and a string literal nested inside a JSX expression + // container, which is a different node kind again from either hole above. + it("does not suppress from a directive after a second template substitution", () => { + const m = suppressedChecks( + "const msg = `${a}${b} // obs-map-disable auth-boundary -- nested substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a string literal nested in a JSX expression container", () => { + const m = suppressedChecks( + `export default function Page() { + return

{"see // obs-map-disable request-context -- nested string"}

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control: a genuine directive still works in a .tsx file, and a directive after a + // template with no substitution (already covered above) is not the only shape that must survive. + it("still reads a genuine directive in a .tsx file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + export default function Page() { return

hi

; }`, + "route.tsx" + ); + expect(m.get("error-classification")).toBe("liveness probe"); + }); + + // Regression control: a generic arrow function is only unambiguous when the file is parsed as + // plain TypeScript, not TSX (`` would otherwise start a JSX element). A .ts file must still + // parse sanely and keep reading a genuine trailing comment correctly. + it("still reads a genuine directive beside a generic arrow function in a .ts file", () => { + const m = suppressedChecks( + "const identity = (x: T): T => x; // obs-map-disable auth-boundary -- generic helper\n" + + "export async function loader() { return identity(1); }" + ); + expect(m.get("auth-boundary")).toBe("generic helper"); + }); }); From fc73a2e485c04353f4ad1d2586fdc1c2cd401088 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:43:08 +0100 Subject: [PATCH 048/117] fix(observability-map): make every published figure pre-suppression I4: measured was fixed to read pre-suppression applicability, but contextGap and auditGap still read `checks`, the post-suppression display view, so suppressing the one request-context finding on an entry shrank CONTEXT's denominator too and raised its printed percentage, on the same screen as the SUPPRESSED line's claim that suppression cannot raise a score. Reproduced exactly: two routes, one failing request-context, suppressing it moved CONTEXT from 1 of 2 (50%) to 1 of 1 (100%). Second half: audit-trail is not in SCORED_CHECK_IDS, so `suppressed` never recorded an audit-trail suppression, so a written directive silently disappeared from the SUPPRESSED line while the audit denominator shrank underneath it, confirmed with a standalone audit-trail suppression: old code counted zero suppressions while auditGap still moved 2/1 to 1/1. ScoredEntry now carries rawChecks, every check exactly as it ran before a suppression can turn a result into not-applicable. contextGap and auditGap both read it instead of `checks`. `suppressed` now reads every check a directive named, scored or not, so audit-trail suppressions are counted too. Also collapsed three copies of the same suppression/denominator explanation (M10) down to one, on the field that actually needs it. Zero real-tree effect (no live suppressions exist today); both repros verified directly against old and new code side by side. --- .../observability-map/src/score.ts | 36 ++++++---- .../observability-map/test/score.test.ts | 71 +++++++++++++++++++ 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 66ed594e2d7..7a5c6924bda 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -9,7 +9,15 @@ export type ScoredEntry = { routePath: string; family: Family; sensitive: boolean; + /** Post-suppression: a suppressed check reads `not-applicable` here, with the reason in + * `detail`. This is the display view; every denominator below reads `rawChecks` instead, so a + * suppression is never invisible to a published figure just because its check is not scored. */ checks: CheckResult[]; + /** Every check exactly as it ran, before a suppression comment can turn a result into + * `not-applicable`. The one true source for any figure that counts applicability: `measured` + * below, and `contextGap`/`auditGap` in `MapReport`, which read this rather than `checks` for + * exactly that reason. */ + rawChecks: CheckResult[]; /** * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable * before suppression. A fully-suppressed entry stays measured, at its capped score, so a @@ -19,7 +27,8 @@ export type ScoredEntry = { * default cannot inflate a figure nobody checked. */ measured: boolean; - /** Scored checks a comment in the source suppressed, in `SCORED_CHECK_IDS` order. */ + /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes + * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ suppressed: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; @@ -67,10 +76,6 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { }; const visible = scored.filter((c) => !suppressed.has(c.id)); - // Pre-suppression: an entry whose only applicable check gets suppressed still had something to - // measure, and must stay in the denominator at its capped score rather than vanish as if nothing - // ever applied. `unmeasured` is reserved for entries with no applicable check at all, suppression - // or no suppression. const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); return { @@ -79,13 +84,12 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { family: familyOf(ep.fileName), sensitive: classifySensitivity(ep).sensitive, checks, - suppressed: scored.filter((c) => suppressed.has(c.id)).map((c) => c.id), - // Suppressing a check takes it out of the numerator and the denominator, and the result is - // capped by what the entry would have scored unsuppressed. Otherwise removing a failing check - // shrinks the denominator and the ratio climbs, which is how 33 became 50 became 100: the - // suppression comment laundered the finding into a point. What a suppression buys is removal - // from the worklist, with a reason on the record. It cannot buy a better number. + rawChecks: raw, + suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), measured: scoredApplicable.length > 0, + // Capped by the pre-suppression ratio: removing a failing check from both the numerator and + // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this + // cap existed. See ScoredEntry.measured for why the denominator itself is pre-suppression too. score: Math.min(ratio(visible), ratio(scored)), }; } @@ -131,12 +135,16 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo // reported here as its own architectural figure instead: how many sensitive mutations have an // audit record, out of how many. Folding it into the score would tank every sensitive route on a // gap that is the same everywhere, and bury the routes that have their own, fixable problems. + // + // Both gaps read `rawChecks`, pre-suppression, the same reason `measured` does: suppressing the + // one request-context or audit-trail finding on an entry must not shrink these denominators and + // raise the printed percentage, on the same screen as a claim that suppression cannot do that. const contextChecks = entries - .map((e) => e.checks.find((c) => c.id === "request-context")) + .map((e) => e.rawChecks.find((c) => c.id === "request-context")) .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); const auditApplicable = entries.filter((e) => - e.checks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + e.rawChecks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") ); const suppressing = entries.filter((e) => e.suppressed.length > 0); @@ -154,7 +162,7 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo auditGap: { sensitiveMutations: auditApplicable.length, withAudit: auditApplicable.filter((e) => - e.checks.some((c) => c.id === "audit-trail" && c.status === "pass") + e.rawChecks.some((c) => c.id === "audit-trail" && c.status === "pass") ).length, }, contextGap: { diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index 05386622993..c64c75bbed8 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -186,6 +186,77 @@ ${BUSY_AND_FAILING}` expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); }); + // I4. contextGap and auditGap read `checks` (post-suppression), so suppressing the one failing + // request-context on an entry removed it from the denominator too, moving CONTEXT from 1 of 2 + // (50%) to 1 of 1 (100%) printed on the same screen as "a suppression does not raise a score". + // Both gaps now read `rawChecks`, pre-suppression, exactly like `measured`. + it("does not let a suppressed request-context finding shrink the context gap's denominator", () => { + const failing = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + const passing = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } +}`; + + const before = buildReport([scanFile("a.ts", failing)!, scanFile("b.ts", passing)!], []); + expect(before.contextGap).toEqual({ applicable: 2, naming: 1 }); + + const after = buildReport( + [ + scanFile( + "a.ts", + `// obs-map-disable request-context -- silence +${failing}` + )!, + scanFile("b.ts", passing)!, + ], + [] + ); + expect(after.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + // I4, second half. A suppressed audit-trail directive never showed up in `suppressed` because + // audit-trail is not in SCORED_CHECK_IDS, so the SUPPRESSED line undercounted while the audit + // denominator silently shrank underneath it. Every suppression is now counted, scored or not. + it("counts an audit-trail suppression and does not let it shrink the audit gap's denominator", () => { + const missingAudit = `import { prisma } from "~/db.server"; +export async function action() { return prisma.token.create({ data: {} }); }`; + const withAudit = `import { auditLog } from "~/services/audit.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); +}`; + + const before = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", missingAudit)!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(before.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + + const after = buildReport( + [ + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable audit-trail -- accepted risk +${missingAudit}` + )!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(after.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + expect(after.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + it("reports the audit gap separately from the score", () => { // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported // as its own figure and must not pull the score down with it. From 9e300335efc8196ffe1e914041afb55c94c4ee5b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 17:51:29 +0100 Subject: [PATCH 049/117] fix(observability-map): shadow-check a binding from any enclosing scope, not just a nested one I5: referencesBinding's shadow logic only fires while searching downward from an if/switch's own condition, so it caught shadowing NESTED inside that condition (what the existing tests exercise) but not a scope that instead WRAPS the if: a for...of loop declaring the same name, or a nested catch clause with the same binding. Confirmed both credited a branch that should not count, plus two more of the same shape (for...in, a classic for with a let-declared shadow) and a destructured const inside a plain block. catchClauseEvidence now tracks shadowing as it descends, the same pattern `inCallback` already uses: once a scope re-declares the binding, a for/ for-of/for-in loop's own variable, a nested catch with the same name, or a block that declares it, everything nested inside stays shadowed and an if/ switch referencing the name there no longer sets branches. Positive controls confirm a genuine reference, including through a for-of loop with a DIFFERENT loop variable, still counts. Second bug found while fixing the first: the function-parameter and catch-clause shadow checks inside referencesBinding only recognised ts.isIdentifier, so a destructured parameter ({ error }) was invisible to them too, even directly inside an if's own condition (items.some(({ error }) => error > 0)), a shape C1/C2 do not otherwise cover. Added a recursive bindingDeclares helper, reused across the function-parameter check, the catch-clause check, declaresInScope, and a new declaresLoopVariable, so every declaration-shadow check in the file recognises a destructured name the same way. Zero real-tree effect, same as A3's original fix: reproduces synthetically, no live occurrence in the current tree. --- .../observability-map/src/scan.ts | 86 ++++++++++++++--- .../observability-map/test/scan.test.ts | 95 +++++++++++++++++++ 2 files changed, 166 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index a3bf3456363..d8b10b5da55 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -159,6 +159,21 @@ function containsInstanceOf(node: ts.Node): boolean { ); } +/** + * Whether a binding name pattern declares `target`, recursively: a plain `error`, a destructured + * `{ error }` (shorthand) or `{ code: error }` (renamed), an array pattern `[error]`, and any of + * those nested inside another. A destructured parameter or declaration re-declares the name just as + * completely as a plain one does, so a shadow check that only recognised `ts.isIdentifier` missed + * every destructured shape, function parameters and variable declarations alike. + */ +function bindingDeclares(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target; + for (const element of name.elements) { + if (!ts.isOmittedExpression(element) && bindingDeclares(element.name, target)) return true; + } + return false; +} + /** Whether `name` is declared by a var/let/const, function or class statement directly in this * statement list. Not recursive: a nested block's own declarations are handled when the walk * reaches that block. */ @@ -168,9 +183,7 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; if ( ts.isVariableStatement(statement) && - statement.declarationList.declarations.some( - (d) => ts.isIdentifier(d.name) && d.name.text === name - ) + statement.declarationList.declarations.some((d) => bindingDeclares(d.name, name)) ) { return true; } @@ -178,14 +191,29 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo return false; } +/** Whether a `for`/`for...of`/`for...in` loop's own declared variable is `name`, so its body + * shadows the rest of the enclosing scope for that name. */ +function declaresLoopVariable( + node: ts.ForStatement | ts.ForOfStatement | ts.ForInStatement, + name: string +): boolean { + const initializer = node.initializer; + return ( + initializer !== undefined && + ts.isVariableDeclarationList(initializer) && + initializer.declarations.some((d) => bindingDeclares(d.name, name)) + ); +} + /** * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the * binding's text without reading it: the property side of a member expression (`fallback.error`) * and an object literal key (`{ error: true }`), both excluded by checking which side of the * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch - * parameter or as a var/let/const/function/class in a block, refers to that declaration instead, - * so the walk stops at the boundary that re-declares it rather than crediting the outer binding. + * parameter (including a destructured one) or as a var/let/const/function/class in a block, refers + * to that declaration instead, so the walk stops at the boundary that re-declares it rather than + * crediting the outer binding. */ function referencesBinding(node: ts.Node, bindingName: string): boolean { if (ts.isIdentifier(node) && node.text === bindingName) { @@ -197,14 +225,14 @@ function referencesBinding(node: ts.Node, bindingName: string): boolean { if ( ts.isFunctionLike(node) && - node.parameters.some((p) => ts.isIdentifier(p.name) && p.name.text === bindingName) + node.parameters.some((p) => bindingDeclares(p.name, bindingName)) ) { return false; } if (ts.isCatchClause(node)) { const decl = node.variableDeclaration; - if (decl && ts.isIdentifier(decl.name) && decl.name.text === bindingName) return false; + if (decl && bindingDeclares(decl.name, bindingName)) return false; } if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; @@ -291,40 +319,68 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc let branches = false; const bindingName = catchBindingName(clause); - const visit = (node: ts.Node) => { + // `shadowed` is true once the walk has passed a scope that re-declares `bindingName`: a nested + // catch clause with the same name, a `for`/`for...of`/`for...in` loop declaring it, or a block + // that does (reusing the same `declaresInScope` a bare block already checks). From there on, an + // `if`/`switch` that references the name textually is referencing the SHADOWING declaration, not + // this clause's own binding, so it must not count as this clause deciding anything. Never reset + // back to false, the same rule `inCallback` follows: once shadowed, everything nested inside is + // still shadowed. + const visit = (node: ts.Node, shadowed: boolean) => { if (ts.isFunctionLike(node)) return; + if (bindingName !== null && ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + const shadowsHere = decl !== undefined && bindingDeclares(decl.name, bindingName); + ts.forEachChild(node, (child) => visit(child, shadowed || shadowsHere)); + return; + } + + if ( + bindingName !== null && + (ts.isForStatement(node) || ts.isForOfStatement(node) || ts.isForInStatement(node)) && + declaresLoopVariable(node, bindingName) + ) { + visit(node.statement, true); + return; + } + if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { - for (const statement of reachableStatements(node.statements)) visit(statement); + const shadowedHere = + shadowed || (bindingName !== null && declaresInScope(node.statements, bindingName)); + for (const statement of reachableStatements(node.statements)) visit(statement, shadowedHere); return; } if (ts.isThrowStatement(node)) rethrows = true; if ( + !shadowed && bindingName !== null && ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) ) { branches = true; } - if (ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) branches = true; + if (!shadowed && ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) { + branches = true; + } // A statically-false condition makes the guarded statement dead code: it can set `branches` // above (the clause still decided to test the error, even if the arm never runs), but nothing // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. if (ts.isIfStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.thenStatement); - if (node.elseStatement) visit(node.elseStatement); + if (!isFalsyLiteral(node.expression)) visit(node.thenStatement, shadowed); + if (node.elseStatement) visit(node.elseStatement, shadowed); return; } if (ts.isWhileStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.statement); + if (!isFalsyLiteral(node.expression)) visit(node.statement, shadowed); return; } - ts.forEachChild(node, visit); + ts.forEachChild(node, (child) => visit(child, shadowed)); }; - visit(clause.block); + visit(clause.block, false); return { rethrows, branches }; } diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index aafc72438db..92f6a093bdb 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1579,3 +1579,98 @@ describe("scanFile: branches requires a genuine read of the binding, not a looka expect(ep!.catches[0]!.branches).toBe(true); }); }); + +// I5. The shadow check only fired inside `referencesBinding`'s own top-down search from an +// if/switch's condition, so it only caught shadowing NESTED inside that condition, exactly what +// the tests above exercise. A shadowing scope that instead WRAPS the if (a for-of loop, a nested +// catch with the same name) was invisible, because nothing walked up from the if to notice it. +// catchClauseEvidence now tracks shadowing as it descends, the same way `inCallback` tracks a +// callback boundary: once a scope re-declares the binding, everything nested inside stays shadowed. +describe("scanFile: a binding shadowed by an enclosing scope, not just a nested one", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return null; + } + } + `; + + it("does not credit an if inside a for...of loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error of errors) { if (error.code === 1) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a for...in loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error in errorsByKey) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a classic for loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (let error = 0; error < 10; error++) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a nested catch clause with the same binding name", () => { + const ep = scanFile( + "x.ts", + swallow("try { doWork(); } catch (error) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a block whose own destructured const shadows the binding", () => { + const ep = scanFile( + "x.ts", + swallow(` + if (attempt > 0) { + const { error } = computeSomething(); + if (error) { doThing(); } + } + `) + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if reading a destructured parameter inside the if's own condition", () => { + const ep = scanFile( + "x.ts", + swallow("if (items.some(({ error }) => error > 0)) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if shadowed by an array-destructured declaration", () => { + const ep = scanFile( + "x.ts", + swallow("const [error] = getErrors();\n if (error) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // Positive controls: a genuine reference to the real binding must still be credited, including + // through an enclosing loop whose OWN variable has a different name. + it("still credits an if that genuinely reads the outer binding directly", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("still credits an if inside a for...of loop with a different loop variable", () => { + const ep = scanFile( + "x.ts", + swallow("for (const item of items) { if (error.code === item) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); From e2e9ab38ea84ac7a8008acd7ef22f567f4373fc4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 18:07:58 +0100 Subject: [PATCH 050/117] fix(observability-map): drop the parse-guard ratio for an absolute, unpaddable count I6: moving the denominator from the entry point to the enclosing function (A6) closed cross-body dilution but the rule was still a ratio, "unrelated statements dilute", wherever the padding lives. Reproduced: padding the SAME action with 11 inert const statements after the try relabelled the identical broad otel.v1.logs.ts-shaped swallow from fail to pass, with zero change to the clause itself. Went back to the absolute rule this codebase already had and hand-read once, before the ratio replaced it: NARROW_TRY_STATEMENTS = 2, removed earlier only because it had become dead code once isParseGuard switched to the ratio, never because the count itself was found wrong. isParseGuard is now guardsParse && tryStatementCount <= 2, over the try block alone. Nothing outside the try can move that number, in the same body or another. Removed CatchEvidence.enclosingStatementCount, dead now that nothing reads it. Measured on the real tree: 7 clauses stop counting as a narrow guard; 2 of those move the overall verdict (5 have other evidence keeping the route's verdict where it was). Both hand-read: bulk-actions/route.tsx pass to not-applicable, a 9-statement catch that does a full presenter call and RBAC check was wrongly excused as a narrow guard and only rethrows, so it is correctly inert now, not credited; stream-basin.ts pass to fail, a 6-statement zod-validation catch that swallows a parse failure with no classification is correctly a swallow, not a guard. Global unchanged at 15. --- .../src/checks/errorClassification.ts | 31 ++++--- .../observability-map/src/scan.ts | 4 +- .../observability-map/src/types.ts | 8 -- .../observability-map/test/checks.test.ts | 91 +++++++++++++++++++ .../observability-map/test/scan.test.ts | 2 - 5 files changed, 112 insertions(+), 24 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 7d5f24f5dc0..6bdfc661474 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -26,20 +26,29 @@ const BUILDERS = new Set([ ]); /** - * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * covers less than half the statements of the body it is actually in. Both halves matter. - * `guardsParse` alone lets `otel.v1.logs.ts` off, whose catch covers 15 of its 18 statements and - * merely happens to contain a `request.json()`, and that is a real swallow. The coverage test is - * relative to the enclosing body rather than a second absolute threshold, so it holds for a - * three-statement route and a fifty-statement one alike. + * How much a try block may guard and still count as narrow. Two, so the guarded operation can bind + * its result (`const stripped = ...; new RegExp(stripped);`), but a third statement means the try + * has started to cover the handler rather than one operation. The idiom this was chosen for and + * hand-read against originally: 55 of 427 entry points, 11 of the failures at the time, all eleven + * the deliberate `try { body = await request.json() } catch { 400 }` shape. * - * Compared against `clause.enclosingStatementCount`, never `ep.statementCount`: the entry point's - * total sums the loader, the action and every one-hop helper together, so an unrelated sibling - * handler or a fat helper in the same file diluted the denominator and relabelled a broad swallow - * as a narrow parse guard, with no change to the clause itself. + * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in + * the same body: padding the action with unrelated statements after the try relabelled the same + * broad swallow as a narrow guard, moving the denominator without touching the clause at all. An + * absolute count over the try block alone cannot be diluted by anything outside it. + */ +const NARROW_TRY_STATEMENTS = 2; + +/** + * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and + * holds at most `NARROW_TRY_STATEMENTS`. Both halves matter. `guardsParse` alone lets + * `otel.v1.logs.ts` off, whose catch covers 7 statements and merely happens to contain a + * `request.json()`, and that is a real swallow; a validating zod `.safeParse` followed by an issue + * check and a bespoke error response is real handling too, not a bind-and-return guard, and stays + * excluded at three statements or more for the same reason. */ function isParseGuard(clause: CatchEvidence): boolean { - return clause.guardsParse && clause.tryStatementCount * 2 < clause.enclosingStatementCount; + return clause.guardsParse && clause.tryStatementCount <= NARROW_TRY_STATEMENTS; } /** diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index d8b10b5da55..77246ca082a 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -754,8 +754,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const helpers: EntryFunction[] = []; const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - const enclosingStatementCount = countFunctionStatements(fn); - statementCount += enclosingStatementCount; + statementCount += countFunctionStatements(fn); if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback @@ -786,7 +785,6 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { branches: clause.branches, guardsParse: guardsParse(node.tryBlock), tryStatementCount, - enclosingStatementCount, }); } } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index bf6e6a79022..a1d3be0b2db 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -32,14 +32,6 @@ export type CatchEvidence = { guardsParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; - /** - * Statements in the body the clause actually sits in: the loader, the action, or the one-hop - * helper it delegates to, whichever function this `try` is directly inside. Never the entry - * point's combined total, which sums the loader, the action and every helper together: comparing - * a clause against that whole-entry-point figure let an unrelated sibling handler or a fat helper - * in the same file relabel a broad swallow as a narrow parse guard. - */ - enclosingStatementCount: number; }; /** A logging call made from a loader/action body, or from a same-file helper the body calls. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 11c606639e2..3187c916b6c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -156,6 +156,48 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // I6. NARROW_TRY_STATEMENTS is an absolute count over the try block alone (guardsParse still + // required), not a ratio against the enclosing body, so it holds the exact boundary regardless of + // how big or small the rest of the function is: two statements binds the parsed result and still + // passes, a third means the try has started to cover the handler and fails, even though both + // guard the same parse. + it("passes a parse guard that binds its result in exactly two statements", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + parsed = new RegExp(raw); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a parse guard that takes a third statement beyond binding the result", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + const trimmed = raw.trim(); + parsed = new RegExp(trimmed); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("fail"); + }); + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still // counts against the entry point. @@ -307,6 +349,55 @@ describe("error-classification", () => { expect(withSiblingAndHelper.status).toBe("fail"); }); + // I6. Moving the denominator from the entry point to the enclosing body (A6) closed + // cross-body dilution but not same-body dilution: the rule was still a ratio, "unrelated + // statements dilute", wherever the unrelated statements live. Padding the SAME action with 11 + // inert statements after the try relabelled the identical broad swallow from fail to pass. + // isParseGuard is now an absolute count over the try block alone (NARROW_TRY_STATEMENTS), + // which nothing outside the try can dilute, in the same body or another. + it("gives the same verdict to a byte-identical swallow whether or not it is padded with inert statements in the same body", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const padding = Array.from({ length: 11 }, (_, i) => `const pad${i} = ${i};`).join("\n"); + const paddedInSameBody = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + ${padding} + return new Response("unreachable", { status: 200 }); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const padded = run("error-classification", "otel.v1.logs.ts", paddedInSameBody); + + expect(alone.status).toBe("fail"); + expect(padded.status).toBe("fail"); + }); + // A3. `referencesBinding` used to match any identifier with the binding's text, including a // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the // caught binding itself, was credited with classifying an error it never inspected. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 92f6a093bdb..4d1c56fb917 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1078,7 +1078,6 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, tryStatementCount: 1, - enclosingStatementCount: 10, }); expect(ep!.catches[1]).toMatchObject({ guardsParse: false, @@ -1180,7 +1179,6 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, tryStatementCount: 4, - enclosingStatementCount: 6, }); }); From 4e4dd812d6a01d047b7982f6b22bd4f218d54756 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 18:15:46 +0100 Subject: [PATCH 051/117] fix(observability-map): tighten the minors (level set, bare env, a dead test, stale figures) M7: the qualifying level set claimed fatal and implicitly trace, neither of which the real logger (packages/core/src/logger.ts) has. It has log, error, warn, info, debug, verbose, and log is level 0, the one TRIGGER_LOG_LEVEL never filters out, so it belongs in the set and was wrongly excluded. QUALIFYING_LEVELS is now { log, error, warn }, correcting the docstring against the actual level list rather than an assumed one. M8: TENANT_FIELD accepted a bare env, so { env: process.env.NODE_ENV } would have read as naming a tenant. env and org now require a real suffix (envId, orgSlug); the full words (environment, organization, project, user) still qualify bare, since those are unambiguous on their own. M9: deleted a test asserting suppressed.score <= plain.score for a suppressed passing check. Removing a passing (maximal) result from a ratio can only lower or hold it, at the entry level and the aggregate level, with or without A1's fix, so the assertion passed unchanged against the pre-fix code and could not discriminate anything. The two tests beside it already exercise "does not raise the score" with real discriminating power, on the failing-check direction that actually distinguishes old code from new. M10: score.ts had a third stale "391 of 412" request-context figure (MapReport.contextGap's docstring) missed when A1's terminal.ts and report.test.ts occurrences were corrected; now 401, matching every other copy in the package. Zero real-tree effect from M7/M8, confirmed directly. Full suite green. --- .../src/checks/requestContext.ts | 29 +++++--- .../observability-map/src/score.ts | 2 +- .../observability-map/test/checks.test.ts | 74 +++++++++++++++++++ .../observability-map/test/score.test.ts | 27 ++----- 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index f2d37aa600e..f54dc8d5f9e 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -6,23 +6,30 @@ const ID = "request-context"; /** * A field name that plausibly names a TENANT: environment, organization, project or user, the four * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, - * in the full and abbreviated camelCase the webapp actually writes for each: - * `environmentId`/`envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, - * `userId`. A bare `id`, and a resource id that happens to share the same `Id`/`Param` suffix, - * `batchId`, `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not - * qualify: those name a resource the failure touched, not who it happened to. + * in the full and abbreviated camelCase the webapp actually writes for each: `environmentId`/ + * `envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, `userId`. A bare + * `id`, and a resource id that happens to share the same `Id`/`Param` suffix, `batchId`, + * `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not qualify: + * those name a resource the failure touched, not who it happened to. + * + * The abbreviated roots, `env` and `org`, require a suffix; the full words do not. A bare `env` is + * ambiguous with a deployment environment name (`{ env: process.env.NODE_ENV }`), which is not a + * tenant, and nothing in the tree relies on it being bare, so the field alone cannot qualify. */ const TENANT_FIELD = - /^(environment|env|organization|org|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$/; + /^(environment|organization|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$|^(env|org)(Id|Ids|Slug|Ref|Param|Identifier)$/; /** - * error, warn and fatal are levels an incident is actually read at; debug and trace are routinely - * dropped or sampled out before anyone looks, so a tenant field logged only at that level is not - * really recorded on the failure path. info is deliberately excluded too: it is not reserved for + * Levels against the real logger (`packages/core/src/logger.ts`): `log`, `error`, `warn`, `info`, + * `debug`, `verbose`, in that order, no `fatal` and no `trace` (the `trace` in + * `apps/webapp/app/services/logger.server.ts` is the AsyncLocalStorage field helper, unrelated to + * log level). `log` is level 0, the level `TRIGGER_LOG_LEVEL` never filters out, so it qualifies + * alongside `error` and `warn`. `info`, `debug` and `verbose` do not: `info` is not reserved for * failure reporting, so a route can log an info line inside a catch that says nothing about the - * catch actually handling anything, and crediting it would launder the same gap `debug` closes. + * catch actually handling anything, and `debug`/`verbose` are routinely dropped or sampled out + * before anyone reads an incident. */ -const QUALIFYING_LEVELS = new Set(["error", "warn", "fatal"]); +const QUALIFYING_LEVELS = new Set(["log", "error", "warn"]); /** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ function logLevel(callee: string): string { diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 7a5c6924bda..420a07b19f4 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -47,7 +47,7 @@ export type MapReport = { sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; /** - * `request-context` fails 391 of the 412 entry points it applies to, so it is reported as a + * `request-context` fails 401 of the 412 entry points it applies to, so it is reported as a * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` * gets. It stays fully in the score: the gap is real and the score is meant to show it. */ diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 3187c916b6c..5a73eca50e5 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -989,6 +989,80 @@ describe("request-context", () => { ); expect(r.status).toBe("pass"); }); + + // M7. The real logger (packages/core/src/logger.ts) has log/error/warn/info/debug/verbose, no + // fatal and no trace, and log is level 0, never filtered by TRIGGER_LOG_LEVEL, so it must + // qualify. verbose is the actual noisiest level this codebase has, not trace. + it("passes a log-level failure log, which the real logger never filters", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.log("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass a verbose-level failure log", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.verbose("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // M8. A bare `env` field is ambiguous with a deployment environment name + // (`{ env: process.env.NODE_ENV }`), not a tenant, so it must not qualify on its own; the + // abbreviated root still works with a real suffix. + it("does not pass a failure log that only names a bare env field", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { env: process.env.NODE_ENV, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes envId, the abbreviated root with a real suffix", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); }); describe("audit-trail", () => { diff --git a/internal-packages/observability-map/test/score.test.ts b/internal-packages/observability-map/test/score.test.ts index c64c75bbed8..d0877736af1 100644 --- a/internal-packages/observability-map/test/score.test.ts +++ b/internal-packages/observability-map/test/score.test.ts @@ -321,26 +321,13 @@ ${BUSY_AND_FAILING}` expect(after.measured).toBe(2); }); - it("suppressing a passing check does not raise the score either", () => { - // error-classification branches on the error (pass); request-context never names a tenant - // (fail). Not sensitive, so auth-boundary sits out. - const source = `import { prisma } from "~/db.server"; -export async function loader() { - try { return await prisma.thing.findMany(); } - catch (e) { if (e instanceof Error) return null; throw e; } -}`; - const plain = scoreEntry(scanFile("api.v1.mixed.ts", source)!); - expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); - - const suppressed = scoreEntry( - scanFile( - "api.v1.mixed.ts", - `// obs-map-disable error-classification -- silence a pass -${source}` - )! - ); - expect(suppressed.score).toBeLessThanOrEqual(plain.score); - }); + // M9. A prior version of this test asserted suppressed.score <= plain.score for a check that was + // passing, which the pre-existing per-entry Math.min cap already guarantees on its own: removing + // a passing (maximal) result from a ratio can only lower or hold it, at every level, with or + // without A1's fix, so the assertion passed unchanged against the pre-fix code and proved + // nothing about the aggregate mechanism A1 actually changed. Deleted rather than kept as + // decoration; "does not raise the score" for a failing suppression is exercised, with real + // discriminating power, by the two tests above. }); /** From 28c034db2161eb4efaadfa9173829448aadca1a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:42:24 +0100 Subject: [PATCH 052/117] fix(observability-map): read a directive off a comment, not off JSX text getLeadingCommentRanges and getTrailingCommentRanges are raw lexers over source text from an offset and never consult the parse tree, so at a leaf token boundary they read the start of a JSX text node as a comment whenever it begins with // or /*. Three shapes suppressed a check that way, and the tree already contains one: resources.branches.create.tsx's // was read as a comment. A range whose start falls inside a JsxText, string literal, template segment or regex literal is now dropped. Measured over the whole route tree: 3344 comment ranges across 480 files, exactly one removed, the known bad one. A [fullStart, getStart) gap filter was tried instead and rejected, it loses same-line trailing comments and JSX expression containers. The docstring said a JSX text node is 'never trivia' to these two functions, which was false. It now says what the code does. --- .../observability-map/src/suppression.ts | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 4740594746d..26a51e13085 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -28,31 +28,62 @@ function leafTokens(node: ts.Node): ts.Node[] { } /** - * Every genuine comment range in the source, read off a real parsed `ts.SourceFile` rather than a - * standalone `ts.createScanner`. A bare scanner has no parser state behind it, and that is exactly - * what let two shapes through: + * Node kinds whose text the parser has already claimed as content, so nothing inside their span can + * be trivia however it is spelled. `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw + * lexers over source text from an offset and consult no parse tree at all, so at a leaf-token + * boundary they will happily lex the inside of one of these as a comment: a JSX text node that + * BEGINS with `//` or `/*` is the shape that reached the real tree, in + * `resources.branches.create.tsx`'s `//`. * - * - a template literal WITH a substitution never gets rescanned as a template continuation by a - * scanner running on its own, so the text after `${x}` reads as ordinary code and a `//` in it is - * a real comment to the scanner, though it never leaves the template literal to the parser. - * - JSX text has no comment syntax at all, but a scanner created in `LanguageVariant.Standard` - * does not know it is looking at JSX text, so a `//` inside `

see https://x

` reads as a - * line comment starting mid-URL. + * `content-is-not-a-comment` in `test/suppression.test.ts` covers each kind, and + * `jsx-text-line-directive`, `jsx-text-after-expression` and `jsx-text-block-directive` in the + * mutation corpus cover the JSX shapes over the whole route tree. + */ +function isClaimedContent(node: ts.Node): boolean { + return ( + ts.isJsxText(node) || + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + ts.isTemplateHead(node) || + ts.isTemplateMiddle(node) || + ts.isTemplateTail(node) || + ts.isRegularExpressionLiteral(node) + ); +} + +/** + * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a + * standalone `ts.createScanner`, and then filtered against the spans above. * - * The actual parser closes both: a template's literal segments and a JSX text node are real nodes - * with their own span here, never trivia, so a directive inside either is content the parser - * already claimed, not a comment. `getLeadingCommentRanges` and `getTrailingCommentRanges` are both - * needed at every token boundary, because which one returns a given comment depends on whether it - * shares a line with the token before it (trailing) or comes after a line break (leading), not on - * which directive it happens to be. + * Both halves are needed. Parsing rather than scanning is what stops a template literal WITH a + * substitution being rescanned as ordinary code after `${x}`, and what makes JSX text a node at all. + * Filtering by span is what stops the two comment-range lexers reading the start of such a node as + * a comment anyway, which they do because they never see the tree the parser built. + * + * The filter is on the range's start offset falling inside a claimed span, not on the gap between a + * token's full start and its start. A gap filter was tried and rejected: it loses a same-line + * trailing comment and a comment inside a JSX expression container, both of which are real. + * + * Both lexers are called at every token boundary, because which one returns a given comment depends + * on whether it shares a line with the token before it (trailing) or comes after a line break + * (leading), not on which directive it happens to be. */ function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const claimed: ts.TextRange[] = []; + const collectClaimed = (node: ts.Node) => { + if (isClaimedContent(node)) claimed.push({ pos: node.getStart(sf), end: node.end }); + ts.forEachChild(node, collectClaimed); + }; + collectClaimed(sf); + const inClaimedSpan = (pos: number) => claimed.some((s) => pos >= s.pos && pos < s.end); + const seen = new Set(); const ranges: ts.CommentRange[] = []; const add = (found: ts.CommentRange[] | undefined) => { for (const range of found ?? []) { if (seen.has(range.pos)) continue; seen.add(range.pos); + if (inClaimedSpan(range.pos)) continue; ranges.push(range); } }; From 68b838061606844a6a3a94aeb8f59387bc5597b9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:42:53 +0100 Subject: [PATCH 053/117] fix(observability-map): judge a catch on its own path, and stop excusing a hidden one Four changes to the catch evidence, each measured against the whole route tree. Global 15 before, 15 after; no route's score rose. rethrows and branches are now read off the clause's straight-line path: its statements, cut at the first definite exit, recursing into a bare block and a do body and nothing else. That replaces the list of statically-false shapes the previous round kept extending. The list recognised if (false) and while (false) and lost to for (;false;), if (true) else, switch (1) { case 2 }, try {} catch, for (const x of []), for (const k in {}), if (""), if (!true) and if (1 === 2), each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without naming any. A branch also has to send an arm somewhere the others do not go, by returning or throwing. if (e instanceof Error) { } was the cheapest no-op in the tool. The cost is three routes that decorate a Response and rethrow, which now read as inert rather than deciding: auth.github.ts, auth.google.ts and login.magic/route.tsx. A catch the boundary rule refuses is counted rather than dropped, and a route whose only catches were refused now fails instead of sitting out. Reading that as not-applicable was worth 50 points to anything that could get the rule to refuse the route's real catch, which Promise.all([0].map(async () => { whole body })) did. A single-element array literal receiver is also no longer read as iteration. Two routes move to fail, both genuine silent swallows: _app.orgs.$organizationSlug/route.tsx ignores JSON parse failures and api.v1.query.dashboards._index.ts drops unserializable dashboards. Statements inside inline callbacks now count towards statementCount, so wrapping a body in trace("x", async () => { ... }) no longer collapses the route to one statement and takes it out of the report as trivial. The parse-guard rule gains awaitsOnlyParse and countStatement counts declarators and comma operands rather than semicolons. Both close ways to make a broad handler look like a narrow guard: merging a seven-statement try into one declaration list, and writing the handler inside the try next to the parse. The design's suggestion of requiring a 4xx was measured first and is not used, it credits the widest swallows in the tree on its own and costs three real guards on top of the other conditions. No route moved from either change. --- .../src/checks/errorClassification.ts | 60 ++- .../observability-map/src/scan.ts | 343 +++++++++++------- .../observability-map/src/types.ts | 41 ++- .../observability-map/test/checks.test.ts | 178 ++++++++- .../observability-map/test/scan.test.ts | 66 +++- 5 files changed, 519 insertions(+), 169 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 6bdfc661474..b377e577cf2 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -34,21 +34,49 @@ const BUILDERS = new Set([ * * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in * the same body: padding the action with unrelated statements after the try relabelled the same - * broad swallow as a narrow guard, moving the denominator without touching the clause at all. An - * absolute count over the try block alone cannot be diluted by anything outside it. + * broad swallow as a narrow guard, moving the denominator without touching the clause at all. + * `inert-statements-after-try` in the mutation corpus is that shape, and it holds. + * + * What the count is NOT is unpaddable, which an earlier docstring and commit subject both claimed. + * `countStatement` now counts declarators and comma operands rather than semicolons, so the two + * known ways to pack a try into fewer statements move the number the same as writing it out; that + * is what `merge-declarations` and `merge-comma-expressions` in the corpus check. A third + * way nobody has written down would work, which is why the count is no longer the only condition + * and no longer the load-bearing one. */ const NARROW_TRY_STATEMENTS = 2; /** - * Whether a catch clause is a guard rather than the route's error handling: it wraps a parse and - * holds at most `NARROW_TRY_STATEMENTS`. Both halves matter. `guardsParse` alone lets - * `otel.v1.logs.ts` off, whose catch covers 7 statements and merely happens to contain a - * `request.json()`, and that is a real swallow; a validating zod `.safeParse` followed by an issue - * check and a bespoke error response is real handling too, not a bind-and-return guard, and stays - * excluded at three statements or more for the same reason. + * Whether a catch clause is a guard rather than the route's error handling: the try block parses, + * waits for nothing except that parse, and is short. + * + * `awaitsOnlyParse` is the condition the previous wave was missing, and it is the one a statement + * count cannot express. `try { const body = await request.json(); return await handleEverything(body); } + * catch { return 500; }` is two statements, one of them a parse, and the whole handler inside it: + * the count reads it as narrow and it is the `otel.v1.logs.ts` swallow written compactly. Asking + * what the block waits for separates them, and unlike the count it does not care how the statements + * are punctuated or how deeply the work is nested inside one of them. + * + * The design's own suggestion, requiring the clause to answer with a 4xx, was measured first and is + * not used. On its own it credits 11 clauses guarding four to thirty statements, the widest swallows + * in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try answers every failure + * with a 400 carrying the internal error message. Added on top it costs three routes their pass, + * all three narrow parse guards that compute a fallback value rather than answering a request + * (`try { return new URL(referer).origin; } catch { return undefined; }`), and it buys only the case + * of a narrow parse guard answering 500. Requiring every CALL to be a parse, rather than every + * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new + * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. + * + * The residual, since awaiting is the signal: a try block that does its non-parse work + * synchronously still reads as a guard. Nothing in the tree does, and it is written down in the + * round A fix 2 report rather than defended. */ function isParseGuard(clause: CatchEvidence): boolean { - return clause.guardsParse && clause.tryStatementCount <= NARROW_TRY_STATEMENTS; + return ( + clause.guardsParse && + clause.awaitsOnlyParse && + clause.tryStatementCount <= NARROW_TRY_STATEMENTS + ); } /** @@ -113,6 +141,13 @@ export function usesBuilder(ep: EntryPoint): boolean { * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error * propagates once the cleanup has run, and reading the old flag as a catch put * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. + * + * `callbackCatches` is the third case, and it is what stops "no catch is not-applicable" from being + * a payout. A route whose catches all sat inside a callback the scanner refused to attribute has + * error handling, the scanner just could not read it as the route's; excusing that is worth 50 + * points to anyone who wraps a body in something the boundary rule refuses, which + * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route + * that genuinely only handles errors per item, which now fails rather than sitting out. */ export const errorClassification = { id: ID, @@ -130,6 +165,13 @@ export const errorClassification = { detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } + if (ep.catches.length === 0 && ep.callbackCatches > 0) { + return { + id: ID, + status: "fail", + detail: "its only error handling sits in a callback the route does not own", + }; + } if (!ep.catches.some(decides)) { return { id: ID, diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 77246ca082a..da46cb4e54f 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -121,29 +121,64 @@ const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$| */ const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); +function isParseCall(node: ts.Node): boolean { + if (ts.isNewExpression(node)) { + return ts.isIdentifier(node.expression) && PARSE_CONSTRUCTORS.has(node.expression.text); + } + if (!ts.isCallExpression(node)) return false; + const text = calleeText(node.expression) ?? calleeName(node.expression); + return text !== null && PARSE_CALLEE.test(text); +} + /** - * Whether the guarded region parses something. A `new URL(x)` counts, and has to be read here as a - * `ts.isNewExpression`, because the call-callee scan that builds `calleeNames` never sees it. + * Body reads: the thing a parse guard waits for before it parses. `request.json()` is in + * `PARSE_CALLEE` already because it reads and parses in one call, and these are the same operation + * with the parse written separately, `const raw = await request.text(); new RegExp(raw);`. + * + * Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which is what bounds it: a body + * read on its own still does not make a try block a parse guard, so the widest this list can do is + * let a block that already parses also read the thing it parses. */ -function guardsParse(tryBlock: ts.Block): boolean { - let found = false; +const BODY_READ_METHODS = new Set(["text", "formData", "arrayBuffer", "blob", "bytes"]); + +function isBodyRead(node: ts.Node): boolean { + if (!ts.isCallExpression(node)) return false; + const callee = unwrap(node.expression); + return ts.isPropertyAccessExpression(callee) && BODY_READ_METHODS.has(callee.name.text); +} + +/** + * What the guarded region does, in the two terms `error-classification` needs to tell a parse guard + * from a handler wrapped around a parse. + * + * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read + * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees + * it. + * + * `awaitsOnlyParse` is whether everything the block waits for is a parse or a read of the body it + * parses. Awaiting is the signal, not calling: the calls that prepare a parse's input are ordinary + * synchronous string work (`matchPattern.startsWith("(?i)")`, `.slice(4)` before a `new RegExp`), + * and refusing those refuses four of the tree's clearest guards, while the swallow this has to + * catch reaches a service: `try { const body = await request.json(); return await + * handleEverything(body); }`. + * + * Nested function bodies are skipped: a callback written inside the try is not work the try is + * guarding on this pass through. + */ +function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyParse: boolean } { + let guardsParse = false; + let awaitsOnlyParse = true; const visit = (node: ts.Node) => { - if (found) return; - if ( - ts.isNewExpression(node) && - ts.isIdentifier(node.expression) && - PARSE_CONSTRUCTORS.has(node.expression.text) - ) { - found = true; - } - if (ts.isCallExpression(node)) { - const text = calleeText(node.expression) ?? calleeName(node.expression); - if (text !== null && PARSE_CALLEE.test(text)) found = true; + if (ts.isFunctionLike(node)) return; + if (isParseCall(node)) guardsParse = true; + if (ts.isAwaitExpression(node)) { + const awaited = unwrap(node.expression); + if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; } ts.forEachChild(node, visit); }; visit(tryBlock); - return found; + return { guardsParse, awaitsOnlyParse }; } /** Whether some node in the tree rooted at `node` matches `predicate`. */ @@ -191,20 +226,6 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo return false; } -/** Whether a `for`/`for...of`/`for...in` loop's own declared variable is `name`, so its body - * shadows the rest of the enclosing scope for that name. */ -function declaresLoopVariable( - node: ts.ForStatement | ts.ForOfStatement | ts.ForInStatement, - name: string -): boolean { - const initializer = node.initializer; - return ( - initializer !== undefined && - ts.isVariableDeclarationList(initializer) && - initializer.declarations.some((d) => bindingDeclares(d.name, name)) - ); -} - /** * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the @@ -248,8 +269,8 @@ function catchBindingName(clause: ts.CatchClause): string | null { /** * Whether a conditional expression tests the error to pick what the clause does, rather than to - * word what it says. It counts only when the whole `return`/`throw` is the conditional, so - * `return e instanceof Response ? e : json({}, { status: 500 })` counts and + * word what it says. The caller only offers it the whole value of a `return`/`throw`, so + * `return e instanceof Response ? e : json({}, { status: 500 })` reaches here and * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. * The second is message formatting: every error leaves by the same path. * @@ -260,9 +281,7 @@ function catchBindingName(clause: ts.CatchClause): string | null { function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; - if (!referencesBinding(node.condition, bindingName)) return false; - const parent = node.parent; - return parent !== undefined && (ts.isReturnStatement(parent) || ts.isThrowStatement(parent)); + return referencesBinding(node.condition, bindingName); } /** A statement that unconditionally leaves the statement list it sits in, so anything after it in @@ -288,110 +307,119 @@ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.S return index === -1 ? statements : statements.slice(0, index + 1); } +/** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one + * inside a nested function. What separates an arm that takes the error somewhere from an arm that + * runs and falls back into the clause's single common exit. */ +function containsExit(node: ts.Node): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isReturnStatement(node) || ts.isThrowStatement(node)) return true; + return ts.forEachChild(node, containsExit) === true; +} + /** - * A literal that is always falsy: `false`, `null`, `undefined`, `0`. Not general constant folding, - * on purpose: `!true`, `1 === 2` and a reference to a `const` declared elsewhere are not covered, so - * `if (false) { throw e; }` and `while (false) { throw e; }` are recognised as dead and nothing more - * elaborate is claimed to be. + * Whether an `if`/`switch` sends at least one arm somewhere the others do not go, by returning or + * throwing from inside it. `if (e instanceof Error) { }` and `if (e instanceof Error) { log(e); }` + * both fail this: every error still leaves the clause by the same path afterwards, so the test + * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, + * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. + * + * Two arms that return the SAME value pass this and should not. That residual is written down in + * the round A fix 2 report rather than defended: telling two returns apart needs the values + * compared, which is a different kind of analysis from anything else here. */ -function isFalsyLiteral(expr: ts.Expression): boolean { - if (expr.kind === ts.SyntaxKind.FalseKeyword || expr.kind === ts.SyntaxKind.NullKeyword) { - return true; +function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { + if (ts.isIfStatement(statement)) { + return ( + containsExit(statement.thenStatement) || + (statement.elseStatement !== undefined && containsExit(statement.elseStatement)) + ); } - if (ts.isIdentifier(expr) && expr.text === "undefined") return true; - return ts.isNumericLiteral(expr) && expr.text === "0"; + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); } /** * What a catch clause does with the error, beyond the fact that it caught one. * - * Stops at every function-like node, not only an iteration callback (contrast `walkBody`'s - * boundary, which lets a route's own single-shot wrapper, `trace(async () => {...})`, - * `mutateWithFallback({ pgMutation })`, `new ReadableStream({ start })`, through so the route's own - * catch is found at all). Here the walk is already inside a catch clause that `walkBody` decided - * belongs to the route; anything the clause does by constructing a further callback, - * `queue.push(() => { throw e; })`, a `.then`, a `setTimeout`, is deferred work the clause merely - * registers, not a decision it makes on its own execution. Both walks refuse a per-item iteration - * callback; this one refuses every other kind of callback too, for that reason. + * Both answers are read off the clause's own straight-line path: the statements of its block, cut + * at the first one that definitely exits, recursing into a bare nested block and nothing else. A + * `throw` or a test that sits inside an `if`, a loop, a `switch`, a nested `try` or a callback is + * not on that path, so it does not count. + * + * That is the whole dead-code defence, and it replaces the list of statically-false shapes the + * previous round kept extending. The list was losing: `if (false)` and `while (false)` were + * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, + * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, + * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without + * naming any of them, and refuses the twelfth nobody has written yet. `dead-*` in the mutation + * corpus is the tree-scale proof, one entry per shape. + * + * The cost is real: `catch (e) { if (transient) throw e; return null; }` no longer reads as a + * rethrow, so it reads as a swallow and fails rather than sitting out. That is the direction to be + * wrong in, since the reverse hands out points. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; let branches = false; const bindingName = catchBindingName(clause); - // `shadowed` is true once the walk has passed a scope that re-declares `bindingName`: a nested - // catch clause with the same name, a `for`/`for...of`/`for...in` loop declaring it, or a block - // that does (reusing the same `declaresInScope` a bare block already checks). From there on, an - // `if`/`switch` that references the name textually is referencing the SHADOWING declaration, not - // this clause's own binding, so it must not count as this clause deciding anything. Never reset - // back to false, the same rule `inCallback` follows: once shadowed, everything nested inside is - // still shadowed. - const visit = (node: ts.Node, shadowed: boolean) => { - if (ts.isFunctionLike(node)) return; - - if (bindingName !== null && ts.isCatchClause(node)) { - const decl = node.variableDeclaration; - const shadowsHere = decl !== undefined && bindingDeclares(decl.name, bindingName); - ts.forEachChild(node, (child) => visit(child, shadowed || shadowsHere)); - return; - } - - if ( - bindingName !== null && - (ts.isForStatement(node) || ts.isForOfStatement(node) || ts.isForInStatement(node)) && - declaresLoopVariable(node, bindingName) - ) { - visit(node.statement, true); - return; - } - - if (ts.isBlock(node) || ts.isCaseClause(node) || ts.isDefaultClause(node)) { - const shadowedHere = - shadowed || (bindingName !== null && declaresInScope(node.statements, bindingName)); - for (const statement of reachableStatements(node.statements)) visit(statement, shadowedHere); - return; - } - - if (ts.isThrowStatement(node)) rethrows = true; - if ( - !shadowed && - bindingName !== null && - ((ts.isIfStatement(node) && referencesBinding(node.expression, bindingName)) || - (ts.isSwitchStatement(node) && referencesBinding(node.expression, bindingName))) - ) { - branches = true; - } - if (!shadowed && ts.isConditionalExpression(node) && selectsAnErrorPath(node, bindingName)) { - branches = true; - } + const walk = (statements: readonly ts.Statement[]) => { + // A block that re-declares the binding name means an `if` below it referencing that name is + // referencing the shadowing declaration, not this clause's error. Nothing in such a block can + // speak for the clause, so the whole list is skipped for branch purposes. + const shadowed = bindingName !== null && declaresInScope(statements, bindingName); - // A statically-false condition makes the guarded statement dead code: it can set `branches` - // above (the clause still decided to test the error, even if the arm never runs), but nothing - // inside it can set `rethrows` or a nested `branches`, so it is not visited at all. - if (ts.isIfStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.thenStatement, shadowed); - if (node.elseStatement) visit(node.elseStatement, shadowed); - return; - } - if (ts.isWhileStatement(node)) { - if (!isFalsyLiteral(node.expression)) visit(node.statement, shadowed); - return; + for (const statement of reachableStatements(statements)) { + if (ts.isThrowStatement(statement)) { + rethrows = true; + continue; + } + if (ts.isBlock(statement)) { + walk(statement.statements); + continue; + } + // A `do` body runs before its condition is ever read, so it is on the straight-line path + // whatever the condition says. The only loop form that is. + if (ts.isDoStatement(statement)) { + const body = statement.statement; + walk(ts.isBlock(body) ? body.statements : [body]); + continue; + } + if (bindingName === null || shadowed) continue; + + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { + branches = true; + continue; + } + if ( + (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && + statement.expression !== undefined + ) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + branches = true; + } + } } - - ts.forEachChild(node, (child) => visit(child, shadowed)); }; - visit(clause.block, false); + walk(clause.block.statements); return { rethrows, branches }; } /** - * Array methods that invoke their callback once per element, never once as a whole. The one - * structural signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a - * fresh catch for every element) from a route's own body expressed through one more layer of - * function nesting (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => - * {...} })`, `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their - * callback exactly once, as the route's own continuation. + * Method names that invoke their callback once per element, never once as a whole. The structural + * signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a fresh catch + * for every element) from a route's own body expressed through one more layer of function nesting + * (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })`, + * `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their callback exactly + * once. + * + * A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. The + * consequence is written down where it matters, on `isIterationCallback`. */ const ITERATION_METHODS = new Set([ "map", @@ -404,14 +432,39 @@ const ITERATION_METHODS = new Set([ "every", ]); -/** Whether the function-like `node` is the callback argument of a call to one of - * `ITERATION_METHODS`, e.g. the arrow function in `items.map((item) => ...)`. */ +/** Whether an expression is an array literal of fewer than two elements, the one receiver shape + * that cannot be a per-item iteration however the method is named. */ +function isAtMostSingletonArray(expr: ts.Expression): boolean { + const target = unwrap(expr); + return ts.isArrayLiteralExpression(target) && target.elements.length < 2; +} + +/** + * Whether the function-like `node` is the callback argument of a per-item iteration, e.g. the arrow + * function in `items.map((item) => ...)`. + * + * Being wrong here is asymmetric. Calling a per-item callback the route's own continuation + * mis-attributes a per-element catch to the route, which was the bug the boundary was added for. + * Calling the route's own continuation a per-item callback hides the route's catch, and + * `error-classification` used to read a route with no catch as not-applicable, which is 50 points + * more than the swallow it was hiding. So the second direction paid, and `[0].map(async () => { + * whole body })` collected it. + * + * Two things changed. A receiver that is an array literal of one element or none is refused here, + * because it cannot iterate. And the direction that pays no longer pays: `walkBody` counts the + * catches it refuses, and `error-classification` fails a route whose only catches were refused + * rather than excusing it. So a wrong answer here costs precision, not points. That is what makes + * the name list survivable, and it is why `Result.map(...)`, which no name list can tell from + * `users.map(...)`, is a corpus entry that passes rather than a hole. + */ function isIterationCallback(node: ts.Node): boolean { const parent = node.parent; if (!parent || !ts.isCallExpression(parent)) return false; if (!parent.arguments.includes(node as ts.Expression)) return false; const callee = unwrap(parent.expression); - return ts.isPropertyAccessExpression(callee) && ITERATION_METHODS.has(callee.name.text); + if (!ts.isPropertyAccessExpression(callee)) return false; + if (!ITERATION_METHODS.has(callee.name.text)) return false; + return !isAtMostSingletonArray(callee.expression); } const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); @@ -529,15 +582,42 @@ function resolveLocal(name: string, locals: LocalDeclarations, seen: Set return analyzeInitializer(local, locals, seen); } +/** + * How many operands a comma expression has, so `a(), b(), c()` is three and not one. Anything else + * is one. + */ +function commaOperands(expr: ts.Expression): number { + const target = unwrap(expr); + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.CommaToken) { + return commaOperands(target.left) + commaOperands(target.right); + } + return 1; +} + /** * Statements in a statement, counting through block-bearing statements so a body wrapped in a * single `try` reports its real size. Does not descend into nested function bodies. + * + * Counts bindings and comma operands rather than semicolons, which is what makes the number mean + * something. `const a = f(), b = g(), c = h();` is three initializers however it is punctuated, and + * `a(), b(), c()` is three calls: scoring either as one let a seven-statement try be rewritten into + * a two-statement one with no change to what it runs, which took `error-classification` from fail + * to pass. `merge-declarations` and `merge-comma-expressions` in the mutation corpus are + * the tree-scale versions. */ function countStatement(statement: ts.Statement): number { if (ts.isBlock(statement)) { return countStatements(statement.statements); } + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.length; + } + + if (ts.isExpressionStatement(statement)) { + return commaOperands(statement.expression); + } + let count = 1; if (ts.isTryStatement(statement)) { @@ -742,6 +822,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { let statementCount = 0; let hasTryCatch = false; + let callbackCatches = 0; const catches: CatchEvidence[] = []; const calleeNames: string[] = []; const logCalls: LogCall[] = []; @@ -759,31 +840,40 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is - // still inside it. `calleeNames` and `logCalls` keep descending regardless, which is what lets - // `isTrivial` see work a short statement count hides. A try/catch does not: a per-item catch is - // not part of this body's own statement list, and `countStatement` already stops at a nested - // function boundary, so counting it here let `tryStatementCount` exceed the entry point's whole - // `statementCount` and judged a per-item error boundary as though it were the route's own. + // still inside it. `calleeNames` and `logCalls` keep descending regardless. A try/catch does + // not: a per-item catch is not part of this body's own statement list, and `countStatement` + // already stops at a nested function boundary, so counting it here let `tryStatementCount` + // exceed the entry point's whole `statementCount` and judged a per-item error boundary as + // though it were the route's own. What is refused is counted in `callbackCatches` instead of + // dropped, so a route whose only error handling was refused is failed rather than excused. // // Only an iteration callback is a boundary, not every function-like node: a route's own body // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's // own continuation one layer of nesting away, and its catch is the route's own error handling. + // + // A nested function's statements count towards `statementCount` too, whichever kind it is. + // They are work the route does, and leaving them out let `trace("x", async () => { whole body + // })` collapse a route to one statement, which is inside the triviality rule's limit: the route + // then read as trivial and every check reported not-applicable for it. `wrap-body-in-trace` in + // the mutation corpus is that shape. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { + if (isEntryFunction(node)) statementCount += countFunctionStatements(node); const entersIterationCallback = inCallback || isIterationCallback(node); ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { hasTryCatch = true; + if (node.catchClause && inCallback) callbackCatches++; if (node.catchClause && !inCallback) { const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); catches.push({ rethrows: clause.rethrows, branches: clause.branches, - guardsParse: guardsParse(node.tryBlock), + ...guardedWork(node.tryBlock), tryStatementCount, }); } @@ -839,6 +929,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { calleeNames, hasTryCatch, catches, + callbackCatches, logCalls, statementCount, }; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index a1d3be0b2db..ff0ad245660 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -12,14 +12,20 @@ export type CheckResult = { * legible instead of collapsing into one boolean. */ export type CatchEvidence = { - /** The clause contains a `throw`. */ + /** + * The clause throws on its own straight-line path: a `throw` among its statements, or among a + * bare nested block's, reached before anything that definitely exits. A throw guarded by an `if`, + * a loop, a `switch`, a nested `try` or a callback does not count, however the guard is spelled. + */ rethrows: boolean; /** - * The clause picks what to do from what it caught: an `if` or `switch` whose condition references - * the caught error binding, or a conditional that is the whole `return`/`throw`. `if (retries > 0)` - * does not count, and a bindingless `catch { ... }` cannot count at all. An `instanceof` used only - * to word a message, `json({ error: e instanceof Error ? e.message : String(e) })`, does not - * count either: every error still leaves by the same path. + * The clause picks what to do from what it caught, on that same straight-line path: an `if` or + * `switch` whose condition references the caught error binding AND at least one of whose arms + * returns or throws, or a conditional that is the whole value of a `return`/`throw`. + * `if (retries > 0)` does not count, `if (e instanceof Error) { }` does not count, and a + * bindingless `catch { ... }` cannot count at all. An `instanceof` used only to word a message, + * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count either: every + * error still leaves by the same path. */ branches: boolean; /** @@ -30,6 +36,15 @@ export type CatchEvidence = { * its catch. */ guardsParse: boolean; + /** + * Everything the guarded region waits for is one of those parses. What separates + * `try { const body = await request.json(); } catch { 400 }` from + * `try { const body = await request.json(); return await handleEverything(body); } catch { 500 }`, + * which the statement count reads as the same size. Synchronous work is not counted here: the + * calls that prepare a parse's input are synchronous, and the swallows this has to catch wait on + * a service. + */ + awaitsOnlyParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; }; @@ -64,12 +79,20 @@ export type EntryPoint = { hasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; + /** + * Catch clauses the scan found but refused to attribute to the route, because they sit inside a + * per-item iteration callback. Kept rather than dropped so `error-classification` can tell "this + * route catches nothing" from "this route's only error handling was refused", which are 50 points + * apart and used to read the same. + */ + callbackCatches: number; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; /** - * Statement count across loader/action bodies, used by the triviality rule. A body that - * delegates to a same-file helper counts that helper's statements too, one hop only: work in a - * helper's own helpers, or in an imported module, is not counted. + * Statement count across loader/action bodies, used by the triviality rule. Includes the + * statements of functions written inline in those bodies, so wrapping a body in a callback does + * not shrink it. A body that delegates to a same-file helper counts that helper's statements too, + * one hop only: work in a helper's own helpers, or in an imported module, is not counted. */ statementCount: number; }; diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 5a73eca50e5..21150799ab4 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -179,6 +179,75 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // C5. The count is not the only condition any more, and this is the shape that showed why: two + // statements, one of them a parse, and the whole handler inside the try. Before `awaitsOnlyParse` + // the count read it as a narrow guard and passed it, which is the `otel.v1.logs.ts` swallow + // written compactly. Three spellings of the same thing, all of which the count reads as narrow. + const COMPACT_SWALLOWS: Array<[string, string]> = [ + [ + "two statements", + `try { const body = await request.json(); return await handleEverything(body); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, the parse nested inside the call", + `try { return await handleEverything(await request.json()); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, merged into a declaration list", + `try { const body = await request.json(), out = await handleEverything(body); return out; } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + ]; + + for (const [label, body] of COMPACT_SWALLOWS) { + it(`fails a whole handler wrapped in a parse-guard-shaped try (${label})`, () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) {\n${body}\n}` + ); + expect(r.status).toBe("fail"); + }); + } + + // The counterpart: the same route with the handler moved out of the try is a real guard and + // still passes, so the rule above is not just "any try containing an await fails". + it("passes the same route once the handler moves out of the try", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "bad json" }, { status: 400 }); } + return await handleEverything(body); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Synchronous string work preparing a parse's input is not what `awaitsOnlyParse` refuses. Four + // real routes are this shape, `admin.llm-models.new.tsx` among them. + it("passes a guard that prepares its input synchronously before parsing", () => { + const r = run( + "error-classification", + "admin.llm-models.new.tsx", + `export async function action({ request }) { + const matchPattern = String(await request.text()); + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return await save(matchPattern); + }` + ); + expect(r.status).toBe("pass"); + }); + it("fails a parse guard that takes a third statement beyond binding the result", () => { const r = run( "error-classification", @@ -432,14 +501,13 @@ describe("error-classification", () => { expect(r.status).toBe("not-applicable"); }); - // A7. A per-item error boundary inside a `.map()` callback used to be judged as the route's own - // catch. The route's own visible body catches nothing, so it is not-applicable, not a pass or a - // fail on the strength of a swallow one level of nesting away. - it("is not applicable to a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { - const r = run( - "error-classification", - "batch.process.ts", - `import { prisma } from "~/db.server"; + // A7 as revised. A per-item error boundary inside a `.map()` callback is still not judged as the + // route's own catch, so it never sets `catches` and never speaks for the route's `tryStatementCount`. + // It is no longer excused either. Reading "no catch of its own" as not-applicable was worth 50 + // points to anything that could get the boundary rule to refuse the route's real catch, which + // `[0].map(...)` did, so a refused catch now fails instead of sitting out. + it("fails a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const source = `import { prisma } from "~/db.server"; export async function action({ request }) { const items = await prisma.item.findMany(); await Promise.all( @@ -452,9 +520,103 @@ describe("error-classification", () => { }) ); return json({ ok: true }); + }`; + const ep = scanFile("batch.process.ts", source)!; + expect(ep.catches).toEqual([]); + expect(ep.callbackCatches).toBe(1); + const r = run("error-classification", "batch.process.ts", source); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("callback the route does not own"); + }); + + // The same route with nothing caught anywhere stays not-applicable, so the fail above is + // attributable to the refused catch and not to the check having stopped excusing anything. + it("is not applicable to a route that catches nothing at all", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all(items.map(async (item) => processItem(item))); + return json({ ok: true }); }` ); expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("catches nothing"); + }); + + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a + // per-item boundary and the route's own catch is found where it always was. Before this, the + // wrapper deleted the route's catches and took a swallow from fail to not-applicable. + it("still fails a swallow wrapped in Promise.all([0].map(...))", () => { + const r = run( + "error-classification", + "wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const [result] = await Promise.all([0].map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + })); + return result; + }` + ); + expect(r.status).toBe("fail"); + }); + + // A second receiver exercising the same mechanism: an empty array literal, which no name list + // would treat differently from a populated one. + it("still fails a swallow wrapped in [].flatMap(...)", () => { + const r = run( + "error-classification", + "wrapped-empty.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return [].flatMap(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // A third, where the name list cannot help at all: a non-array receiver whose method is called + // `map`. The boundary rule still refuses the callback, so the route has no catch of its own, and + // the refusal now fails rather than excusing. This is the shape the name list cannot tell from + // `users.map(...)`, and it is why the refusal had to stop paying. + it("still fails a swallow wrapped in a non-array receiver's .map(...)", () => { + const r = run( + "error-classification", + "wrapped-result.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return await Result.map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); }); }); diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 4d1c56fb917..9c5bc9e468c 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -658,15 +658,31 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - it("does not set rethrows for a throw inside a statically-false if", () => { - const ep = scanFile("x.ts", swallow("if (false) { throw e; }")); - expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); - }); - - it("does not set rethrows for a throw inside a statically-false while", () => { - const ep = scanFile("x.ts", swallow("while (false) { throw e; }")); - expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); - }); + // Every shape anyone has found that puts a `throw` somewhere it can never run. The previous + // round recognised the first two by folding the literal `false`, and lost to the other nine. + // None of them is named in the rule now: a throw counts when it is unconditional, and every + // one of these is guarded by something. `dead-*` in the mutation corpus runs the same list over + // the whole route tree. + const DEAD_SHAPES: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { doThing(); } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["inner try/catch", "try { doThing(); } catch { throw e; }"], + ["for...of an empty array", "for (const item of []) { throw e; }"], + ["for...in an empty object", "for (const key in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_SHAPES) { + it(`does not set rethrows for a throw inside ${label}`, () => { + const ep = scanFile("x.ts", swallow(shape)); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + } it("does not set rethrows for a throw merely registered in a constructed callback", () => { const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); @@ -811,7 +827,10 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - expect(ep!.catches[0]!.rethrows).toBe(true); + // The `throw e` here is guarded by an `if`, so it is not on the clause's straight-line path and + // does not read as a rethrow. The `if` itself does: it reads the binding and one arm throws, so + // the clause decides. The verdict the checks care about is unchanged. + expect(ep!.catches[0]!.rethrows).toBe(false); expect(ep!.catches[0]!.branches).toBe(true); }); @@ -1077,6 +1096,7 @@ describe("scanFile: per-catch evidence", () => { rethrows: false, branches: false, guardsParse: true, + awaitsOnlyParse: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ @@ -1178,6 +1198,7 @@ describe("scanFile: per-catch evidence", () => { rethrows: false, branches: false, guardsParse: false, + awaitsOnlyParse: false, tryStatementCount: 4, }); }); @@ -1204,7 +1225,10 @@ describe("scanFile: per-catch evidence", () => { ` ); expect(ep!.catches).toHaveLength(2); - expect(ep!.catches.filter((c) => c.rethrows && c.branches)).toHaveLength(1); + // `if (e instanceof Response) throw e;` branches (it reads the binding and one arm throws) and + // does not rethrow (the throw is guarded, so it is not on the clause's own path). The action's + // catch does neither. What this test is for is that the two clauses stay separate. + expect(ep!.catches.filter((c) => !c.rethrows && c.branches)).toHaveLength(1); expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); }); @@ -1657,18 +1681,26 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested expect(ep!.catches[0]!.branches).toBe(false); }); - // Positive controls: a genuine reference to the real binding must still be credited, including - // through an enclosing loop whose OWN variable has a different name. + // Positive control: a genuine reference to the real binding, on the clause's own path, with an + // arm that takes the error somewhere the other arm does not go. it("still credits an if that genuinely reads the outer binding directly", () => { - const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { return badRequest(); }")); expect(ep!.catches[0]!.branches).toBe(true); }); - it("still credits an if inside a for...of loop with a different loop variable", () => { + // Two shapes that read the real binding and are still not credited, for reasons that are not + // shadowing. Both are precision the straight-line rule gives up on purpose, and both are + // recorded here so a later reader can tell a deliberate limit from a bug. + it("does not credit an if whose arm does not take the error anywhere", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if nested inside a loop, even with a different loop variable", () => { const ep = scanFile( "x.ts", - swallow("for (const item of items) { if (error.code === item) { doThing(); } }") + swallow("for (const item of items) { if (error.code === item) { return item; } }") ); - expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); }); }); From fb8ca4dee79cc19afd32dbe3b6701532f65aef38 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 19:43:13 +0100 Subject: [PATCH 054/117] test(observability-map): make the score-cannot-rise property a corpus Three rounds argued the property shape by shape and each was broken by a shape nobody had thought of. This replaces the argument with evidence. test/mutationCorpus.test.ts applies 30 whole-tree rewrites to a temp copy of apps/webapp/app/routes and holds each to three assertions: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no route's score rises and no measured route drops out of the measured set. The third is what catches a mutation that raises one route and lowers another; the second holds the population fixed so the comparison is about scores rather than denominators. test/mutations.ts holds the rewrites, each labelled preserving or deleting. Every laundering shape found on this branch is there, plus five dead-code shapes and two iteration receivers found while writing it. They are text splices at AST positions rather than reprints, so a failure diffs down to the one construct that moved. A mutation that touches fewer than 20 files, breaks parsing, or loses routes fails rather than passing vacuously. All 30 hold. Against the code as it was at the start of this round, 16 did not. It takes about three minutes, so it is gated behind OBS_MAP_MUTATION_CORPUS=1 and runs as its own job in the observability-map workflow rather than in pnpm test. That job is allowed to fail the build, and unlike the report job it runs for fork PRs. README figures refreshed to the current tree and the two-invariant section replaced by a pointer to the corpus. --- .github/workflows/observability-map.yml | 35 + internal-packages/observability-map/README.md | 45 +- .../test/mutationCorpus.test.ts | 255 ++++++++ .../observability-map/test/mutations.ts | 605 ++++++++++++++++++ 4 files changed, 924 insertions(+), 16 deletions(-) create mode 100644 internal-packages/observability-map/test/mutationCorpus.test.ts create mode 100644 internal-packages/observability-map/test/mutations.ts diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index f63409adb96..0406f1c3730 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -16,6 +16,41 @@ permissions: pull-requests: write jobs: + # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, + # asserting the score does not rise. Roughly three minutes, which is why it is gated out of the + # package's default `pnpm test` and run here instead. Unlike the report job below it has no token + # to lose, so it runs for fork PRs too, and unlike the report job it is allowed to fail the build. + mutation-corpus: + name: 🧬 Mutation corpus + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🧬 Run the corpus + env: + OBS_MAP_MUTATION_CORPUS: "1" + run: | + pnpm --filter @internal/observability-map exec vitest run \ + test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept + report: runs-on: warp-ubuntu-latest-x64-4x # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 76638b7e7ab..a058282c727 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -4,7 +4,7 @@ Scores every webapp entry point on whether it could explain itself during an inc the ones worth fixing. An entry point is a Remix `loader` or `action` under `apps/webapp/app/routes`, 427 of them at the time of writing. -The number it prints today is 17 out of 100. That is not a bug, and the rest of this file is mostly +The number it prints today is 15 out of 100. That is not a bug, and the rest of this file is mostly about why you should believe it. ## Running it @@ -27,28 +27,38 @@ against the PR's merge base, with the score, what changed, and the current fix l report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a later phase decides to add one. See `.github/workflows/observability-map.yml`. -## What 17 means +## What 15 means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. It is low because the webapp does -not attach tenant identity to its failures: **21 of 412 entry points name an environment, project, -organization, run or user on a failure path.** Everything else, when it breaks at 3am, tells you the +not attach tenant identity to its failures: **11 of 412 entry points name an environment, project, +organization or user on a failure path.** Everything else, when it breaks at 3am, tells you the route and the request id and nothing about whose request it was. The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying every catch clause in the tree used to score it 100, which meant the metric paid you for deleting error handling. -Two invariants hold now, and both are asserted in `test/score.test.ts` rather than measured once: +The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` +applies 30 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +and asserts three things for each: the published global does not rise, the mean over the routes +measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's +score rises or drops out of the measured set. Every laundering shape a reviewer has found on this +branch is an entry in it, `test/mutations.ts` holds them, and each entry says which it is. -- **Removing error handling must not raise the score.** Deleting every catch clause drops it to 8, - and deleting the logs as well drops it to 2. -- **Adding error handling that does nothing must not raise the score.** Wrapping every body in - `try { ... } catch (e) { throw e }` leaves the score unchanged. That mutation used to be worth 27 - points across the tree, because a rethrow-only clause counted as a pass while no catch at all was - not-applicable, and the two are observationally identical. +Two of them are worth naming because they are the ones the design turns on. Deleting every catch +clause in the tree drops the score from 15 to 2, so the metric does not pay you for removing error +handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves it unchanged, so it does +not pay you for adding error handling that does nothing either. -If you change this package, check both directions still hold. +The honest statement is "these 30 rewrites are defended, and here they are", not "unpaddable". The +corpus takes about three minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its +own CI job rather than in `pnpm test`. If you change this package, run it: + +```bash +OBS_MAP_MUTATION_CORPUS=1 pnpm --filter @internal/observability-map exec vitest run \ + test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept +``` So the number is deliberately unflattering, and one platform change would move most of it. Nothing central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line @@ -72,7 +82,7 @@ rather than celebrating. ## Two findings are headlines, not list entries -`audit-trail` fails 19 of 19, and `request-context` fails 391 of 412. Printing either one per route +`audit-trail` fails 19 of 19, and `request-context` fails 401 of 412. Printing either one per route would bury the route-specific findings under the same sentence repeated hundreds of times, so both are reported as a figure: the `AUDIT` and `CONTEXT` lines. 329 entry points fail nothing except `request-context` and appear only in that figure, which leaves 71 in the fix list. An entry that @@ -152,7 +162,7 @@ stays visible. value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it up to the wrong variable or a constant, would move the score exactly as far as one that wired it up correctly. Measured on the real tree: adding a synthetic `environmentId` field to every in-catch log -call, with no other change, takes the global score from 17 to 27. +call, with no other change, takes the global score from 15 to 26. That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a @@ -180,8 +190,11 @@ Read these before trusting a specific verdict. - **Only the first object-literal argument is read** for identifier fields, and only its property names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither does a second object. -- **Inline callbacks are not descended into** when counting statements, so a two-statement body can - hold a pile of work inside a `.map()`. The call count is what catches those cases, imperfectly. +- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` + is a fresh boundary per element, so its clause is not read as the route's own error handling. The + test is the method name, which cannot tell `users.map` from `Result.map`. Being wrong there costs + precision rather than points: a refused catch fails the route rather than excusing it, so no + wrapper can turn a swallow into a not-applicable by getting the boundary rule to refuse it. - **Sensitivity is a heuristic**: a symbol list plus path segments. It was circular until recently, counting `requireAdminApiRequest` as a hazard when it is a mitigation, which made 34 of 67 sensitive routes sensitive purely for being guarded. Expect it to need pruning again as routes diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts new file mode 100644 index 00000000000..f74076c1d5b --- /dev/null +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -0,0 +1,255 @@ +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { scanDirectory } from "../src/scan.js"; +import { buildReport } from "../src/score.js"; +import { MUTATIONS, type Mutation } from "./mutations.js"; + +/** + * The tree-scale mutation corpus. + * + * The tool's central claim is that no semantics-preserving edit to a route raises its score. Three + * rounds argued that claim shape by shape and lost each time. This file turns it into evidence + * instead: every laundering shape anyone has found is a corpus entry, and each entry rewrites the + * whole real route tree in a temp copy and is held to three assertions. + * + * - the published global does not rise. That is the figure the claim is about. + * - the mean over the routes measured in BOTH runs does not rise. Same comparison at full + * precision, with the population held fixed so it measures scores rather than denominators. + * - for a semantics-preserving rewrite, no individual route's score rises and no measured route + * drops out of the measured set. The tree mean can hide a route going up by taking another down; + * `[0].map(...)` is exactly that shape. + * + * Tree scale, not per-fixture, because that is where laundering pays. A shape that moves one + * hand-written fixture by 50 points may move the tree by nothing; a shape that moves the tree is the + * one worth defending. + * + * The honest statement this file supports is "these N mutations are defended, and here they are", + * never "unpaddable". + * + * Runtime is roughly six seconds per entry, which is why the whole file is gated behind + * `OBS_MAP_MUTATION_CORPUS=1`. The `observability-map` workflow sets it, so the gate keeps the + * default suite fast without making this the thing nobody runs. + */ + +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); +const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; + +/** + * Where a corpus entry goes when the tool does not defend it. Empty as of round A fix 2: all 30 + * entries hold. When a reviewer finds a shape that cannot be defended, add it to the corpus and + * name it here rather than leaving it out, and write down why in the round's report. `it.fails` + * keeps such an entry running, so closing the hole later turns this file red until the entry is + * moved back out deliberately. + */ +const KNOWN_GAPS = new Set([]); + +type SourceFile = { relativeName: string; source: string }; + +/** Route modules exactly as `scanDirectory` enumerates them: flat files, plus one `route.ts(x)` per + * directory. Read once; every mutation rewrites this list rather than the tree on disk. */ +function readTree(dir: string): SourceFile[] { + const files: SourceFile[] = []; + const take = (absolutePath: string, relativeName: string) => { + files.push({ relativeName, source: readFileSync(absolutePath, "utf8") }); + }; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + take(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); + } + continue; + } + if (!entry.isFile() || !/\.tsx?$/.test(entry.name) || entry.name.endsWith(".d.ts")) continue; + take(join(dir, entry.name), entry.name); + } + return files; +} + +function materialize(files: SourceFile[]): string { + const root = join( + tmpdir(), + `obs-map-corpus-${process.pid}-${Math.random().toString(36).slice(2)}` + ); + for (const file of files) { + const target = join(root, file.relativeName); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, file.source); + } + return root; +} + +type Measurement = { + global: number | null; + /** The unrounded mean the global is a rounding of. A rise of less than half a point is invisible + * in `global` and is still a rise, so the assertions read this. */ + exactMean: number; + measured: number; + entryPoints: number; + parseFailures: number; + /** Per route file, so a mutation that raises one route while lowering the tree is still caught. + * `[0].map(...)` is exactly that shape: it deletes a route's catches, which takes a failing route + * to 100 and a passing one to nothing, and the two cancel in the global. */ + perEntry: Map; +}; + +function measure(files: SourceFile[]): Measurement { + const root = materialize(files); + try { + const { entryPoints, parseFailures } = scanDirectory(root); + const report = buildReport(entryPoints, parseFailures); + const scores = report.entries.filter((e) => e.measured).map((e) => e.score); + const perEntry = new Map(); + for (const entry of report.entries) { + perEntry.set(entry.fileName, { + score: entry.score, + measured: entry.measured, + checks: entry.rawChecks.map((c) => `${c.id}=${c.status}`).join(" "), + }); + } + return { + global: report.global, + exactMean: scores.length === 0 ? 0 : scores.reduce((a, b) => a + b, 0) / scores.length, + measured: report.measured, + entryPoints: entryPoints.length, + parseFailures: parseFailures.length, + perEntry, + }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +type Rise = { fileName: string; from: number; to: number; before: string; after: string }; + +/** + * Route files the mutation made look better, worst first. Two ways to qualify, both counted: the + * score went up, or a route that was being measured stopped being measured. The second is a rise + * too. An unmeasured route's score is the vacuous 100 and it leaves every mean, so dropping out of + * the measured set is the most complete form of the thing the property forbids. + * + * A route the mutation removed from the report entirely is not counted here; the entry-point guard + * catches that instead. + */ +function risesIn(baseline: Measurement, after: Measurement): Rise[] { + const rises: Rise[] = []; + for (const [fileName, before] of baseline.perEntry) { + const now = after.perEntry.get(fileName); + if (!now) continue; + const droppedOut = before.measured && !now.measured; + if (!droppedOut && now.score <= before.score) continue; + rises.push({ + fileName, + from: before.score, + to: now.score, + before: before.checks, + after: now.checks, + }); + } + return rises.sort((a, b) => b.to - b.from - (a.to - a.from)); +} + +/** Mean score over the routes measured in BOTH runs. The plain mean moves when the measured + * population moves, which a mutation can do without making any route look better: an inert + * try/catch takes 15 trivial routes off the exemption list and into the report, and a route joining + * at 50 raises a tree averaging 15 while itself having gone from an unmeasured 100 to a measured 50. + * Holding the population fixed is what makes the comparison about the scores. */ +function commonMean(baseline: Measurement, after: Measurement): { before: number; after: number } { + let sumBefore = 0; + let sumAfter = 0; + let n = 0; + for (const [fileName, before] of baseline.perEntry) { + const now = after.perEntry.get(fileName); + if (!before.measured || !now || !now.measured) continue; + sumBefore += before.score; + sumAfter += now.score; + n++; + } + return n === 0 ? { before: 0, after: 0 } : { before: sumBefore / n, after: sumAfter / n }; +} + +function mutate(files: SourceFile[], mutation: Mutation): { files: SourceFile[]; changed: number } { + let changed = 0; + const out = files.map((file) => { + const source = mutation.apply(file.relativeName, file.source); + if (source === null || source === file.source) return file; + changed++; + return { relativeName: file.relativeName, source }; + }); + return { files: out, changed }; +} + +const describeCorpus = ENABLED && existsSync(ROUTES) ? describe : describe.skip; + +describeCorpus("mutation corpus over the real route tree", () => { + const files = ENABLED && existsSync(ROUTES) ? readTree(ROUTES) : []; + const baseline = ENABLED && existsSync(ROUTES) ? measure(files) : null; + + it("has a baseline worth mutating", () => { + expect(baseline).not.toBeNull(); + expect(baseline!.entryPoints).toBeGreaterThan(300); + expect(baseline!.global).not.toBeNull(); + console.log( + `[corpus] baseline global=${baseline!.global} mean=${baseline!.exactMean.toFixed(3)} ` + + `measured=${baseline!.measured} eps=${baseline!.entryPoints} files=${files.length}` + ); + }); + + /** + * The number of files a mutation must touch before its result means anything. A mutation that + * silently matched nothing would otherwise "pass" by leaving the tree alone, which is the exact + * failure mode that let earlier rounds believe a shape was defended. + */ + const MINIMUM_FILES_TOUCHED = 20; + + for (const mutation of MUTATIONS) { + const run = KNOWN_GAPS.has(mutation.id) ? it.fails : it; + + run(`${mutation.kind}: ${mutation.what} (${mutation.id})`, () => { + const { files: mutated, changed } = mutate(files, mutation); + expect(changed).toBeGreaterThanOrEqual(MINIMUM_FILES_TOUCHED); + + const after = measure(mutated); + + // A mutation that stops the tree parsing, or that hides most of the routes from the scanner, + // has not tested the property: whatever the score does afterwards is measuring a different + // tree. Both guards fail loudly rather than letting such a mutation report a pass. + expect(after.parseFailures).toBe(baseline!.parseFailures); + expect(after.entryPoints).toBeGreaterThanOrEqual(baseline!.entryPoints - 5); + + const rises = risesIn(baseline!, after); + const common = commonMean(baseline!, after); + console.log( + `[corpus] ${mutation.id}: global ${baseline!.global} -> ${after.global} ` + + `(mean ${baseline!.exactMean.toFixed(3)} -> ${after.exactMean.toFixed(3)}, ` + + `common mean ${common.before.toFixed(3)} -> ${common.after.toFixed(3)}, ` + + `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, ` + + `routes raised ${rises.length})` + + rises + .slice(0, 3) + .map( + (r) => + `\n ${r.fileName} ${r.from}->${r.to}\n was: ${r.before}\n now: ${r.after}` + ) + .join("") + ); + + // The published figure, which is what the claim is about. + expect(after.global!).toBeLessThanOrEqual(baseline!.global!); + // The same comparison at full precision, over a fixed population so it measures the scores + // and not who is in the denominator. + expect(common.after).toBeLessThanOrEqual(common.before + 1e-9); + + // Per route, for the preserving half of the corpus. This is the property as stated: an edit + // that does not change what a route does must not make that route look better, whatever it + // does to the tree's mean. The deleting half is exempt on purpose: a route whose only failing + // check was error-classification really does leave the denominator when its catch goes, which + // the design chose over crediting a route for deleting its error handling, and the global + // figure above is where that trade is held to account. + if (mutation.kind === "preserving") { + expect(rises.map((r) => `${r.fileName} ${r.from}->${r.to}`)).toEqual([]); + } + }); + } +}); diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts new file mode 100644 index 00000000000..4b8f6df4671 --- /dev/null +++ b/internal-packages/observability-map/test/mutations.ts @@ -0,0 +1,605 @@ +import ts from "typescript"; + +/** + * Source-to-source mutations for the tree-scale corpus in `mutationCorpus.test.ts`. + * + * Every mutation here is a *text* rewrite driven by AST positions, never a reprint. A reprint would + * change formatting everywhere and make a failure impossible to read; splicing at node positions + * leaves the rest of the file byte-identical, so a corpus failure can be diffed down to the one + * construct that moved. + * + * Two kinds of entry live in the corpus and they are labelled `preserving` and `deleting`: + * + * - `preserving`: the rewrite does not change what the route does. Dead code that can never run, + * a wrapper that runs the same statements once, a comment, a merge of adjacent `const`s. The + * property under test is the one the tool claims: no such edit may raise the score. + * - `deleting`: the rewrite removes error handling or logging. The route is worse afterwards, so + * the score must not rise either, for a different and simpler reason. + * + * Neither kind is ever executed. "Semantics-preserving" here means preserving the observable + * behaviour of the route as written, which is what the scanner claims to measure; it is not a + * claim that the mutated tree compiles against its real types. + */ + +export type MutationKind = "preserving" | "deleting"; + +export type Mutation = { + id: string; + kind: MutationKind; + /** What the rewrite does, in one line, for the corpus table in the report. */ + what: string; + /** The mutated source, or null when this file has nothing for the mutation to touch. */ + apply(fileName: string, source: string): string | null; +}; + +type Edit = { start: number; end: number; text: string }; + +function parse(fileName: string, source: string): ts.SourceFile { + return ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS + ); +} + +/** + * Splice edits into `source`, right to left so earlier offsets stay valid. + * + * An edit that falls inside an earlier edit's range is dropped rather than applied: a mutation that + * deletes a catch clause and one that rewrites a statement inside that clause would otherwise + * produce overlapping splices. Dropping the inner one is what "the outer rewrite won" means. + */ +function applyEdits(source: string, edits: Edit[]): string | null { + if (edits.length === 0) return null; + const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end); + const kept: Edit[] = []; + for (const edit of sorted) { + const last = kept[kept.length - 1]; + if (last && edit.start < last.end) continue; + kept.push(edit); + } + let out = source; + for (let i = kept.length - 1; i >= 0; i--) { + const edit = kept[i]!; + out = out.slice(0, edit.start) + edit.text + out.slice(edit.end); + } + return out === source ? null : out; +} + +function insert(at: number, text: string): Edit { + return { start: at, end: at, text }; +} + +function forEachNode(node: ts.Node, visit: (n: ts.Node) => void): void { + visit(node); + ts.forEachChild(node, (child) => { + forEachNode(child, visit); + }); +} + +// -- entry points --------------------------------------------------------------------------- + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +function propertyNameOf(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) { + return property.name.text; + } + return null; +} + +/** + * Handler functions on a builder's object argument, in the two shapes the route builders use: + * `handler` at the top level and `methods.POST.handler`. + * + * Deliberately a copy of the same shapes `src/scan.ts` recognises rather than an import of them. + * The harness has to be able to disagree with the scanner about where a route body is; sharing the + * scanner's own notion would let a bug in that notion hide a laundering shape from the corpus. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyNameOf(property); + const value = unwrap(property.initializer); + if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + for (const method of value.properties) { + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (ts.isObjectLiteralExpression(config)) collectNamedHandlers(config, out); + } + } + } +} + +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; + } + return current; + } +} + +function fromInitializer(expr: ts.Expression, out: EntryFunction[]): void { + const target = unwrap(expr); + if (isEntryFunction(target)) { + out.push(target); + return; + } + if (!ts.isCallExpression(target)) return; + for (const arg of rootCall(target).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } +} + +const ENTRY_NAMES = new Set(["loader", "action"]); + +function isExported(node: ts.Node): boolean { + return ( + ts.canHaveModifiers(node) && + ts.getModifiers(node)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true + ); +} + +/** Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. */ +function entryBodies(sf: ts.SourceFile): ts.Block[] { + const functions: EntryFunction[] = []; + for (const statement of sf.statements) { + if (!isExported(statement)) continue; + if (ts.isFunctionDeclaration(statement) && statement.name) { + if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions); + } + } + const bodies: ts.Block[] = []; + for (const fn of functions) { + if (fn.body && ts.isBlock(fn.body)) bodies.push(fn.body); + } + return bodies; +} + +// -- generic mutation shapes ---------------------------------------------------------------- + +function catchClauses(sf: ts.SourceFile): ts.CatchClause[] { + const out: ts.CatchClause[] = []; + forEachNode(sf, (node) => { + if (ts.isCatchClause(node)) out.push(node); + }); + return out; +} + +function bindingNameOf(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; +} + +/** + * Append a statement at the end of every catch clause that names its binding. `snippet` receives + * the binding name. Appending is the position that matters: a shape spliced in after a `return` is + * already unreachable and proves nothing. + */ +function appendToEveryCatch( + id: string, + kind: MutationKind, + what: string, + snippet: (binding: string) => string +): Mutation { + return { + id, + kind, + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + edits.push(insert(clause.block.end - 1, `\n${snippet(binding)}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Wrap every route body in a single-shot wrapper, `open` before its statements and `close` after. */ +function wrapEveryBody(id: string, what: string, open: string, close: string): Mutation { + return { + id, + kind: "preserving", + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push(insert(body.getStart() + 1, `\n${open}\n`)); + edits.push(insert(body.end - 1, `\n${close}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Prepend text at the very top of every file, before the first token's leading trivia. */ +function prependToEveryFile(id: string, what: string, text: string): Mutation { + return { + id, + kind: "preserving", + what, + apply(_fileName, source) { + return `${text}\n${source}`; + }, + }; +} + +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +function logStatementEdits(sf: ts.SourceFile): Edit[] { + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isExpressionStatement(node)) return; + const call = unwrap(node.expression); + if (!ts.isCallExpression(call)) return; + const text = calleeText(call.expression); + if (text !== null && LOGGER_CALLEE.test(text)) { + edits.push({ start: node.getStart(), end: node.end, text: ";" }); + } + }); + return edits; +} + +/** + * Remove the catch clause from every `try`. With a `finally` present the clause alone goes and the + * `try`/`finally` stands; without one the whole `try` collapses to the bare block it guarded, which + * is still a legal statement. Point edits either way, so a nested rewrite inside the clause is + * simply dropped by `applyEdits` rather than colliding. + */ +function catchDeletionEdits(sf: ts.SourceFile): Edit[] { + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isTryStatement(node) || !node.catchClause) return; + if (node.finallyBlock) { + edits.push({ start: node.catchClause.getStart(), end: node.catchClause.end, text: " " }); + return; + } + edits.push({ start: node.getStart(), end: node.tryBlock.getStart(), text: "" }); + edits.push({ start: node.tryBlock.end, end: node.end, text: "" }); + }); + return edits; +} + +// -- the corpus ------------------------------------------------------------------------------- + +/** + * Every laundering shape found by a reviewer on this branch, plus the five extra dead-code shapes + * and two extra iteration receivers found while writing this file. Each entry is a whole-tree + * rewrite; `mutationCorpus.test.ts` asserts the global score does not rise for any of them. + */ +export const MUTATIONS: Mutation[] = [ + prependToEveryFile( + "suppress-every-check", + "prepend an obs-map-disable directive for every check to every file", + [ + "// obs-map-disable error-classification -- mutation corpus", + "// obs-map-disable request-context -- mutation corpus", + "// obs-map-disable auth-boundary -- mutation corpus", + "// obs-map-disable audit-trail -- mutation corpus", + ].join("\n") + ), + + { + id: "jsx-text-line-directive", + kind: "preserving", + what: "add a component whose JSX text begins with a // directive", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`; + }, + }, + { + id: "jsx-text-after-expression", + kind: "preserving", + what: "add a component whose JSX text starts a // directive right after an expression container", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`; + }, + }, + { + id: "jsx-text-block-directive", + kind: "preserving", + what: "add a component whose JSX text is a /* */ directive", + apply(fileName, source) { + if (!fileName.endsWith(".tsx")) return null; + return `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`; + }, + }, + + { + id: "delete-every-catch", + kind: "deleting", + what: "remove every catch clause", + apply(fileName, source) { + return applyEdits(source, catchDeletionEdits(parse(fileName, source))); + }, + }, + { + id: "delete-every-log", + kind: "deleting", + what: "remove every logger call statement", + apply(fileName, source) { + return applyEdits(source, logStatementEdits(parse(fileName, source))); + }, + }, + { + id: "delete-every-catch-and-log", + kind: "deleting", + what: "remove every catch clause and every logger call statement", + apply(fileName, source) { + const sf = parse(fileName, source); + return applyEdits(source, [...catchDeletionEdits(sf), ...logStatementEdits(sf)]); + }, + }, + + wrapEveryBody( + "wrap-body-in-rethrow", + "wrap every route body in try { ... } catch (e) { throw e }", + "try {", + "} catch (obsMapMutationError) { throw obsMapMutationError; }" + ), + wrapEveryBody( + "wrap-body-in-trace", + 'wrap every route body in trace("x", async () => { ... })', + 'return obsMapTrace("obs-map-mutation", async () => {', + "});" + ), + wrapEveryBody( + "wrap-body-in-single-element-map", + "wrap every route body in Promise.all([0].map(async () => { ... }))", + "return Promise.all([0].map(async () => {", + "})).then((obsMapResults) => obsMapResults[0]);" + ), + wrapEveryBody( + "wrap-body-in-single-element-flatmap", + "wrap every route body in Promise.all([0].flatMap(async () => { ... }))", + "return Promise.all([0].flatMap(async () => {", + "})).then((obsMapResults) => obsMapResults[0]);" + ), + wrapEveryBody( + "wrap-body-in-non-array-map", + "wrap every route body in a non-array receiver's .map(...)", + "return obsMapResult.map(async () => {", + "});" + ), + wrapEveryBody( + "wrap-body-in-non-array-filter", + "wrap every route body in a non-array receiver's .filter(...)", + "return obsMapPipe.filter(async () => {", + "});" + ), + + { + id: "throw-after-return-in-catch", + kind: "preserving", + what: "append throw e; after the first return in every catch", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const returned = clause.block.statements.find(ts.isReturnStatement); + if (!returned) continue; + edits.push(insert(returned.end, ` throw ${binding};`)); + } + return applyEdits(source, edits); + }, + }, + + appendToEveryCatch( + "dead-if-false", + "preserving", + "append if (false) { throw e; } to every catch", + (e) => `if (false) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-while-false", + "preserving", + "append while (false) { throw e; } to every catch", + (e) => `while (false) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-false", + "preserving", + "append for (;false;) { throw e; } to every catch", + (e) => `for (;false;) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-true-else", + "preserving", + "append if (true) { 0; } else { throw e; } to every catch", + (e) => `if (true) { 0; } else { throw ${e}; }` + ), + appendToEveryCatch( + "dead-switch-no-case", + "preserving", + "append switch (1) { case 2: throw e; } to every catch", + (e) => `switch (1) { case 2: throw ${e}; }` + ), + appendToEveryCatch( + "dead-inner-try", + "preserving", + "append try { 0; } catch { throw e; } to every catch", + (e) => `try { 0; } catch { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-of-empty", + "preserving", + "append for (const x of []) { throw e; } to every catch", + (e) => `for (const obsMapItem of []) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-for-in-empty", + "preserving", + "append for (const k in {}) { throw e; } to every catch", + (e) => `for (const obsMapKey in {}) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-empty-string", + "preserving", + 'append if ("") { throw e; } to every catch', + (e) => `if ("") { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-not-true", + "preserving", + "append if (!true) { throw e; } to every catch", + (e) => `if (!true) { throw ${e}; }` + ), + appendToEveryCatch( + "dead-if-const-compare", + "preserving", + "append if (1 === 2) { throw e; } to every catch", + (e) => `if (1 === 2) { throw ${e}; }` + ), + appendToEveryCatch( + "registered-throw", + "preserving", + "append [].push(() => { throw e; }) to every catch", + (e) => `[].push(() => { throw ${e}; });` + ), + appendToEveryCatch( + "empty-instanceof-if", + "preserving", + "append if (e instanceof Error) { } to every catch", + (e) => `if (${e} instanceof Error) { }` + ), + + { + id: "merge-declarations", + kind: "preserving", + what: "merge adjacent const statements into one declaration list", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isBlock(node)) return; + const statements = node.statements; + for (let i = 1; i < statements.length; i++) { + const previous = statements[i - 1]!; + const current = statements[i]!; + if (!isSingleConst(previous) || !isSingleConst(current)) continue; + if (source[previous.end - 1] !== ";") continue; + const declaration = current.declarationList.declarations[0]!; + edits.push({ start: previous.end - 1, end: declaration.getStart(), text: ", " }); + } + }); + return applyEdits(source, edits); + }, + }, + + { + id: "merge-comma-expressions", + kind: "preserving", + what: "merge adjacent expression statements into one comma expression", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + forEachNode(sf, (node) => { + if (!ts.isBlock(node) && !ts.isSourceFile(node)) return; + const statements = node.statements; + for (let i = 1; i < statements.length; i++) { + const previous = statements[i - 1]!; + const current = statements[i]!; + if (!ts.isExpressionStatement(previous) || !ts.isExpressionStatement(current)) continue; + if (source[previous.end - 1] !== ";") continue; + edits.push({ start: previous.end - 1, end: current.getStart(), text: ", " }); + } + }); + return applyEdits(source, edits); + }, + }, + + { + id: "inert-statements-after-try", + kind: "preserving", + what: "append five unused const declarations after every try statement in a route body", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + let n = 0; + for (const body of entryBodies(sf)) { + forEachNode(body, (node) => { + if (!ts.isTryStatement(node)) return; + if (!node.parent || !ts.isBlock(node.parent)) return; + const filler = Array.from({ length: 5 }, () => `const obsMapInert${n++} = 1;`).join(" "); + edits.push(insert(node.end, `\n${filler}\n`)); + }); + } + return applyEdits(source, edits); + }, + }, +]; + +function isSingleConst(statement: ts.Statement): statement is ts.VariableStatement { + return ( + ts.isVariableStatement(statement) && + (statement.declarationList.flags & ts.NodeFlags.Const) !== 0 && + statement.declarationList.declarations.length === 1 && + statement.declarationList.declarations[0]!.initializer !== undefined + ); +} From eaafa3ddb9b593e65f54aff04a0824602dd0a14b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:15 +0100 Subject: [PATCH 055/117] test(observability-map): check that a docstring names a test that exists Six docstrings on this branch have asserted something unverifiable or false, and the sixth named `content-is-not-a-comment`, a test nobody ever wrote. The rule has been asked for in prose six times, so it is mechanical now. test/docstringReferences.test.ts scans src for three kinds of reference and asserts each resolves: a backticked kebab-case token, which in this package is always a check id, a corpus id or a test name; a backticked glob like dead-*, which must match a corpus entry by prefix; and a backticked prose phrase long enough and clean enough of code punctuation to be a test title. Three more cases run the same predicates over invented references, so the checker is known to be able to fail rather than merely green. What it does not check is written down beside it: a reference without backticks, a title of fewer than five words, and anything outside src. The kebab half is the half that has actually failed. It caught three references in this very change before they shipped. --- .../test/docstringReferences.test.ts | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 internal-packages/observability-map/test/docstringReferences.test.ts diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts new file mode 100644 index 00000000000..3800066b2c4 --- /dev/null +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -0,0 +1,185 @@ +import ts from "typescript"; +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { CHECKS } from "../src/checks/index.js"; +import { MUTATIONS } from "./mutations.js"; + +/** + * Every test name a docstring in `src/` claims to be covered by must exist. + * + * The rule this enforces has been asked for six times in prose and broken six times, most recently + * by a docstring naming `content-is-not-a-comment`, a test that was never written. Prose cannot + * enforce itself, so this does. + * + * What is checked, precisely, because a checker that overstates its reach is the same defect again: + * + * - every backticked kebab-case token in a `src/` comment, e.g. `empty-instanceof-if`. Those are + * never valid JavaScript identifiers, so in this package they are always a check id, a mutation + * corpus id, a test name, or one of the handful of domain words in `NOT_A_TEST_NAME` below. + * - a backticked glob, `dead-*`, which must match at least one corpus id by prefix. + * - every backticked prose phrase of `MINIMUM_TITLE_WORDS` words or more that contains no code + * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like + * and what a code sample does not. + * + * What is NOT checked: a reference written without backticks, a test title of fewer than + * `MINIMUM_TITLE_WORDS` words (`throw e` and `new URL` are code, and telling a short title from + * short code needs more than punctuation), and anything outside `src/`. A docstring can still name + * a nonexistent short test. The kebab half is the half that has actually failed. + */ + +const SRC = resolve(__dirname, "../src"); +const TESTS = resolve(__dirname); + +/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added + * here is a deliberate statement that the token names no test, and shows up in review as such. */ +const NOT_A_TEST_NAME = new Set([ + // A `CheckStatus` value. + "not-applicable", + // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. + "obs-map-disable-next-line", +]); + +/** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ +const MINIMUM_TITLE_WORDS = 5; + +/** Characters that mean a backticked phrase is a code sample rather than a test title. */ +const CODE_PUNCTUATION = /[{}()[\];=<>"'`|&$/\\]|\.\.\.|\.tsx?\b/; + +function walkFiles(dir: string, suffix: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, suffix, out); + else if (entry.name.endsWith(suffix)) out.push(path); + } + return out; +} + +/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two + * lines reads as one phrase rather than one with a stray asterisk in it. */ +function commentText(file: string): string { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const seen = new Set(); + const parts: string[] = []; + const visit = (node: ts.Node) => { + for (const range of ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + parts.push(source.slice(range.pos, range.end)); + } + ts.forEachChild(node, visit); + }; + visit(sf); + return parts.join("\n").replace(/\n\s*\*\s?/g, " "); +} + +/** Static titles from every `it`/`test`/`describe` call, including the literal chunks of a + * template-literal title, so a reference to part of a generated name still resolves. */ +function testTitles(): Set { + const titles = new Set(); + const add = (value: string) => { + const trimmed = value.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) titles.add(trimmed); + }; + for (const file of walkFiles(TESTS, ".test.ts")) { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const visit = (node: ts.Node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + const root = ts.isPropertyAccessExpression(callee) ? callee.expression : callee; + if (ts.isIdentifier(root) && ["it", "test", "describe"].includes(root.text)) { + const first = node.arguments[0]; + if (first) { + if (ts.isStringLiteralLike(first)) add(first.text); + if (ts.isTemplateExpression(first)) { + add(first.head.text); + for (const span of first.templateSpans) add(span.literal.text); + } + } + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return titles; +} + +describe("docstrings in src name things that exist", () => { + const known = new Set([ + ...CHECKS.map((c) => c.id), + ...MUTATIONS.map((m) => m.id), + ...NOT_A_TEST_NAME, + ]); + const titles = testTitles(); + const corpusIds = MUTATIONS.map((m) => m.id); + const files = walkFiles(SRC, ".ts"); + + it("finds source files and test titles to check against", () => { + expect(files.length).toBeGreaterThan(5); + expect(titles.size).toBeGreaterThan(50); + expect(corpusIds.length).toBeGreaterThan(20); + }); + + it("every backticked kebab-case token names a check, a corpus entry or a test", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/g)) { + const token = match[1]!; + if (known.has(token)) continue; + if ([...titles].some((t) => t.includes(token))) continue; + unknown.push(`${file}: ${token}`); + } + } + expect(unknown).toEqual([]); + }); + + it("every backticked glob matches at least one corpus entry", () => { + const unmatched: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9-]*)-\*`/g)) { + const prefix = `${match[1]!}-`; + if (corpusIds.some((id) => id.startsWith(prefix))) continue; + unmatched.push(`${file}: ${prefix}*`); + } + } + expect(unmatched).toEqual([]); + }); + + it("every backticked prose phrase long enough to be a test title is one", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][^`\n]*)`/g)) { + const phrase = match[1]!.replace(/\s+/g, " ").trim(); + if (phrase.split(" ").length < MINIMUM_TITLE_WORDS) continue; + if (CODE_PUNCTUATION.test(phrase)) continue; + if (titles.has(phrase)) continue; + unknown.push(`${file}: ${phrase}`); + } + } + expect(unknown).toEqual([]); + }); + + // The checker has to be able to fail, or it is decoration. These run the same predicates over an + // invented docstring rather than over `src/`, so the guarantee does not rest on `src/` currently + // happening to contain a bad reference. + it("would reject a docstring naming a test that does not exist", () => { + const invented = "see `content-is-not-a-comment` for the proof"; + const token = /`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/.exec(invented)![1]!; + expect(known.has(token)).toBe(false); + expect([...titles].some((t) => t.includes(token))).toBe(false); + }); + + it("would reject a docstring naming a corpus glob that matches nothing", () => { + const prefix = /`([a-z][a-z0-9-]*)-\*`/.exec("covered by `no-such-family-*` above")![1]!; + expect(corpusIds.some((id) => id.startsWith(`${prefix}-`))).toBe(false); + }); + + it("would reject a docstring naming a prose test title that does not exist", () => { + const phrase = "reads a directive that nobody ever wrote down anywhere"; + expect(phrase.split(" ").length).toBeGreaterThanOrEqual(MINIMUM_TITLE_WORDS); + expect(CODE_PUNCTUATION.test(phrase)).toBe(false); + expect(titles.has(phrase)).toBe(false); + }); +}); From 3a95d7e9b15bee5e93e4d77a4768153b58efbc7a Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:32 +0100 Subject: [PATCH 056/117] test(observability-map): test the JSX suppression fix, and name a test that exists The fix shipped last round with no test that fails without it. The one test that mentioned JSX used text with the // mid-string, which the comment-range lexers never read as a comment anyway, so removing ts.isJsxText left the whole suite green. The corpus cannot cover this by construction: scoreEntry caps an entry at its pre-suppression ratio, so a suppression can only lower a score, and a harness that watches for the score rising is blind to it. Four cases now cover the shapes that needed the fix, JSX text that BEGINS with // or /*, in four tree positions. Verified by removing ts.isJsxText: those four fail and nothing else does. A positive control beside them keeps a real comment in a JSX expression container working, so the filter cannot be widened until it eats real comments. The docstring named content-is-not-a-comment, which does not exist. It now names the tests that do. --- .../observability-map/src/suppression.ts | 13 ++++- .../test/suppression.test.ts | 58 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 26a51e13085..476c2a5a2ee 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -35,9 +35,16 @@ function leafTokens(node: ts.Node): ts.Node[] { * BEGINS with `//` or `/*` is the shape that reached the real tree, in * `resources.branches.create.tsx`'s `//`. * - * `content-is-not-a-comment` in `test/suppression.test.ts` covers each kind, and - * `jsx-text-line-directive`, `jsx-text-after-expression` and `jsx-text-block-directive` in the - * mutation corpus cover the JSX shapes over the whole route tree. + * The four cases in `jsx text is content, not a comment` (`test/suppression.test.ts`) are the ones + * that fail without `ts.isJsxText` here; the positive control beside them, `still reads a directive + * from a comment in a JSX expression container`, is what stops the filter being widened until it + * eats real comments. `does not suppress from a directive inside a template literal` and the two + * substitution cases cover the template kinds, and `ignores the directive inside a string literal` + * covers the string kind. + * + * The mutation corpus does NOT cover any of this, and cannot: a suppression can only lower an + * entry's score, because `scoreEntry` caps it at the pre-suppression ratio. Suppression bugs are + * invisible to a harness that watches for the score rising, so they need ordinary unit tests. */ function isClaimedContent(node: ts.Node): boolean { return ( diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index 7ef32f9f94e..c40d1332894 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -142,6 +142,64 @@ describe("suppressedChecks", () => { expect(m.size).toBe(0); }); + // S4. The case above passes without any JSX handling at all, because the comment-range lexers + // only find a comment at the exact offset they are asked about and the `//` there is mid-text. + // These are the shapes that actually needed the fix: JSX text that BEGINS with a comment marker, + // which is what the lexers see when they are pointed at the start of a JsxText node. Removing + // `ts.isJsxText` from `isClaimedContent` makes all four fail and nothing else in the suite. + describe("jsx text is content, not a comment", () => { + const page = (body: string) => `const name = "x"; + export default function Page() { + return ${body}; + }`; + + it("does not suppress from JSX text beginning with a line comment marker", () => { + const m = suppressedChecks( + page(`

// obs-map-disable error-classification -- jsx line

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text beginning with a block comment marker", () => { + const m = suppressedChecks( + page(`

/* obs-map-disable audit-trail -- jsx block */

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text starting right after an expression container", () => { + const m = suppressedChecks( + page(`

{name}// obs-map-disable request-context -- after expression

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // A fourth input, exercising the same mechanism at a different tree position: the text is not + // the first child of the outermost element, so the token boundary the lexer is pointed at is a + // different one again. + it("does not suppress from JSX text nested several elements deep", () => { + const m = suppressedChecks( + page(`
// obs-map-disable auth-boundary -- nested jsx
`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control for the same code path: a real comment inside a JSX expression container is + // not JSX text and must survive. A filter that dropped it would pass every test above for the + // wrong reason. + it("still reads a directive from a comment in a JSX expression container", () => { + const m = suppressedChecks( + page(`

{/* obs-map-disable audit-trail -- real comment */}

`), + "route.tsx" + ); + expect(m.get("audit-trail")).toBe("real comment"); + }); + }); + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism // differently: a second substitution, and a string literal nested inside a JSX expression // container, which is a different node kind again from either hole above. From 07dc007e63cbca774ce0f7495ea85f7834dbf974 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:54:55 +0100 Subject: [PATCH 057/117] fix(observability-map): refuse three ways of adding error handling that does nothing Every laundering shape found before this round took real signal away or moved it about. These three add fake signal, which is what someone reaches for when a CI comment nags them, and two of them are the largest holes ever found here. Global 15 before and after; no route's score rose from any of them. A catch whose try block cannot raise is no longer read as error handling. Prepending try { 0; } catch (e) { if (e instanceof Error) { return 400; } throw e; } to every body took the tree from 15 to 42 and raised 224 routes, because 261 routes catch nothing and sat at not-applicable, and a dead clause moved every one of them to pass. guardedWork now reports whether the block does anything that could reach the clause: a call, a construction, an await, a member access, a throw, an iteration, an instanceof. What is not on that list, a temporal-dead-zone read and a coercion that raises, is written down. A throw written after something that already exited no longer reads as a rethrow. definitelyExits sees through a bare block, a do body, an if/else where both arms exit, a switch with a returning default and a try/finally, where before it recognised only a bare return or throw. The if (true) form needs constant folding this file will not do, and is answered instead by rethrows requiring the clause to contain no reachable return at all: the claim rethrows feeds is that the error leaves the way it arrived, which is only true when throwing is the only way out. A ternary on the error has to send its arms somewhere different, which the if/switch path has required since the round before. return e instanceof Error ? (X) : (X) was worth 50 points a route for a test that decides nothing, and the same comparison now applies to an if/else with identical arms. The 4xx and pMap claims in two docstrings were false in one direction each and now say what is true. Each fix has a test that fails without it, listed in the round A fix 3 report. --- .../src/checks/errorClassification.ts | 32 ++- .../observability-map/src/scan.ts | 218 +++++++++++++---- .../observability-map/src/types.ts | 16 +- .../observability-map/test/checks.test.ts | 57 +++++ .../test/integration.test.ts | 3 +- .../observability-map/test/scan.test.ts | 221 +++++++++++++++++- 6 files changed, 488 insertions(+), 59 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index b377e577cf2..22a431ab764 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -67,9 +67,12 @@ const NARROW_TRY_STATEMENTS = 2; * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. * - * The residual, since awaiting is the signal: a try block that does its non-parse work - * synchronously still reads as a guard. Nothing in the tree does, and it is written down in the - * round A fix 2 report rather than defended. + * Two residuals, since awaiting is the signal. A try block that does its non-parse work + * synchronously still reads as a guard. And `guardedWork` looks for a `ts.AwaitExpression`, which + * `for await (const chunk of work(await request.json()))` and `await using` are not, so a block + * whose only non-parse work is one of those reads as a guard too. Neither occurs in the tree and + * neither is reachable by rewriting a real route, since both need work that is not there to begin + * with. Both are in the round A fix 3 report. */ function isParseGuard(clause: CatchEvidence): boolean { return ( @@ -148,6 +151,14 @@ export function usesBuilder(ep: EntryPoint): boolean { * points to anyone who wraps a body in something the boundary rule refuses, which * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route * that genuinely only handles errors per item, which now fails rather than sitting out. + * + * A clause whose try block cannot raise is read as no clause at all, `guardCanRaise` on the + * evidence. It is not error handling, and crediting one was the largest hole ever found here: + * prepending `try { 0; } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } + * throw e; }` to every body took the tree from 15 to 42 and raised 224 routes, because the 261 + * routes that catch nothing were sitting at not-applicable and a dead clause moved each of them to + * pass. Dropping it here rather than in the scan keeps the evidence honest about what is written + * and puts the judgement where the other judgements are. */ export const errorClassification = { id: ID, @@ -155,30 +166,33 @@ export const errorClassification = { if (isTrivial(ep)) { return { id: ID, status: "not-applicable", detail: "trivial route" }; } - const swallowed = ep.catches.filter(swallows); + const reachable = ep.catches.filter((c) => c.guardCanRaise); + const swallowed = reachable.filter(swallows); if (swallowed.length > 0) { const which = - ep.catches.length > 1 ? ` (${swallowed.length} of ${ep.catches.length} catches)` : ""; + reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; return { id: ID, status: "fail", detail: `catches its errors and takes one way out regardless of what was thrown${which}`, }; } - if (ep.catches.length === 0 && ep.callbackCatches > 0) { + if (reachable.length === 0 && ep.callbackCatches > 0) { return { id: ID, status: "fail", detail: "its only error handling sits in a callback the route does not own", }; } - if (!ep.catches.some(decides)) { + if (!reachable.some(decides)) { return { id: ID, status: "not-applicable", detail: - ep.catches.length === 0 - ? "catches nothing, so it classifies nothing" + reachable.length === 0 + ? ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "guards nothing that can throw, so it classifies nothing" : "every catch rethrows and nothing else, so it classifies nothing", }; } diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index da46cb4e54f..eaf14cd4aa6 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -148,8 +148,35 @@ function isBodyRead(node: ts.Node): boolean { } /** - * What the guarded region does, in the two terms `error-classification` needs to tell a parse guard - * from a handler wrapped around a parse. + * Syntax that can raise. Everything a try block might do that produces something for a catch clause + * to catch: a call, a construction, a tagged template, an `await` or `yield` (the awaited promise + * rejects), a member access (the base may be null or undefined), a `throw`, an iteration (the + * iterator protocol raises on a non-iterable), and `instanceof`/`in` (a TypeError on a non-object + * right side). + * + * See `guardedWork` for what is NOT on this list and why that is a disclosed residual rather than + * an oversight. + */ +function canRaise(node: ts.Node): boolean { + return ( + ts.isCallExpression(node) || + ts.isNewExpression(node) || + ts.isTaggedTemplateExpression(node) || + ts.isAwaitExpression(node) || + ts.isYieldExpression(node) || + ts.isPropertyAccessExpression(node) || + ts.isElementAccessExpression(node) || + ts.isThrowStatement(node) || + ts.isForOfStatement(node) || + ts.isForInStatement(node) || + (ts.isBinaryExpression(node) && + (node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword || + node.operatorToken.kind === ts.SyntaxKind.InKeyword)) + ); +} + +/** + * What the guarded region does, in the three terms `error-classification` needs. * * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees @@ -162,15 +189,36 @@ function isBodyRead(node: ts.Node): boolean { * catch reaches a service: `try { const body = await request.json(); return await * handleEverything(body); }`. * - * Nested function bodies are skipped: a callback written inside the try is not work the try is - * guarding on this pass through. + * `canRaise` is whether the block does anything at all that could reach the clause. A clause whose + * try block cannot raise is not error handling, and reading one as classification paid 50 points a + * route to anyone willing to prepend `try { 0; } catch (e) { if (e instanceof Error) { return + * json(x, { status: 400 }); } throw e; }` to a body: on the real tree that took the global from 15 + * to 42 and raised 224 routes, which is more than every other shape found on this branch put + * together. `dead-classifying-try` in the mutation corpus is the tree-scale version. + * + * The alternative considered was proving unreachability the other way round, by ruling out every + * expression that could throw. It is the same predicate read backwards and it fails the same way, + * so the cheaper direction won: ask what the block DOES, and require it to do something. The + * residual is what `canRaise` does not list. A temporal-dead-zone read (`try { const x = later; }`), + * a coercion that raises (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object are + * all treated as unable to raise. None is expressible as a preserving rewrite of a real route, and + * all of them would need types the scanner does not have. + * + * Nested function bodies are skipped throughout: a callback written inside the try is not work the + * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. */ -function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyParse: boolean } { +function guardedWork(tryBlock: ts.Block): { + guardsParse: boolean; + awaitsOnlyParse: boolean; + guardCanRaise: boolean; +} { let guardsParse = false; let awaitsOnlyParse = true; + let guardCanRaise = false; const visit = (node: ts.Node) => { if (ts.isFunctionLike(node)) return; if (isParseCall(node)) guardsParse = true; + if (canRaise(node)) guardCanRaise = true; if (ts.isAwaitExpression(node)) { const awaited = unwrap(node.expression); if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; @@ -178,7 +226,7 @@ function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; awaitsOnlyPars ts.forEachChild(node, visit); }; visit(tryBlock); - return { guardsParse, awaitsOnlyParse }; + return { guardsParse, awaitsOnlyParse, guardCanRaise }; } /** Whether some node in the tree rooted at `node` matches `predicate`. */ @@ -267,6 +315,11 @@ function catchBindingName(clause: ts.CatchClause): string | null { return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; } +/** A node's source text with all whitespace removed, for comparing two branch arms. */ +function normalizedText(node: ts.Node): string { + return node.getText().replace(/\s+/g, ""); +} + /** * Whether a conditional expression tests the error to pick what the clause does, rather than to * word what it says. The caller only offers it the whole value of a `return`/`throw`, so @@ -277,36 +330,88 @@ function catchBindingName(clause: ts.CatchClause): string | null { * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding * is not a decision made on the error, and a bindingless catch has nothing here to reference. + * + * The two arms also have to differ, which is the same requirement `selectsADistinctPath` makes of + * an `if`. `return e instanceof Error ? (X) : (X)` is a test whose outcome is the same either way, + * and it was worth 50 points a route; `same-arms-ternary` in the mutation corpus is the tree-scale + * version, and `test/scan.test.ts` has the unit case. Parentheses and whitespace are stripped + * before the comparison, so the shape has to differ in something a reader would call a difference. + * The residual both branch tests share is stated once, on `selectsADistinctPath`. */ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { if (bindingName === null) return false; if (!containsInstanceOf(node.condition)) return false; - return referencesBinding(node.condition, bindingName); + if (!referencesBinding(node.condition, bindingName)) return false; + return normalizedText(unwrap(node.whenTrue)) !== normalizedText(unwrap(node.whenFalse)); } -/** A statement that unconditionally leaves the statement list it sits in, so anything after it in - * the same list never runs. */ -function isDefiniteExit(statement: ts.Statement): boolean { - return ( +/** + * A statement that leaves the statement list it sits in on every path through itself, so anything + * after it in the same list never runs. + * + * Recognises a nested construct, not only a bare `return`/`throw`/`break`/`continue`. Recognising + * only the bare form is what let a dead `throw error;` count as a rethrow when the statement before + * it was a block, a `do` body or an `if`/`else` that returned; `dead-throw-after-*` in the mutation + * corpus is that family, and `test/scan.test.ts` has one case per construct. + * + * A sound under-approximation. `if` without an `else`, a labelled statement (a `break` to the label + * escapes it) and every other loop form answer false, because none of them is guaranteed to run its + * body. Saying false when the truth is true only leaves a later statement in the list, which is the + * direction that withholds evidence rather than inventing it. + */ +function definitelyExits(statement: ts.Statement): boolean { + if ( ts.isReturnStatement(statement) || ts.isThrowStatement(statement) || ts.isContinueStatement(statement) || ts.isBreakStatement(statement) - ); + ) { + return true; + } + if (ts.isBlock(statement)) return statement.statements.some(definitelyExits); + // A `do` body runs before its condition is ever read. + if (ts.isDoStatement(statement)) return definitelyExits(statement.statement); + if (ts.isIfStatement(statement)) { + return ( + statement.elseStatement !== undefined && + definitelyExits(statement.thenStatement) && + definitelyExits(statement.elseStatement) + ); + } + if (ts.isTryStatement(statement)) { + if (statement.finallyBlock && definitelyExits(statement.finallyBlock)) return true; + if (!definitelyExits(statement.tryBlock)) return false; + return statement.catchClause === undefined || definitelyExits(statement.catchClause.block); + } + if (ts.isSwitchStatement(statement)) { + const clauses = statement.caseBlock.clauses; + const last = clauses[clauses.length - 1]; + if (!clauses.some(ts.isDefaultClause) || last === undefined) return false; + // An empty clause falls through to the next one, so it does not have to exit itself; the last + // clause has nothing to fall through to and does. + return ( + clauses.every((c) => c.statements.length === 0 || c.statements.some(definitelyExits)) && + last.statements.some(definitelyExits) + ); + } + return false; } -/** - * `statements` up to and including the first one that definitely exits. Not full flow analysis: - * an `if`/`else` where both branches return is not itself recognised as an exit, only a bare - * `return`, `throw`, `continue` or `break` is. That is enough to make a `throw e;` appended after - * a `return` dead code rather than evidence the clause rethrows, which is the one shape a mutation - * testing this check actually produced. - */ +/** `statements` up to and including the first one that definitely exits. */ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { - const index = statements.findIndex(isDefiniteExit); + const index = statements.findIndex(definitelyExits); return index === -1 ? statements : statements.slice(0, index + 1); } +/** Whether the clause returns anywhere at all, nested functions excluded. A clause with a `return` + * on any path has an exit that is not the throw, so the error does not leave it the way it arrived. + */ +function containsReturn(node: ts.Node): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isReturnStatement(node)) return true; + return ts.forEachChild(node, containsReturn) === true; +} + /** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one * inside a nested function. What separates an arm that takes the error somewhere from an arm that * runs and falls back into the clause's single common exit. */ @@ -323,16 +428,25 @@ function containsExit(node: ts.Node): boolean { * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. * - * Two arms that return the SAME value pass this and should not. That residual is written down in - * the round A fix 2 report rather than defended: telling two returns apart needs the values - * compared, which is a different kind of analysis from anything else here. + * An `if`/`else` whose two arms are textually identical does not count, the same comparison + * `selectsAnErrorPath` makes of a ternary's arms. + * + * The residual both branch tests share, stated here once for both: two arms that produce the same + * outcome by different spellings still read as a real decision. + * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides + * nothing, and so does the `if` with no `else` whose arm returns what the statement after it + * returns. Telling those apart needs the produced values compared for meaning rather than for text, + * which is a different kind of analysis from anything else in this file. The textual comparison is + * the cheapest thing that catches the copy-paste form, which is the one a mutation produces. */ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { if (ts.isIfStatement(statement)) { - return ( - containsExit(statement.thenStatement) || - (statement.elseStatement !== undefined && containsExit(statement.elseStatement)) - ); + const otherwise = statement.elseStatement; + if (otherwise !== undefined) { + if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; + return containsExit(statement.thenStatement) || containsExit(otherwise); + } + return containsExit(statement.thenStatement); } return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); } @@ -345,20 +459,29 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b * `throw` or a test that sits inside an `if`, a loop, a `switch`, a nested `try` or a callback is * not on that path, so it does not count. * - * That is the whole dead-code defence, and it replaces the list of statically-false shapes the - * previous round kept extending. The list was losing: `if (false)` and `while (false)` were + * That is the whole dead-code defence, and it replaces the list of statically-false shapes an + * earlier round kept extending. The list was losing: `if (false)` and `while (false)` were * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without - * naming any of them, and refuses the twelfth nobody has written yet. `dead-*` in the mutation - * corpus is the tree-scale proof, one entry per shape. + * naming any of them. `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. * - * The cost is real: `catch (e) { if (transient) throw e; return null; }` no longer reads as a - * rethrow, so it reads as a swallow and fails rather than sitting out. That is the direction to be - * wrong in, since the reverse hands out points. + * `rethrows` asks for one thing more: that the clause contains no `return` at all. The claim it + * feeds is that the clause passes the error through unchanged, which is only true when throwing is + * the ONLY way out. Without it a `throw error;` written after a statement that already exited read + * as a rethrow, in seven spellings: after a bare block, a `do` body, an `if (true)`, an `if`/`else` + * where both arms return, a `switch` with a returning default, and a `try`/`finally` that returns. + * `definitelyExits` handles most of those on its own, and the no-return rule handles the rest + * without any constant folding. `dead-throw-after-*` in the mutation corpus covers them. + * + * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer + * reads as a rethrow, so it reads as a swallow and fails rather than sitting out, and neither does + * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. + * That is the direction to be wrong in, since the reverse hands out points. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { let rethrows = false; + let returns = false; let branches = false; const bindingName = catchBindingName(clause); @@ -378,12 +501,16 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc continue; } // A `do` body runs before its condition is ever read, so it is on the straight-line path - // whatever the condition says. The only loop form that is. + // whatever the condition says. The only loop form that is; `definitelyExits` agrees. if (ts.isDoStatement(statement)) { const body = statement.statement; walk(ts.isBlock(body) ? body.statements : [body]); continue; } + // Any other reachable statement that could return means throwing is not the only way out. + // Read here rather than over the whole clause so a `return` the walk has already cut as dead + // does not count, which is what a `do { throw e; } while (false); return null;` produces. + if (containsReturn(statement)) returns = true; if (bindingName === null || shadowed) continue; if ( @@ -407,7 +534,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc }; walk(clause.block.statements); - return { rethrows, branches }; + return { rethrows: rethrows && !returns, branches }; } /** @@ -451,11 +578,20 @@ function isAtMostSingletonArray(expr: ts.Expression): boolean { * whole body })` collected it. * * Two things changed. A receiver that is an array literal of one element or none is refused here, - * because it cannot iterate. And the direction that pays no longer pays: `walkBody` counts the - * catches it refuses, and `error-classification` fails a route whose only catches were refused - * rather than excusing it. So a wrong answer here costs precision, not points. That is what makes - * the name list survivable, and it is why `Result.map(...)`, which no name list can tell from - * `users.map(...)`, is a corpus entry that passes rather than a hole. + * because it cannot iterate. And the direction that used to pay no longer pays: `walkBody` counts + * the catches it refuses, and `error-classification` fails a route whose only catches were refused + * rather than excusing it. That is what makes the name list survivable, and it is why + * `Result.map(...)`, which no name list can tell from `users.map(...)`, is a corpus entry that + * passes rather than a hole. + * + * The other direction still costs points and the earlier version of this comment said otherwise. + * A per-item callback under a callee the name list does not know, `pMap(items, cb)` or + * `Array.prototype.map.call(items, cb)`, is attributed to the route, so a per-element catch that + * decides can carry the route to `pass`. No mutation of a real route produces it: the reviewer + * tried `Array.prototype.map.call` over the tree and it moved nothing, because a route has to + * already be iterating for the shape to exist. It is a wrong verdict waiting for a route to be + * written that way, not a laundering path, and it is why this list is worth extending when a new + * iteration helper shows up in the tree. */ function isIterationCallback(node: ts.Node): boolean { const parent = node.parent; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index ff0ad245660..e05d5e7595d 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -13,9 +13,12 @@ export type CheckResult = { */ export type CatchEvidence = { /** - * The clause throws on its own straight-line path: a `throw` among its statements, or among a - * bare nested block's, reached before anything that definitely exits. A throw guarded by an `if`, - * a loop, a `switch`, a nested `try` or a callback does not count, however the guard is spelled. + * Throwing is the clause's only way out. Two conditions: a `throw` is reached on its + * straight-line path (its own statements and a bare nested block's or a `do` body's, cut at the + * first statement that definitely exits, see `definitelyExits`), and the clause contains no + * `return` anywhere. A throw guarded by an `if`, a loop, a `switch`, a nested `try` or a callback + * does not count, however the guard is spelled, and neither does one written after something that + * has already returned. */ rethrows: boolean; /** @@ -36,6 +39,13 @@ export type CatchEvidence = { * its catch. */ guardsParse: boolean; + /** + * The guarded region does something that could raise at all: a call, a construction, an `await`, + * a member access, a `throw`, an iteration, an `instanceof`. A clause whose try block cannot + * raise is unreachable, so it is not error handling and `error-classification` reads no evidence + * off it. See `canRaise` in `scan.ts` for what is not on that list. + */ + guardCanRaise: boolean; /** * Everything the guarded region waits for is one of those parses. What separates * `try { const body = await request.json(); } catch { 400 }` from diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index 21150799ab4..bca5e4f6f4c 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -546,6 +546,63 @@ describe("error-classification", () => { expect(r.detail).toContain("catches nothing"); }); + // S2 at the check level. The evidence tests in `scan.test.ts` pin `guardCanRaise` itself; these + // pin the check reading it, which is where the 50 points were. Prepending this to a route that + // catches nothing took it from not-applicable to pass, and 224 routes were in exactly that state. + it("is not applicable to a route whose only catch guards a try that cannot throw", () => { + const r = run( + "error-classification", + "prepended.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + it("still fails a swallow that a dead classifying catch was prepended to", () => { + const r = run( + "error-classification", + "prepended-swallow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + try { + return json(await prisma.thing.findMany()); + } catch (error) { + return new Response(null, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes the same classifying catch once its try does real work", () => { + const r = run( + "error-classification", + "live.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { await prisma.thing.findMany(); } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a // per-item boundary and the route's own catch is found where it always was. Before this, the // wrapper deleted the route's catches and took a swallow from fail to not-applicable. diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index 032f960fb5e..4db7dfc223a 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -64,5 +64,6 @@ describe("scanning the real webapp routes", () => { expect(after.measured).toBe(before.measured); expect(after.unmeasured).toBe(before.unmeasured); expect(after.global).not.toBeGreaterThan(before.global!); - }); + // Two full tree scans plus a re-scan of every source, which does not fit the suite default. + }, 60_000); }); diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 9c5bc9e468c..5e08f0f3a5a 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -658,11 +658,13 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - // Every shape anyone has found that puts a `throw` somewhere it can never run. The previous - // round recognised the first two by folding the literal `false`, and lost to the other nine. - // None of them is named in the rule now: a throw counts when it is unconditional, and every - // one of these is guarded by something. `dead-*` in the mutation corpus runs the same list over - // the whole route tree. + // Eleven shapes that put a `throw` somewhere it can never run, all of them found by review + // rather than by this suite. An earlier round recognised the first two by folding the literal + // `false` and lost to the other nine. None of them is named in the rule now: a throw counts + // when it is unconditional, and every one of these is guarded by something. There is no claim + // that the list is complete, and a twelfth family arrived the round after it was written, see + // `dead throw written after something that already exited`. `dead-*` in the mutation corpus + // runs the same list over the whole route tree. const DEAD_SHAPES: Array<[string, string]> = [ ["if (false)", "if (false) { throw e; }"], ["while (false)", "while (false) { throw e; }"], @@ -713,6 +715,159 @@ describe("scanFile: catch clause evidence", () => { }); }); + // S1. The other end of the same problem. `reachableStatements` used to cut the statement list + // only on a BARE `return`/`throw`, while the walk descended into blocks and `do` bodies, so a + // `throw e;` written after a nested construct that had already returned was still read as the + // clause rethrowing. Every one of these takes a swallow from `fail` to `not-applicable`, worth 50 + // points a route, and they are semantics-preserving because the throw cannot run. + // + // Two rules answer them together. `definitelyExits` sees through the block, the `do` and the + // `if`/`else`, the `switch` and the `try`/`finally`; the `if (true)` form needs constant folding + // that this file deliberately does not do, and is answered instead by `rethrows` requiring the + // clause to contain no reachable `return` at all. `dead-throw-after-*` in the mutation corpus + // runs all six over the whole route tree. + describe("dead throw written after something that already exited", () => { + const exiting = (wrapped: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${wrapped} + throw e; + } + } + `; + + const EXITED: Array<[string, string]> = [ + ["a bare block", "{ logger.error(e); return null; }"], + ["a do body", "do { return null; } while (false);"], + ["an if (true)", "if (true) { return null; }"], + ["an if/else where both arms return", "if (pick()) { return null; } else { return 0; }"], + ["a switch with a returning default", "switch (1) { default: return null; }"], + ["a try/finally that returns", "try { return null; } finally { }"], + ]; + + for (const [label, wrapped] of EXITED) { + it(`does not set rethrows for a throw after ${label}`, () => { + const ep = scanFile("x.ts", exiting(wrapped)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // The same six wrappers on the branches side, which is what pins `definitelyExits` itself: the + // no-return rule above says nothing about branch credit, so only the cut sees these. An error + // test written after a construct that already returned is dead code and must not read as the + // clause deciding anything. + // + // `an if (true)` is absent from this list on purpose. Nothing here evaluates a condition, so + // the cut cannot see that the wrapper always exits, and the error test after it is still + // credited. That residual is `dead-branch-after-if-true` in the mutation corpus, which runs as + // an expected failure with the two rejected alternatives written out beside it. + const BRANCH_EXITED = EXITED.filter(([label]) => label !== "an if (true)"); + + for (const [label, wrapped] of BRANCH_EXITED) { + it(`does not credit an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${wrapped} + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + it("still credits an error test with nothing exiting before it", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + logger.error(e); + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Positive control: nothing before the throw exits, so the throw is real and the clause has no + // other way out. + it("still sets rethrows when nothing before the throw returns", () => { + const ep = scanFile("x.ts", exiting("logger.error(e);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Second positive control, for the no-return half specifically: a `return` the walk has already + // cut as dead must not count against the rethrow. + it("still sets rethrows when the only return is dead code after the throw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { throw e; return null; } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // S2. A clause whose try block cannot throw is unreachable, so it is not error handling and + // nothing should be read off it. Crediting one was the largest hole ever found here: prepending + // this to a body took the real tree from 15 to 42 and raised 224 routes, because the 261 routes + // that catch nothing sat at `not-applicable` and a dead clause moved every one of them to `pass`. + // `dead-classifying-try` in the mutation corpus is the tree-scale version. + describe("a catch over a try block that cannot throw", () => { + const guarding = (guarded: string) => ` + export async function loader({ request }) { + try { ${guarded} } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return await prisma.thing.findMany(); + } + `; + + const INERT: Array<[string, string]> = [ + ["an empty block", ""], + ["a literal expression statement", "0;"], + ["a literal declaration", "const x = 1;"], + ["arithmetic on literals", "const x = 1 + 2 * 3;"], + ["a bare identifier read", "const x = someLocal;"], + ]; + + for (const [label, guarded] of INERT) { + it(`is not read as error handling when the try holds only ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(false); + }); + } + + // Positive controls, one per reason `canRaise` recognises, so the predicate is not passing the + // cases above by being false for everything. + const LIVE: Array<[string, string]> = [ + ["a call", "doThing();"], + ["a construction", "new Thing();"], + ["an await", "await later;"], + ["a member access", "const x = thing.value;"], + ["an element access", "const x = thing[0];"], + ["a throw", "throw new Error('x');"], + ["an iteration", "for (const item of items) { }"], + ["an instanceof", "const x = thing instanceof Error;"], + ]; + + for (const [label, guarded] of LIVE) { + it(`is read as error handling when the try holds ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(true); + }); + } + }); + it("leaves both flags false when the catch only returns", () => { const ep = scanFile( "swallow.ts", @@ -1097,6 +1252,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: true, awaitsOnlyParse: true, + guardCanRaise: true, tryStatementCount: 1, }); expect(ep!.catches[1]).toMatchObject({ @@ -1199,6 +1355,7 @@ describe("scanFile: per-catch evidence", () => { branches: false, guardsParse: false, awaitsOnlyParse: false, + guardCanRaise: true, tryStatementCount: 4, }); }); @@ -1704,3 +1861,57 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested expect(ep!.catches[0]!.branches).toBe(false); }); }); + +// S3. The ternary path checked only that the condition tested the error, never that the two arms +// went anywhere different, while the `if`/`switch` path had checked exactly that since the round +// before. Rewriting `return X;` as `return e instanceof Error ? (X) : (X)` was therefore worth 50 +// points a route for a change that decides nothing, and it is semantics-preserving. +// `same-arms-ternary` in the mutation corpus is the tree-scale version. +describe("a ternary on the error has to send its arms somewhere different", () => { + const returning = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + return ${value}; + } + } + `; + + it("does not credit a ternary whose arms are identical", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json({})) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in whitespace", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json( {} )) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? ((json({}))) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits a ternary whose arms go somewhere different", () => { + const ep = scanFile( + "x.ts", + returning("error instanceof Response ? error : json({}, { status: 500 })") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // The same comparison on the `if` path, which had the exit test but not the arm test. + it("does not credit an if/else whose two arms are identical", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (error instanceof Error) { return json({}); } else { return json({}); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); From d919ee2567edce07219a01c61604ce8d4a723064 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 20:55:19 +0100 Subject: [PATCH 058/117] test(observability-map): add the additive half of the corpus Every entry was subtractive: delete the catches, wrap the body, merge the statements, splice in dead code. None added fake signal, and both of the shapes that got past the corpus this round lived in that blind spot. Nine entries now cover it, ADDITIVE_IDS names them, and the harness asserts the class is not empty so it cannot quietly go away again. Three other things the corpus was getting wrong. The anti-vacuity guard counted files, which says a rewrite touched a file and not that it reached anything inside it. prependToEveryCatch used to splice at the END of a clause, and 234 of the tree's 260 clauses end in a return or a throw, so eleven entries reported 172 files while landing somewhere that mattered for 26 clauses. The splice moved to the head, where every clause is reachable, and mutations now report the number of sites they edited so the threshold is on something real. The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no verdict anywhere, which is what defended means. Routes are no longer allowed to disappear. risesIn and commonMean both skip a route the mutated scan cannot see, and the entry-point guard tolerated five going missing. It is exact now, with an explicit check that every baseline route is still there. dead-branch-after-if-true is in the corpus as an expected failure. It wraps a catch body in if (true) { ... } and writes a dead error test after it, taking the tree from 15 to 23 and raising 68 routes. Both ways to close it are worse than the hole: folding the literal true starts the list an earlier round lost with false, and cutting branch credit at the first statement that MIGHT exit is sound but accuses 78 real routes and takes the tree to 6. Measured both, chose neither, said so. --- internal-packages/observability-map/README.md | 14 +- .../test/mutationCorpus.test.ts | 79 +++-- .../observability-map/test/mutations.ts | 299 +++++++++++++++--- 3 files changed, 328 insertions(+), 64 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index a058282c727..c62c4ad4360 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -40,7 +40,7 @@ every catch clause in the tree used to score it 100, which meant the metric paid error handling. The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` -applies 30 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +applies 39 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy and asserts three things for each: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a reviewer has found on this @@ -51,8 +51,16 @@ clause in the tree drops the score from 15 to 2, so the metric does not pay you handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves it unchanged, so it does not pay you for adding error handling that does nothing either. -The honest statement is "these 30 rewrites are defended, and here they are", not "unpaddable". The -corpus takes about three minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its +The rewrites come in two directions and both matter. A subtractive one takes real signal away or +moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake +signal in: a classifying catch over a try that cannot throw, a test whose two arms are the same, a +rethrow that can never run. The corpus had only the subtractive half for a while, and the two +largest holes ever found here were both additive. + +The honest statement is "these 38 rewrites are defended, here they are, and here is the one that is +not", not "unpaddable". One entry, `dead-branch-after-if-true`, runs as an expected failure with the +residual written out beside it. The corpus takes about four minutes, so it is gated behind +`OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If you change this package, run it: ```bash diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts index f74076c1d5b..8dd03ee6ded 100644 --- a/internal-packages/observability-map/test/mutationCorpus.test.ts +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { scanDirectory } from "../src/scan.js"; import { buildReport } from "../src/score.js"; -import { MUTATIONS, type Mutation } from "./mutations.js"; +import { ADDITIVE_IDS, MUTATIONS, type Mutation } from "./mutations.js"; /** * The tree-scale mutation corpus. @@ -36,13 +36,20 @@ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; /** - * Where a corpus entry goes when the tool does not defend it. Empty as of round A fix 2: all 30 - * entries hold. When a reviewer finds a shape that cannot be defended, add it to the corpus and - * name it here rather than leaving it out, and write down why in the round's report. `it.fails` - * keeps such an entry running, so closing the hole later turns this file red until the entry is - * moved back out deliberately. + * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, + * so closing the hole later turns this file red until the entry is moved back out deliberately. + * + * `dead-branch-after-if-true` wraps a catch clause's body in `if (true) { ... }` and writes an + * error test after it. The test can never run, and it takes the tree from 15 to 23 and raises 68 + * routes. Nothing in the scanner evaluates a condition, so `definitelyExits` cannot see that the + * wrapper always exits, and the two ways to make it see are both worse than the hole. Folding the + * literal `true` starts the same list an earlier round lost with `false`, where every spelling + * nobody thought of pays. Cutting the branch credit at the first statement that MIGHT exit is + * sound and folding-free, and it was measured: it takes the tree from 15 to 6 and accuses 78 real + * routes, because `catch (e) { if (rare) return null; if (e instanceof X) return y; }` is an + * ordinary shape. See the round A fix 3 report. */ -const KNOWN_GAPS = new Set([]); +const KNOWN_GAPS = new Set(["dead-branch-after-if-true"]); type SourceFile = { relativeName: string; source: string }; @@ -169,15 +176,20 @@ function commonMean(baseline: Measurement, after: Measurement): { before: number return n === 0 ? { before: 0, after: 0 } : { before: sumBefore / n, after: sumAfter / n }; } -function mutate(files: SourceFile[], mutation: Mutation): { files: SourceFile[]; changed: number } { +function mutate( + files: SourceFile[], + mutation: Mutation +): { files: SourceFile[]; changed: number; sites: number } { let changed = 0; + let sites = 0; const out = files.map((file) => { - const source = mutation.apply(file.relativeName, file.source); - if (source === null || source === file.source) return file; + const result = mutation.apply(file.relativeName, file.source); + if (result === null || result.source === file.source) return file; changed++; - return { relativeName: file.relativeName, source }; + sites += result.sites; + return { relativeName: file.relativeName, source: result.source }; }); - return { files: out, changed }; + return { files: out, changed, sites }; } const describeCorpus = ENABLED && existsSync(ROUTES) ? describe : describe.skip; @@ -186,6 +198,15 @@ describeCorpus("mutation corpus over the real route tree", () => { const files = ENABLED && existsSync(ROUTES) ? readTree(ROUTES) : []; const baseline = ENABLED && existsSync(ROUTES) ? measure(files) : null; + it("covers the additive direction, not only the subtractive one", () => { + // Every corpus entry once removed or restructured real signal, and none added fake signal. The + // two largest holes ever found here lived in that blind spot, so the class is asserted rather + // than left to whoever edits the list next. + const ids = new Set(MUTATIONS.map((m) => m.id)); + expect(ADDITIVE_IDS.filter((id) => !ids.has(id))).toEqual([]); + expect(ADDITIVE_IDS.length).toBeGreaterThanOrEqual(8); + }); + it("has a baseline worth mutating", () => { expect(baseline).not.toBeNull(); expect(baseline!.entryPoints).toBeGreaterThan(300); @@ -197,26 +218,42 @@ describeCorpus("mutation corpus over the real route tree", () => { }); /** - * The number of files a mutation must touch before its result means anything. A mutation that - * silently matched nothing would otherwise "pass" by leaving the tree alone, which is the exact - * failure mode that let earlier rounds believe a shape was defended. + * How much a mutation must reach before its result means anything. A mutation that silently + * matched nothing would otherwise "pass" by leaving the tree alone, which is the exact failure + * mode that let earlier rounds believe a shape was defended. + * + * Sites, not only files, and sites are what the threshold is really on. A file count says a + * rewrite touched a file, not that it reached anything inside it: eleven entries reported 172 + * files while landing in a position that mattered for 26 of the tree's 260 catch clauses, because + * the splice went after statements that had already returned. `prependToEveryCatch` now splices + * at the head of the clause, so all 260 count, and this threshold is what would notice if a later + * change quietly took that back. + * + * The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no + * verdict anywhere, and that is precisely what "defended" means, so requiring movement would fail + * every entry that works. Site count is the reachable version of the same intent. */ const MINIMUM_FILES_TOUCHED = 20; + const MINIMUM_SITES_TOUCHED = 40; for (const mutation of MUTATIONS) { const run = KNOWN_GAPS.has(mutation.id) ? it.fails : it; run(`${mutation.kind}: ${mutation.what} (${mutation.id})`, () => { - const { files: mutated, changed } = mutate(files, mutation); + const { files: mutated, changed, sites } = mutate(files, mutation); expect(changed).toBeGreaterThanOrEqual(MINIMUM_FILES_TOUCHED); + expect(sites).toBeGreaterThanOrEqual(MINIMUM_SITES_TOUCHED); const after = measure(mutated); - // A mutation that stops the tree parsing, or that hides most of the routes from the scanner, - // has not tested the property: whatever the score does afterwards is measuring a different - // tree. Both guards fail loudly rather than letting such a mutation report a pass. + // A mutation that stops the tree parsing, or that hides a route from the scanner, has not + // tested the property: whatever the score does afterwards is measuring a different tree. The + // route guard is exact rather than tolerant, because a route the mutated scan cannot see is + // one `risesIn` and `commonMean` both skip, and a rewrite that makes a route unscannable is + // itself a finding. expect(after.parseFailures).toBe(baseline!.parseFailures); - expect(after.entryPoints).toBeGreaterThanOrEqual(baseline!.entryPoints - 5); + expect(after.entryPoints).toBe(baseline!.entryPoints); + expect([...baseline!.perEntry.keys()].filter((f) => !after.perEntry.has(f))).toEqual([]); const rises = risesIn(baseline!, after); const common = commonMean(baseline!, after); @@ -224,7 +261,7 @@ describeCorpus("mutation corpus over the real route tree", () => { `[corpus] ${mutation.id}: global ${baseline!.global} -> ${after.global} ` + `(mean ${baseline!.exactMean.toFixed(3)} -> ${after.exactMean.toFixed(3)}, ` + `common mean ${common.before.toFixed(3)} -> ${common.after.toFixed(3)}, ` + - `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, ` + + `measured ${baseline!.measured} -> ${after.measured}, files ${changed}, sites ${sites}, ` + `routes raised ${rises.length})` + rises .slice(0, 3) diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts index 4b8f6df4671..894dd8da9d4 100644 --- a/internal-packages/observability-map/test/mutations.ts +++ b/internal-packages/observability-map/test/mutations.ts @@ -16,6 +16,14 @@ import ts from "typescript"; * - `deleting`: the rewrite removes error handling or logging. The route is worse afterwards, so * the score must not rise either, for a different and simpler reason. * + * Within `preserving` there are two directions, and for a long time the corpus only had one of + * them. A subtractive rewrite takes real signal away or moves it about: delete the catches, wrap + * the body, merge the statements. An ADDITIVE rewrite puts fake signal in: a classifying catch over + * a try that cannot throw, a test whose two arms are the same, a rethrow that can never run. The + * additive direction is the one someone reaches for when a CI comment nags them, and it is where + * the two largest holes ever found here lived. `ADDITIVE_IDS` lists the entries that cover it, and + * `mutationCorpus.test.ts` asserts the class is not empty so it cannot quietly go away again. + * * Neither kind is ever executed. "Semantics-preserving" here means preserving the observable * behaviour of the route as written, which is what the scanner claims to measure; it is not a * claim that the mutated tree compiles against its real types. @@ -23,13 +31,21 @@ import ts from "typescript"; export type MutationKind = "preserving" | "deleting"; +/** + * The result of rewriting one file: the new source, and how many places in it the rewrite actually + * landed. `sites` is what the corpus's anti-vacuity guard reads. A file count says nothing about + * whether the rewrite reached anything, and a mutation that quietly matched two constructs in a + * file would otherwise look identical to one that matched forty. + */ +export type MutationResult = { source: string; sites: number }; + export type Mutation = { id: string; kind: MutationKind; /** What the rewrite does, in one line, for the corpus table in the report. */ what: string; - /** The mutated source, or null when this file has nothing for the mutation to touch. */ - apply(fileName: string, source: string): string | null; + /** The mutated file, or null when this file has nothing for the mutation to touch. */ + apply(fileName: string, source: string): MutationResult | null; }; type Edit = { start: number; end: number; text: string }; @@ -51,7 +67,7 @@ function parse(fileName: string, source: string): ts.SourceFile { * deletes a catch clause and one that rewrites a statement inside that clause would otherwise * produce overlapping splices. Dropping the inner one is what "the outer rewrite won" means. */ -function applyEdits(source: string, edits: Edit[]): string | null { +function applyEdits(source: string, edits: Edit[]): MutationResult | null { if (edits.length === 0) return null; const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end); const kept: Edit[] = []; @@ -65,7 +81,7 @@ function applyEdits(source: string, edits: Edit[]): string | null { const edit = kept[i]!; out = out.slice(0, edit.start) + edit.text + out.slice(edit.end); } - return out === source ? null : out; + return out === source ? null : { source: out, sites: kept.length }; } function insert(at: number, text: string): Edit { @@ -214,11 +230,17 @@ function bindingNameOf(clause: ts.CatchClause): string | null { } /** - * Append a statement at the end of every catch clause that names its binding. `snippet` receives - * the binding name. Appending is the position that matters: a shape spliced in after a `return` is - * already unreachable and proves nothing. + * Splice a statement in at the HEAD of every catch clause that names its binding. `snippet` + * receives the binding name. + * + * The head, not the tail, and that is the whole point of the helper. Appending put the shape after + * whatever the clause already did, and 234 of the tree's 260 clauses end in a `return` or a + * `throw`, so in those the spliced shape was dead by ordering before the rule under test ever + * looked at it: eleven corpus entries reported touching 172 files while exercising 26 clauses. At + * the head every clause is reachable, so every clause exercises the rule. The shapes spliced this + * way are dead wherever they sit, so moving them does not make the rewrite any less preserving. */ -function appendToEveryCatch( +function prependToEveryCatch( id: string, kind: MutationKind, what: string, @@ -234,7 +256,73 @@ function appendToEveryCatch( for (const clause of catchClauses(sf)) { const binding = bindingNameOf(clause); if (binding === null) continue; - edits.push(insert(clause.block.end - 1, `\n${snippet(binding)}\n`)); + edits.push(insert(clause.block.getStart() + 1, `\n${snippet(binding)}\n`)); + } + return applyEdits(source, edits); + }, + }; +} + +/** Whether a statement list ends in a way that makes anything spliced in after it dead. Used only + * to keep the dead-throw mutations honest: appending `throw e;` after statements that might fall + * through would change what the route does, and this corpus is not allowed to do that. */ +function endsInAnExit(statements: readonly ts.Statement[]): boolean { + const last = statements[statements.length - 1]; + return last !== undefined && (ts.isReturnStatement(last) || ts.isThrowStatement(last)); +} + +/** Whether the tree rooted at `node` contains a `break` or `continue` outside a nested function or + * a loop of its own. Wrapping such statements in a `do` or a `switch` would rebind them. */ +function containsLooseJump(node: ts.Node): boolean { + let found = false; + const visit = (n: ts.Node) => { + if (found) return; + if (ts.isFunctionLike(n)) return; + if ( + ts.isForStatement(n) || + ts.isForOfStatement(n) || + ts.isForInStatement(n) || + ts.isWhileStatement(n) || + ts.isDoStatement(n) || + ts.isSwitchStatement(n) + ) { + return; + } + if (ts.isBreakStatement(n) || ts.isContinueStatement(n)) found = true; + ts.forEachChild(n, visit); + }; + ts.forEachChild(node, visit); + return found; +} + +/** + * Wrap every catch clause's body in a construct that definitely exits, then write `throw e;` after + * it. The throw can never run, and before `definitelyExits` learned to see through the wrapper each + * of these read as the clause rethrowing, which is `not-applicable` instead of `fail` and worth 50 + * points a route. + * + * Only applied to a clause whose statements already end in a `return` or a `throw`, so the appended + * throw really is unreachable, and never to one holding a loose `break` or `continue`, which a `do` + * or a `switch` would capture. + */ +function deadThrowAfter(id: string, what: string, wrap: (body: string) => string): Mutation { + return { + id, + kind: "preserving", + what, + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const statements = clause.block.statements; + if (statements.length === 0 || !endsInAnExit(statements)) continue; + if (containsLooseJump(clause.block)) continue; + const first = statements[0]!.getStart(); + const last = statements[statements.length - 1]!.end; + const body = source.slice(first, last); + edits.push({ start: first, end: last, text: `${wrap(body)}\nthrow ${binding};` }); } return applyEdits(source, edits); }, @@ -266,7 +354,7 @@ function prependToEveryFile(id: string, what: string, text: string): Mutation { kind: "preserving", what, apply(_fileName, source) { - return `${text}\n${source}`; + return { source: `${text}\n${source}`, sites: 1 }; }, }; } @@ -347,7 +435,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text begins with a // directive", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationA() {\n return

// obs-map-disable error-classification -- mutation corpus

;\n}\n`, + sites: 1, + }; }, }, { @@ -356,7 +447,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text starts a // directive right after an expression container", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationB({ name }: { name: string }) {\n return

{name}// obs-map-disable request-context -- mutation corpus

;\n}\n`, + sites: 1, + }; }, }, { @@ -365,7 +459,10 @@ export const MUTATIONS: Mutation[] = [ what: "add a component whose JSX text is a /* */ directive", apply(fileName, source) { if (!fileName.endsWith(".tsx")) return null; - return `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`; + return { + source: `${source}\nexport function ObsMapMutationC() {\n return

/* obs-map-disable audit-trail -- mutation corpus */

;\n}\n`, + sites: 1, + }; }, }, @@ -435,7 +532,7 @@ export const MUTATIONS: Mutation[] = [ { id: "throw-after-return-in-catch", kind: "preserving", - what: "append throw e; after the first return in every catch", + what: "splice throw e; after the first return in every catch", apply(fileName, source) { const sf = parse(fileName, source); const edits: Edit[] = []; @@ -450,85 +547,188 @@ export const MUTATIONS: Mutation[] = [ }, }, - appendToEveryCatch( + prependToEveryCatch( "dead-if-false", "preserving", - "append if (false) { throw e; } to every catch", + "splice if (false) { throw e; } into every catch", (e) => `if (false) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-while-false", "preserving", - "append while (false) { throw e; } to every catch", + "splice while (false) { throw e; } into every catch", (e) => `while (false) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-false", "preserving", - "append for (;false;) { throw e; } to every catch", + "splice for (;false;) { throw e; } into every catch", (e) => `for (;false;) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-true-else", "preserving", - "append if (true) { 0; } else { throw e; } to every catch", + "splice if (true) { 0; } else { throw e; } into every catch", (e) => `if (true) { 0; } else { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-switch-no-case", "preserving", - "append switch (1) { case 2: throw e; } to every catch", + "splice switch (1) { case 2: throw e; } into every catch", (e) => `switch (1) { case 2: throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-inner-try", "preserving", - "append try { 0; } catch { throw e; } to every catch", + "splice try { 0; } catch { throw e; } into every catch", (e) => `try { 0; } catch { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-of-empty", "preserving", - "append for (const x of []) { throw e; } to every catch", + "splice for (const x of []) { throw e; } into every catch", (e) => `for (const obsMapItem of []) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-for-in-empty", "preserving", - "append for (const k in {}) { throw e; } to every catch", + "splice for (const k in {}) { throw e; } into every catch", (e) => `for (const obsMapKey in {}) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-empty-string", "preserving", 'append if ("") { throw e; } to every catch', (e) => `if ("") { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-not-true", "preserving", - "append if (!true) { throw e; } to every catch", + "splice if (!true) { throw e; } into every catch", (e) => `if (!true) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "dead-if-const-compare", "preserving", - "append if (1 === 2) { throw e; } to every catch", + "splice if (1 === 2) { throw e; } into every catch", (e) => `if (1 === 2) { throw ${e}; }` ), - appendToEveryCatch( + prependToEveryCatch( "registered-throw", "preserving", - "append [].push(() => { throw e; }) to every catch", + "splice [].push(() => { throw e; }) into every catch", (e) => `[].push(() => { throw ${e}; });` ), - appendToEveryCatch( + prependToEveryCatch( "empty-instanceof-if", "preserving", - "append if (e instanceof Error) { } to every catch", + "splice if (e instanceof Error) { } into every catch", (e) => `if (${e} instanceof Error) { }` ), + // The additive class. Everything above either takes signal away or moves it about; these put in + // signal that is not real, which is the direction the corpus was blind to. + { + id: "dead-classifying-try", + kind: "preserving", + what: "prepend a classifying try/catch over a try block that cannot throw", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push( + insert( + body.getStart() + 1, + "\ntry { 0; } catch (obsMapDead) {" + + " if (obsMapDead instanceof Error) { return new Response(null, { status: 400 }); }" + + " throw obsMapDead; }\n" + ) + ); + } + return applyEdits(source, edits); + }, + }, + { + id: "same-arms-ternary", + kind: "preserving", + what: "rewrite a catch's return value as a ternary on the error with identical arms", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + for (const statement of clause.block.statements) { + if (!ts.isReturnStatement(statement) || !statement.expression) continue; + const value = source.slice(statement.expression.getStart(), statement.expression.end); + edits.push({ + start: statement.expression.getStart(), + end: statement.expression.end, + text: `${binding} instanceof Error ? (${value}) : (${value})`, + }); + } + } + return applyEdits(source, edits); + }, + }, + deadThrowAfter( + "dead-throw-after-block", + "wrap every catch body in a bare block and write throw e; after it", + (body) => `{\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-do", + "wrap every catch body in do { ... } while (false) and write throw e; after it", + (body) => `do {\n${body}\n} while (false);` + ), + deadThrowAfter( + "dead-throw-after-if-true", + "wrap every catch body in if (true) { ... } and write throw e; after it", + (body) => `if (true) {\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-if-else", + "wrap every catch body in both arms of an if/else and write throw e; after it", + (body) => `if (obsMapPick()) {\n${body}\n} else {\n${body}\n}` + ), + deadThrowAfter( + "dead-throw-after-switch", + "wrap every catch body in a switch default and write throw e; after it", + (body) => `switch (1) { default: {\n${body}\n} }` + ), + deadThrowAfter( + "dead-throw-after-try-finally", + "wrap every catch body in try { ... } finally { } and write throw e; after it", + (body) => `try {\n${body}\n} finally { }` + ), + { + id: "dead-branch-after-if-true", + kind: "preserving", + what: "wrap every catch body in if (true) { ... } and write a dead error test after it", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const clause of catchClauses(sf)) { + const binding = bindingNameOf(clause); + if (binding === null) continue; + const statements = clause.block.statements; + if (statements.length === 0 || !endsInAnExit(statements)) continue; + if (containsLooseJump(clause.block)) continue; + const first = statements[0]!.getStart(); + const last = statements[statements.length - 1]!.end; + const body = source.slice(first, last); + edits.push({ + start: first, + end: last, + text: + `if (true) {\n${body}\n}\n` + + `if (${binding} instanceof Error) { return new Response(null, { status: 400 }); }`, + }); + } + return applyEdits(source, edits); + }, + }, + { id: "merge-declarations", kind: "preserving", @@ -577,7 +777,7 @@ export const MUTATIONS: Mutation[] = [ { id: "inert-statements-after-try", kind: "preserving", - what: "append five unused const declarations after every try statement in a route body", + what: "splice five unused const declarations after every try statement in a route body", apply(fileName, source) { const sf = parse(fileName, source); const edits: Edit[] = []; @@ -595,6 +795,25 @@ export const MUTATIONS: Mutation[] = [ }, ]; +/** + * The entries that add fake signal rather than removing or restructuring real signal. Named so + * `mutationCorpus.test.ts` can assert the class exists: the corpus went three rounds with this half + * of the property untested, and an empty list here is exactly that state coming back. + */ +export const ADDITIVE_IDS = [ + "dead-classifying-try", + "same-arms-ternary", + "dead-throw-after-block", + "dead-throw-after-do", + "dead-throw-after-if-true", + "dead-throw-after-if-else", + "dead-throw-after-switch", + "dead-throw-after-try-finally", + "wrap-body-in-rethrow", + "empty-instanceof-if", + "registered-throw", +]; + function isSingleConst(statement: ts.Statement): statement is ts.VariableStatement { return ( ts.isVariableStatement(statement) && From a11b6616b667a39ac7987b18f815f131033ab16b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 21:40:08 +0100 Subject: [PATCH 059/117] fix(observability-map): cut a dead error test at the right moment The exit cut for branch credit was rejected last round on a measurement of the wrong thing. The variant measured raised the exit flag before checking the current statement, so every deciding statement refused itself: `if (e instanceof X) return y` contains an exit by definition. That variant does take the tree from 15 to 6 and accuse 78 routes, and it is not the rule. Raising the flag after the statement's own branch check leaves the real-tree report and all 240 clauses' evidence byte-identical, verified by comparing the full report and every CatchEvidence field before and after. It closes dead-branch-after-if-true, which went from 15 to 23 with 68 routes raised and now raises none, and it closes the labelled-block, for-of and while members of the same family. None of them needs a condition evaluated, so that claim is gone from the corpus and from the docstrings. The precision given up is a conditional exit before the error test, which also stops the credit. No clause in the tree is that shape, which is why the report does not move, and the case is pinned as a test so it stays a decision. Also two false statements the check could make. A clause that throws for some errors was told it takes one way out regardless of what was thrown. It does not. The strengthened rethrows makes such a clause a swallow by this check's definition, since it decides nothing about the error, and 16 clauses flipped rethrows this round. CatchEvidence now carries the raw throw alongside the strengthened one and the wording follows it. Whether fail is the right verdict for those clauses is a separate question, parked. A route that owns a real classifying catch could be told it owns none. canRaise does not list destructuring and `const { a } = undefined` throws, so the owned catch dropped out of the reachable set and the callback branch, ordered off that set, accused the route of keeping all its error handling in a callback. It reads ep.catches now. --- .../src/checks/errorClassification.ts | 39 +++++++--- .../observability-map/src/scan.ts | 74 ++++++++++++------- .../observability-map/src/types.ts | 11 ++- .../observability-map/test/checks.test.ts | 60 +++++++++++++++ .../observability-map/test/scan.test.ts | 48 +++++++++--- 5 files changed, 186 insertions(+), 46 deletions(-) diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 22a431ab764..c380b80a946 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -152,13 +152,23 @@ export function usesBuilder(ep: EntryPoint): boolean { * `Promise.all([0].map(async () => { ... }))` did. It fails instead. The precision cost is a route * that genuinely only handles errors per item, which now fails rather than sitting out. * - * A clause whose try block cannot raise is read as no clause at all, `guardCanRaise` on the - * evidence. It is not error handling, and crediting one was the largest hole ever found here: - * prepending `try { 0; } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } - * throw e; }` to every body took the tree from 15 to 42 and raised 224 routes, because the 261 - * routes that catch nothing were sitting at not-applicable and a dead clause moved each of them to - * pass. Dropping it here rather than in the scan keeps the evidence honest about what is written - * and puts the judgement where the other judgements are. + * A clause whose try block holds nothing that could raise is read as no clause at all, + * `guardCanRaise` on the evidence. Prepending `try { 0; } catch (e) { if (e instanceof Error) { + * return json(x, { status: 400 }); } throw e; }` to every body took the tree from 15 to 42 and + * raised 224 routes, because the 261 routes that catch nothing were sitting at not-applicable and a + * dead clause moved each of them to pass. + * + * What that refuses is `try { 0; }`, and it is defeated by one inert call: `try { String(0); }` + * reads as classification and pays the same 224 routes, because `canRaise` accepts any call at all. + * The rule closes the shape that was found, not the family, and telling an inert call from a + * throwing one needs types the scanner does not have. `dead-classifying-try-with-call` in the + * mutation corpus is the open shape, running as an expected failure. + * + * The filter also has to run BEFORE nothing. Ordering the callback branch off `reachable` rather + * than `ep.catches` accused a route that owns a real classifying catch of owning none, whenever + * `canRaise` missed what that catch guarded: a destructuring declaration is not on its list, so + * `try { const { a } = undefined; } catch (e) { ... }` beside a per-item `.map` catch failed with + * "its only error handling sits in a callback the route does not own", which was simply untrue. */ export const errorClassification = { id: ID, @@ -171,13 +181,24 @@ export const errorClassification = { if (swallowed.length > 0) { const which = reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; + // "One way out" is only true of a clause that never throws. A clause holding a `throw` that + // is not its only exit is a swallow by this check's definition (it decides nothing about the + // error) and it is NOT one way out, so saying so was a false accusation. 16 clauses in the + // tree changed `rethrows` from true to false this round and every one of them would have + // been eligible for it. + const everyWayOut = swallowed.every((c) => !c.throws); return { id: ID, status: "fail", - detail: `catches its errors and takes one way out regardless of what was thrown${which}`, + detail: everyWayOut + ? `catches its errors and takes one way out regardless of what was thrown${which}` + : `catches its errors and chooses what to do without looking at what was thrown${which}`, }; } - if (reachable.length === 0 && ep.callbackCatches > 0) { + // Read off `ep.catches`, not `reachable`: a route that owns a catch owns one, whether or not + // `canRaise` could see what it guarded. Ordering this off `reachable` turned every `canRaise` + // miss on a route that also has a per-item catch into an accusation that was flatly false. + if (ep.catches.length === 0 && ep.callbackCatches > 0) { return { id: ID, status: "fail", diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index eaf14cd4aa6..5521833a382 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -196,13 +196,20 @@ function canRaise(node: ts.Node): boolean { * to 42 and raised 224 routes, which is more than every other shape found on this branch put * together. `dead-classifying-try` in the mutation corpus is the tree-scale version. * - * The alternative considered was proving unreachability the other way round, by ruling out every - * expression that could throw. It is the same predicate read backwards and it fails the same way, - * so the cheaper direction won: ask what the block DOES, and require it to do something. The - * residual is what `canRaise` does not list. A temporal-dead-zone read (`try { const x = later; }`), - * a coercion that raises (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object are - * all treated as unable to raise. None is expressible as a preserving rewrite of a real route, and - * all of them would need types the scanner does not have. + * What this refuses is `try { 0; }` and nothing cleverer. `canRaise` accepts ANY call, member + * access or `in`, and none of those has to be able to throw, so one inert call defeats the rule: + * `try { String(0); } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } throw + * e; }` reads as classification and takes the tree back to 15 to 42, exactly as `try { 0; }` did. + * `dead-classifying-try-with-call` in the mutation corpus is that shape, running as an expected + * failure. Telling a call that can throw from one that cannot needs types the scanner does not have, + * so the rule closes the shape found rather than the family it belongs to. Read the docstrings that + * point here as "refuses `try { 0; }`", never as "an unreachable catch cannot be credited". + * + * The list also misses things that CAN raise, which is the safe direction, and the misses matter + * because a real clause can be dropped by one: a destructuring declaration (`const { a } = undefined` + * throws), a temporal-dead-zone read (`try { const x = later; }`), a coercion that raises + * (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object all read as unable to + * raise. * * Nested function bodies are skipped throughout: a callback written inside the try is not work the * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. @@ -479,10 +486,22 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. * That is the direction to be wrong in, since the reverse hands out points. */ -function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branches: boolean } { +function catchClauseEvidence(clause: ts.CatchClause): { + rethrows: boolean; + throws: boolean; + branches: boolean; +} { let rethrows = false; let returns = false; let branches = false; + // Set once a statement the walk has already passed could have left the clause. An error test + // after one of those is dead code, so it decides nothing. Raised at the END of each statement, + // after that statement's own branch check: a deciding statement contains an exit by definition, + // so raising it first makes every such statement refuse itself, which was measured at 78 routes + // losing their pass and the tree dropping from 15 to 6. This ordering leaves the real-tree report + // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written + // after something that already exited`. + let exited = false; const bindingName = catchBindingName(clause); const walk = (statements: readonly ts.Statement[]) => { @@ -494,10 +513,12 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc for (const statement of reachableStatements(statements)) { if (ts.isThrowStatement(statement)) { rethrows = true; + exited = true; continue; } if (ts.isBlock(statement)) { walk(statement.statements); + if (containsExit(statement)) exited = true; continue; } // A `do` body runs before its condition is ever read, so it is on the straight-line path @@ -505,36 +526,38 @@ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; branc if (ts.isDoStatement(statement)) { const body = statement.statement; walk(ts.isBlock(body) ? body.statements : [body]); + if (containsExit(statement)) exited = true; continue; } // Any other reachable statement that could return means throwing is not the only way out. // Read here rather than over the whole clause so a `return` the walk has already cut as dead // does not count, which is what a `do { throw e; } while (false); return null;` produces. if (containsReturn(statement)) returns = true; - if (bindingName === null || shadowed) continue; - - if ( - (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && - referencesBinding(statement.expression, bindingName) && - selectsADistinctPath(statement) - ) { - branches = true; - continue; - } - if ( - (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && - statement.expression !== undefined - ) { - const value = unwrap(statement.expression); - if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + + if (bindingName !== null && !shadowed && !exited) { + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { branches = true; + } else if ( + (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) && + statement.expression !== undefined + ) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + branches = true; + } } } + + if (containsExit(statement)) exited = true; } }; walk(clause.block.statements); - return { rethrows: rethrows && !returns, branches }; + return { rethrows: rethrows && !returns, throws: rethrows, branches }; } /** @@ -1008,6 +1031,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const clause = catchClauseEvidence(node.catchClause); catches.push({ rethrows: clause.rethrows, + throws: clause.throws, branches: clause.branches, ...guardedWork(node.tryBlock), tryStatementCount, diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index e05d5e7595d..3ce969494f4 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -21,6 +21,10 @@ export type CatchEvidence = { * has already returned. */ rethrows: boolean; + /** A `throw` is reached on that same straight-line path, whether or not it is the only way out. + * `rethrows` is this AND no reachable `return`. Kept separately so a verdict can say what is true + * of a clause that both throws and returns. */ + throws: boolean; /** * The clause picks what to do from what it caught, on that same straight-line path: an `if` or * `switch` whose condition references the caught error binding AND at least one of whose arms @@ -41,9 +45,10 @@ export type CatchEvidence = { guardsParse: boolean; /** * The guarded region does something that could raise at all: a call, a construction, an `await`, - * a member access, a `throw`, an iteration, an `instanceof`. A clause whose try block cannot - * raise is unreachable, so it is not error handling and `error-classification` reads no evidence - * off it. See `canRaise` in `scan.ts` for what is not on that list. + * a member access, a `throw`, an iteration, an `instanceof`. False means `try { 0; }` and little + * else: any call counts, including one that cannot throw, so `try { String(0); }` reads as true. + * See `canRaise` in `scan.ts` for both directions of that, including the destructuring + * declaration it misses. */ guardCanRaise: boolean; /** diff --git a/internal-packages/observability-map/test/checks.test.ts b/internal-packages/observability-map/test/checks.test.ts index bca5e4f6f4c..b10d6c44e85 100644 --- a/internal-packages/observability-map/test/checks.test.ts +++ b/internal-packages/observability-map/test/checks.test.ts @@ -603,6 +603,66 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); + // I3. "Takes one way out regardless of what was thrown" is false of a clause that throws for + // some errors, and the strengthened `rethrows` makes such a clause a swallow by this check's + // definition: it decides nothing about the error, but it does not send everything the same way + // either. 16 clauses in the tree flipped `rethrows` this round and every one was eligible for the + // false wording. Whether `fail` is the right verdict for them is a separate question, parked. + it("does not accuse a clause that throws of taking one way out", () => { + const r = run( + "error-classification", + "mixed.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { if (rare) { return null; } throw e; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("without looking at what was thrown"); + expect(r.detail).not.toContain("one way out"); + }); + + it("still says one way out for a clause that never throws", () => { + const r = run( + "error-classification", + "swallow.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { logger.error(e); return null; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("one way out"); + }); + + // I4. A route that owns a real classifying catch must never be told it owns none. `canRaise` does + // not list destructuring, and `const { a } = undefined` throws, so the owned catch dropped out of + // `reachable`; with the callback branch ordered off `reachable` the route was then accused of + // having all its error handling in a callback, which was simply false. + it("does not accuse a route that owns a catch of owning none", () => { + const r = run( + "error-classification", + "owned.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.detail).not.toContain("callback the route does not own"); + }); + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a // per-item boundary and the route's own catch is found where it always was. Before this, the // wrapper deleted the route's catches and took a swallow from fail to not-applicable. diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index 5e08f0f3a5a..c10f6d2af66 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -754,16 +754,26 @@ describe("scanFile: catch clause evidence", () => { }); } - // The same six wrappers on the branches side, which is what pins `definitelyExits` itself: the - // no-return rule above says nothing about branch credit, so only the cut sees these. An error - // test written after a construct that already returned is dead code and must not read as the - // clause deciding anything. + // The same wrappers on the branches side, plus three the rethrow list has no use for. An error + // test written after a statement that could already have left the clause is dead code and must + // not read as the clause deciding anything. // - // `an if (true)` is absent from this list on purpose. Nothing here evaluates a condition, so - // the cut cannot see that the wrapper always exits, and the error test after it is still - // credited. That residual is `dead-branch-after-if-true` in the mutation corpus, which runs as - // an expected failure with the two rejected alternatives written out beside it. - const BRANCH_EXITED = EXITED.filter(([label]) => label !== "an if (true)"); + // This asks a weaker question than `definitelyExits` does, and on purpose: "could this have + // exited", not "must it have". That is why `if (true)`, a labelled block, a `for...of` and a + // `while` are all on the list even though none of them is guaranteed to run its body. Nothing + // here evaluates a condition, and none of these needs one evaluated. + // + // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the + // END of each statement, after that statement's own branch check. Raising it first makes every + // deciding statement refuse itself, because `if (e instanceof X) return y` contains an exit by + // definition; that variant was measured and it takes the tree from 15 to 6 and accuses 78 + // routes. This one leaves the real-tree report and all 240 clauses' evidence byte-identical. + const BRANCH_EXITED: Array<[string, string]> = [ + ...EXITED, + ["a labelled block", "outer: { return null; }"], + ["a for...of that returns", "for (const q of items) { return q; }"], + ["a while that returns", "while (go) { return null; }"], + ]; for (const [label, wrapped] of BRANCH_EXITED) { it(`does not credit an error test written after ${label}`, () => { @@ -781,6 +791,24 @@ describe("scanFile: catch clause evidence", () => { }); } + // The precision this gives up, pinned so it is a decision and not a surprise: a conditional + // exit before the error test also stops the credit, because the walk cannot tell a guard that + // usually falls through from one that always leaves. No clause in the route tree is this shape, + // which is why the report is byte-identical, but one could be written tomorrow. + it("does not credit an error test written after a conditional return", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (rare) { return null; } + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + it("still credits an error test with nothing exiting before it", () => { const ep = scanFile( "x.ts", @@ -1250,6 +1278,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches[0]).toEqual({ rethrows: false, branches: false, + throws: false, guardsParse: true, awaitsOnlyParse: true, guardCanRaise: true, @@ -1353,6 +1382,7 @@ describe("scanFile: per-catch evidence", () => { expect(ep!.catches[0]).toEqual({ rethrows: false, branches: false, + throws: false, guardsParse: false, awaitsOnlyParse: false, guardCanRaise: true, From 3e1ee436ccab6fc9bda8b122524b19e984ecc292 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 21:40:30 +0100 Subject: [PATCH 060/117] docs(observability-map): say what the unreachable-catch rule actually refuses It refuses `try { 0; }`. Five places said or implied it refuses an unreachable catch, which is a family, and canRaise accepts any call, member access or `in`, none of which has to be able to throw. `try { String(0); }` therefore reads as real classification and pays the same 224 routes and the same 15 to 42 that `try { 0; }` did before it was refused. Telling an inert call from one that can throw needs types the scanner does not have, so the shape is parked rather than chased: dead-classifying-try-with-call is in the corpus as an expected failure, and scan.ts, types.ts, errorClassification.ts, the README and the corpus entry all now say refuses `try { 0; }` and is defeated by one call. canRaise also misses things that can raise, which drops a real clause rather than crediting a fake one. Destructuring is now named in that residual, since it is the miss that produced the false accusation fixed alongside this. Two more claims that ran ahead of the code. The corpus said a defended mutation moves no verdict anywhere, which several defended entries plainly disprove; the true reason movement cannot be required is narrower, that the ideal defended shape is one the scanner is blind to and those move nothing. And the docstring checker collects leading comment ranges only, so a comment with no node after it is never scanned; that is now in its list of what it does not check, alongside the other three. --- internal-packages/observability-map/README.md | 14 +++++--- .../test/docstringReferences.test.ts | 16 ++++++--- .../test/mutationCorpus.test.ts | 33 +++++++++++-------- .../observability-map/test/mutations.ts | 21 ++++++++++++ 4 files changed, 62 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index c62c4ad4360..6f864877490 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -40,7 +40,7 @@ every catch clause in the tree used to score it 100, which meant the metric paid error handling. The property behind that is now a test corpus rather than a claim. `test/mutationCorpus.test.ts` -applies 39 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy +applies 40 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy and asserts three things for each: the published global does not rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a reviewer has found on this @@ -53,13 +53,17 @@ not pay you for adding error handling that does nothing either. The rewrites come in two directions and both matter. A subtractive one takes real signal away or moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake -signal in: a classifying catch over a try that cannot throw, a test whose two arms are the same, a +signal in: a classifying catch over a try that does nothing, a test whose two arms are the same, a rethrow that can never run. The corpus had only the subtractive half for a while, and the two largest holes ever found here were both additive. -The honest statement is "these 38 rewrites are defended, here they are, and here is the one that is -not", not "unpaddable". One entry, `dead-branch-after-if-true`, runs as an expected failure with the -residual written out beside it. The corpus takes about four minutes, so it is gated behind +One of those is still open and the corpus says so. A catch over `try { 0; }` is refused, but +`canRaise` accepts any call, so `try { String(0); }` reads as real error handling and pays the same +224 routes. Telling an inert call from one that can throw needs types the scanner does not have. + +The honest statement is "these 39 rewrites are defended, here they are, and here is the one that is +not", not "unpaddable". One entry, `dead-classifying-try-with-call`, runs as an expected failure +with the residual written out beside it. The corpus takes about four minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If you change this package, run it: diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts index 3800066b2c4..51bfefa5d16 100644 --- a/internal-packages/observability-map/test/docstringReferences.test.ts +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -21,10 +21,18 @@ import { MUTATIONS } from "./mutations.js"; * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like * and what a code sample does not. * - * What is NOT checked: a reference written without backticks, a test title of fewer than - * `MINIMUM_TITLE_WORDS` words (`throw e` and `new URL` are code, and telling a short title from - * short code needs more than punctuation), and anything outside `src/`. A docstring can still name - * a nonexistent short test. The kebab half is the half that has actually failed. + * What is NOT checked, and each of these is a place a bad reference can still hide: + * + * - a reference written without backticks. + * - a test title of fewer than `MINIMUM_TITLE_WORDS` words. `throw e` and `new URL` are code, and + * telling a short title from short code needs more than punctuation. + * - a comment with no node after it. `commentText` collects leading ranges only, so a comment on + * the last line of a block or at the end of a file is never scanned at all. Every docstring in + * this package precedes a declaration, which is why the collector was written that way, and it + * is a coverage hole rather than a design choice. + * - anything outside `src/`, including the docstrings in this file and in `mutations.ts`. + * + * The kebab half is the half that has actually failed. */ const SRC = resolve(__dirname, "../src"); diff --git a/internal-packages/observability-map/test/mutationCorpus.test.ts b/internal-packages/observability-map/test/mutationCorpus.test.ts index 8dd03ee6ded..02685541f2d 100644 --- a/internal-packages/observability-map/test/mutationCorpus.test.ts +++ b/internal-packages/observability-map/test/mutationCorpus.test.ts @@ -39,17 +39,20 @@ const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, * so closing the hole later turns this file red until the entry is moved back out deliberately. * - * `dead-branch-after-if-true` wraps a catch clause's body in `if (true) { ... }` and writes an - * error test after it. The test can never run, and it takes the tree from 15 to 23 and raises 68 - * routes. Nothing in the scanner evaluates a condition, so `definitelyExits` cannot see that the - * wrapper always exits, and the two ways to make it see are both worse than the hole. Folding the - * literal `true` starts the same list an earlier round lost with `false`, where every spelling - * nobody thought of pays. Cutting the branch credit at the first statement that MIGHT exit is - * sound and folding-free, and it was measured: it takes the tree from 15 to 6 and accuses 78 real - * routes, because `catch (e) { if (rare) return null; if (e instanceof X) return y; }` is an - * ordinary shape. See the round A fix 3 report. + * `dead-classifying-try-with-call` is the shape `dead-classifying-try` only looked like it closed. + * `canRaise` accepts any call at all, so `try { String(0); }` reads as a clause guarding real work + * and takes the tree from 15 to 42, raising 224 routes, exactly as `try { 0; }` did before it was + * refused. Telling an inert call from one that can throw needs types the scanner does not have. + * The docstrings in `scan.ts`, `types.ts` and `errorClassification.ts` say the rule refuses + * `try { 0; }` and is defeated by one call, rather than claiming the family is closed. + * + * `dead-branch-after-if-true` used to be listed here on a measurement that was wrong. See the round + * A fix 3 report; the short version is that the rejected alternative was implemented with the exit + * flag raised before each statement's own branch check, which makes every deciding statement refuse + * itself. Raising it after is byte-identical on the real tree and closes the shape, so the entry is + * defended now and the `if (true)` family needed no condition folding after all. */ -const KNOWN_GAPS = new Set(["dead-branch-after-if-true"]); +const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); type SourceFile = { relativeName: string; source: string }; @@ -229,9 +232,13 @@ describeCorpus("mutation corpus over the real route tree", () => { * at the head of the clause, so all 260 count, and this threshold is what would notice if a later * change quietly took that back. * - * The guard the design asked for, verdict movement, cannot be used: a defended mutation moves no - * verdict anywhere, and that is precisely what "defended" means, so requiring movement would fail - * every entry that works. Site count is the reachable version of the same intent. + * The guard the design asked for, verdict movement, cannot be used, though not for the reason an + * earlier version of this comment gave. Plenty of defended entries move verdicts hard: + * `delete-every-catch` takes the tree from 15 to 2 and `dead-throw-after-switch` to 6. The + * narrower true reason is that the IDEAL defended shape is one the scanner is blind to, and those + * move nothing at all: `dead-if-false` and the ten entries beside it are defended precisely + * because the tree comes out identical. Requiring movement would fail exactly the entries that + * work best. Site count is the reachable version of the same intent. */ const MINIMUM_FILES_TOUCHED = 20; const MINIMUM_SITES_TOUCHED = 40; diff --git a/internal-packages/observability-map/test/mutations.ts b/internal-packages/observability-map/test/mutations.ts index 894dd8da9d4..7d94064efa1 100644 --- a/internal-packages/observability-map/test/mutations.ts +++ b/internal-packages/observability-map/test/mutations.ts @@ -648,6 +648,26 @@ export const MUTATIONS: Mutation[] = [ return applyEdits(source, edits); }, }, + { + id: "dead-classifying-try-with-call", + kind: "preserving", + what: "prepend a classifying try/catch over a try block whose only work is an inert call", + apply(fileName, source) { + const sf = parse(fileName, source); + const edits: Edit[] = []; + for (const body of entryBodies(sf)) { + edits.push( + insert( + body.getStart() + 1, + "\ntry { String(0); } catch (obsMapDead) {" + + " if (obsMapDead instanceof Error) { return new Response(null, { status: 400 }); }" + + " throw obsMapDead; }\n" + ) + ); + } + return applyEdits(source, edits); + }, + }, { id: "same-arms-ternary", kind: "preserving", @@ -802,6 +822,7 @@ export const MUTATIONS: Mutation[] = [ */ export const ADDITIVE_IDS = [ "dead-classifying-try", + "dead-classifying-try-with-call", "same-arms-ternary", "dead-throw-after-block", "dead-throw-after-do", From b2e030b718fa81a04f5e72e37bedb114e5dcdc76 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:22:40 +0100 Subject: [PATCH 061/117] fix(observability-map): read parse diagnostics through public typescript api The parse-failure guard read sf.parseDiagnostics, an internal property the compiler is free to rename. If it went away the detection would stop working silently and the report would keep saying zero parse failures, while an undetected failure shrinks the denominator and inflates the score. It now asks a ts.Program for the syntactic diagnostics, with a host that hands the program the source file already parsed, so nothing is parsed twice. The cost is real and measured: a full scan of the route tree goes from about 850ms to about 1450ms over five runs of each. A slower scan is the cheaper of the two prices. The change is invisible to every existing test, both spellings find the same malformed files, so the new guard is a source-level one. Three more malformed shapes are covered as well: an unclosed jsx element, an unterminated template literal, and a stray closing brace. --- .../observability-map/src/scan.ts | 40 ++++++++++++++-- .../observability-map/test/scan.test.ts | 47 ++++++++++++++++++- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 5521833a382..f2ddbd77156 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -891,15 +891,47 @@ function collectLocalFunctions(sf: ts.SourceFile): Map { return functions; } +/** + * Compiler options for the throwaway program below. `noLib` and `noResolve` keep it from going to + * disk: nothing here needs a type, only the syntax the parser already produced. + */ +const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, allowJs: true }; + +/** + * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off + * the diagnostics array the parser hangs on the source file, which is internal and which the + * compiler is free to rename. The whole parse-failure discipline rests on this, and an undetected + * parse failure shrinks the denominator and inflates the score, so it must not be the kind of + * thing a compiler upgrade can switch off silently. + * + * The host hands the program the `sf` we already have, so this does not parse the source a second + * time. The cost is the program machinery around it, and it is not free: a full scan of the real + * route tree went from about 850ms to about 1450ms, measured over five runs of each. A slower + * scan of a tool that runs once a pull request is the cheaper of the two prices. + */ +function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { + const host: ts.CompilerHost = { + getSourceFile: (name) => (name === sf.fileName ? sf : undefined), + getDefaultLibFileName: () => "lib.d.ts", + writeFile: () => {}, + getCurrentDirectory: () => "", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (name) => name === sf.fileName, + readFile: () => undefined, + }; + return ts.createProgram([sf.fileName], SYNTAX_ONLY_OPTIONS, host).getSyntacticDiagnostics(sf); +} + export function scanFile(fileName: string, source: string): EntryPoint | null { const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are // the only signal that a file did not parse. - const parseDiagnostics = (sf as ts.SourceFile & { parseDiagnostics?: ts.Diagnostic[] }) - .parseDiagnostics; - if (parseDiagnostics && parseDiagnostics.length > 0) { - const first = parseDiagnostics[0]!; + const diagnostics = syntacticDiagnostics(sf); + if (diagnostics.length > 0) { + const first = diagnostics[0]!; throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); } diff --git a/internal-packages/observability-map/test/scan.test.ts b/internal-packages/observability-map/test/scan.test.ts index c10f6d2af66..7e571f33720 100644 --- a/internal-packages/observability-map/test/scan.test.ts +++ b/internal-packages/observability-map/test/scan.test.ts @@ -1,6 +1,6 @@ -import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { ParseFailureError, scanDirectory, scanFile } from "../src/scan.js"; const LOADER = ` @@ -494,6 +494,49 @@ describe("scanFile: parse failures", () => { ).toThrow(ParseFailureError); }); + // B8. The detection used to read `sf.parseDiagnostics`, an internal property. These are the + // shapes that prove the public route through `ts.Program` still sees a malformed file. + it("throws on an unclosed jsx element in a tsx route", () => { + expect(() => + scanFile( + "broken.route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi; }` + ) + ).toThrow(ParseFailureError); + }); + + it("throws on an unterminated template literal", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return \`unterminated; }`) + ).toThrow(ParseFailureError); + }); + + it("throws on a stray closing brace after a complete function", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return json({}); } }`) + ).toThrow(ParseFailureError); + }); + + it("names the diagnostic rather than reporting a bare failure", () => { + try { + scanFile("broken.ts", `export async function loader() { const a = ; }`); + expect.unreachable("scanFile should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(ParseFailureError); + expect((error as ParseFailureError).diagnostic.length).toBeGreaterThan(0); + } + }); + + // The change from `sf.parseDiagnostics` to a program-backed lookup is invisible to every test + // above: both spellings find the same malformed files today. What a compiler upgrade can break + // is the private one, and only a source-level guard can fail for that. + it("reads its diagnostics through public typescript api rather than a private field", () => { + const source = readFileSync(resolve(__dirname, "../src/scan.ts"), "utf8"); + expect(source).not.toContain("parseDiagnostics"); + expect(source).toContain("getSyntacticDiagnostics"); + }); + it("does not throw on a well-formed tsx route", () => { expect(() => scanFile( From d0d50f859e345d2a157166f9c30dc2a83a754465 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:02 +0100 Subject: [PATCH 062/117] fix(observability-map): report a suppression that names no check instead of dropping it `// obs-map-disable eror-classification -- typo` parsed, landed in the suppression map, matched nothing and appeared nowhere. The author reads the finding as acknowledged and the tool goes on reporting it with no hint why. The parser now splits directives by whether the id names a check in CHECKS. The unknown ones are carried on the scored entry and on the report, and the terminal report prints one UNKNOWN SUPPRESSION line per file naming the file, the bad ids, and the ids that would have worked. It is not thrown and it is not dropped. --- .../observability-map/src/report/terminal.ts | 26 +++++++- .../observability-map/src/score.ts | 14 ++++- .../observability-map/src/suppression.ts | 33 +++++++++-- .../test/docstringReferences.test.ts | 3 + .../test/suppression.test.ts | 59 ++++++++++++++++++- 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index e85c629bb2d..a0f5e779b52 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -1,5 +1,5 @@ import type { MapReport, ScoredEntry } from "../score.js"; -import { SCORED_CHECK_IDS } from "../checks/index.js"; +import { CHECKS, SCORED_CHECK_IDS } from "../checks/index.js"; const NOT_MEASURED = "not measured".padEnd(15); @@ -32,6 +32,24 @@ export const contextOnly = (e: ScoredEntry) => { return failures.length === 1 && failures[0]!.id === "request-context"; }; +/** + * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. Empty when every + * directive named a real check. A typo suppresses nothing, so without this the author reads the + * finding as acknowledged and the tool goes on reporting it with no hint why. + */ +export function unknownSuppressionLine(fileName: string, ids: string[]): string { + return ( + `UNKNOWN SUPPRESSION ${fileName}: ${ids.join(", ")} ` + + `(no such check, nothing suppressed). Known: ${CHECKS.map((c) => c.id).join(", ")}.` + ); +} + +export function unknownSuppressionLines(report: MapReport): string[] { + return report.unknownSuppressions.map(({ fileName, ids }) => + unknownSuppressionLine(fileName, ids) + ); +} + /** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when * there is nothing to report, i.e. no sensitive mutation exists. */ export function auditLine(report: MapReport): string | null { @@ -93,6 +111,12 @@ export function renderTerminal(report: MapReport): string { lines.push(context); } + const unknown = unknownSuppressionLines(report); + if (unknown.length > 0) { + lines.push(""); + lines.push(...unknown); + } + if (report.suppressions.checks > 0) { const { entries, checks } = report.suppressions; lines.push( diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 420a07b19f4..7f56c5783aa 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -1,6 +1,6 @@ import type { CheckResult, EntryPoint } from "./types.js"; import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; -import { suppressedChecks } from "./suppression.js"; +import { parseSuppressions } from "./suppression.js"; import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; import { classifySensitivity } from "./sensitivity.js"; @@ -30,6 +30,10 @@ export type ScoredEntry = { /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ suppressed: string[]; + /** Ids in a suppression directive that name no check, so they suppress nothing. Carried here so + * the renderers can say so: dropping them silently is what made a typo look like an + * acknowledgement. */ + unknownSuppressions: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; }; @@ -43,6 +47,8 @@ export type MapReport = { unmeasured: number; /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ suppressions: { entries: number; checks: number }; + /** Suppression directives naming no check, per file, so a typo is reported rather than dropped. */ + unknownSuppressions: { fileName: string; ids: string[] }[]; byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; @@ -57,7 +63,7 @@ export type MapReport = { }; export function scoreEntry(ep: EntryPoint): ScoredEntry { - const suppressed = suppressedChecks(ep.source, ep.fileName); + const { byId: suppressed, unknown } = parseSuppressions(ep.source, ep.fileName); const raw = CHECKS.map((c) => c.run(ep)); const checks = raw.map((result) => { const reason = suppressed.get(result.id); @@ -86,6 +92,7 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { checks, rawChecks: raw, suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), + unknownSuppressions: unknown, measured: scoredApplicable.length > 0, // Capped by the pre-suppression ratio: removing a failing check from both the numerator and // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this @@ -157,6 +164,9 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo entries: suppressing.length, checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), }, + unknownSuppressions: entries + .filter((e) => e.unknownSuppressions.length > 0) + .map((e) => ({ fileName: e.fileName, ids: e.unknownSuppressions })), byFamily, sensitiveCohort: groupStats(sensitive), auditGap: { diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 476c2a5a2ee..7499cf74179 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -1,4 +1,7 @@ import ts from "typescript"; +import { CHECKS } from "./checks/index.js"; + +const KNOWN_CHECK_IDS = new Set(CHECKS.map((c) => c.id)); /** * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` @@ -120,14 +123,26 @@ function commentLines(source: string, sf: ts.SourceFile): string[] { return lines; } +export type Suppressions = { + /** Check id to reason, for ids that name a check in `CHECKS`. */ + byId: Map; + /** + * Ids that parsed as a directive but name no check, in source order and deduplicated. A typo + * (`eror-classification`) used to land in the map, match nothing and appear nowhere, so the + * author read the finding as acknowledged while the tool kept reporting it. + */ + unknown: string[]; +}; + /** - * Check id to reason. A suppression without a reason, or outside a comment, is ignored. + * Every suppression directive in the source, split by whether its id names a real check. A + * directive without a reason, or outside a comment, is ignored either way. * * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly * distinguished from a generic type argument list (`(x) => x`), when the file is really a * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. */ -export function suppressedChecks(source: string, fileName = "check.ts"): Map { +export function parseSuppressions(source: string, fileName = "check.ts"): Suppressions { const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; const sf = ts.createSourceFile( fileName, @@ -137,13 +152,21 @@ export function suppressedChecks(source: string, fileName = "check.ts"): Map(); + const byId = new Map(); + const unknown = new Set(); for (const line of commentLines(source, sf)) { const match = PATTERN.exec(line); if (!match) continue; const [, id, reason] = match; const trimmedReason = reason?.trim(); - if (id && trimmedReason && trimmedReason.length > 0) out.set(id, trimmedReason); + if (!id || !trimmedReason || trimmedReason.length === 0) continue; + if (KNOWN_CHECK_IDS.has(id)) byId.set(id, trimmedReason); + else unknown.add(id); } - return out; + return { byId, unknown: [...unknown] }; +} + +/** The known half of `parseSuppressions`, for callers that only apply suppressions. */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + return parseSuppressions(source, fileName).byId; } diff --git a/internal-packages/observability-map/test/docstringReferences.test.ts b/internal-packages/observability-map/test/docstringReferences.test.ts index 51bfefa5d16..20e922bae92 100644 --- a/internal-packages/observability-map/test/docstringReferences.test.ts +++ b/internal-packages/observability-map/test/docstringReferences.test.ts @@ -45,6 +45,9 @@ const NOT_A_TEST_NAME = new Set([ "not-applicable", // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. "obs-map-disable-next-line", + // The worked example of a mistyped check id in `suppression.ts`. A misspelling of a check id is + // the thing being described, so it names no test by construction. + "eror-classification", ]); /** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ diff --git a/internal-packages/observability-map/test/suppression.test.ts b/internal-packages/observability-map/test/suppression.test.ts index c40d1332894..10679133550 100644 --- a/internal-packages/observability-map/test/suppression.test.ts +++ b/internal-packages/observability-map/test/suppression.test.ts @@ -1,4 +1,4 @@ -import { suppressedChecks } from "../src/suppression.js"; +import { parseSuppressions, suppressedChecks } from "../src/suppression.js"; describe("suppressedChecks", () => { it("reads a suppression with its reason", () => { @@ -243,3 +243,60 @@ describe("suppressedChecks", () => { expect(m.get("auth-boundary")).toBe("generic helper"); }); }); + +// B6. `// obs-map-disable eror-classification -- typo` used to parse, land in the map, match no +// check and appear nowhere, so the author read the finding as acknowledged. +describe("a suppression naming a check that does not exist", () => { + it("suppresses nothing and is reported as unknown", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(r.byId.size).toBe(0); + expect(r.unknown).toEqual(["eror-classification"]); + }); + + it("does not swallow the real suppressions beside it", () => { + const r = parseSuppressions( + `// obs-map-disable auth-boundry -- typo + // obs-map-disable auth-boundary -- public by design + export async function loader() { return 1; }` + ); + expect(r.byId.get("auth-boundary")).toBe("public by design"); + expect(r.unknown).toEqual(["auth-boundry"]); + }); + + it("reports each unknown id once however many times it appears", () => { + const r = parseSuppressions( + `// obs-map-disable request-contex -- typo + /* obs-map-disable request-contex -- typo again */ + // obs-map-disable audit-trial -- another typo + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual(["request-contex", "audit-trial"]); + }); + + it("is not reported when the directive had no reason, since it was never a suppression", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("is not read out of a string literal any more than a real one is", () => { + const r = parseSuppressions( + `const help = "// obs-map-disable eror-classification -- typo"; + export async function loader() { return help; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("keeps suppressedChecks returning only the ids that name a check", () => { + const m = suppressedChecks( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); From e0860dd2492b072d6cf34d9b3c1a074fde181fe2 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:19 +0100 Subject: [PATCH 063/117] test(observability-map): cover the unknown-suppression line, and guard the fix-list slices Two slices took `out.indexOf("FIX FIRST")` without checking it. When the section is absent that is -1, `slice(-1)` is the last character of the report, and the `not.toContain` below it passes for the wrong reason. It is the trap this file's own comment already warned about. Both now go through one helper that asserts both ends, and the helper has a test of its own so it can fail. Adds the terminal coverage for the unknown-suppression line: the file and the bad id are named, the known ids are listed, the finding is not counted as suppressed, and one line is printed per file rather than one for the run. --- .../observability-map/test/report.test.ts | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/test/report.test.ts b/internal-packages/observability-map/test/report.test.ts index 7beed431da1..7979800a02a 100644 --- a/internal-packages/observability-map/test/report.test.ts +++ b/internal-packages/observability-map/test/report.test.ts @@ -20,7 +20,24 @@ const report = () => ["broken.ts"] ); +/** The FIX FIRST section, with the index asserted rather than assumed. `indexOf` returns -1 for a + * section that is not there, and `slice(-1)` is the last character of the report, which every + * `not.toContain` below would pass against. */ +function sliceFixFirst(out: string): string { + const start = out.indexOf("FIX FIRST"); + expect(start).toBeGreaterThan(-1); + const end = out.indexOf("no findings:"); + expect(end).toBeGreaterThan(start); + return out.slice(start, end); +} + describe("renderTerminal", () => { + // The guard has to be able to fail, or it is decoration in a file whose own comment warns about + // exactly this trap. + it("refuses to slice a report with no fix list rather than returning its last character", () => { + expect(() => sliceFixFirst("a report with neither section in it")).toThrow(); + }); + it("shows the global score, the audit gap and the fix list", () => { const out = renderTerminal(report()); expect(out).toContain("COVERAGE"); @@ -119,9 +136,7 @@ describe("renderTerminal", () => { // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for // "already solid" and the assertions were quietly running against the whole tail. - const listEnd = out.indexOf("no findings:"); - expect(listEnd).toBeGreaterThan(-1); - const fixFirst = out.slice(out.indexOf("FIX FIRST"), listEnd); + const fixFirst = sliceFixFirst(out); const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); @@ -232,7 +247,9 @@ describe("collapsing the house-style finding", () => { // single finding repeated. Same reasoning that keeps audit-trail out of the list. it("keeps an entry whose only finding is request-context out of the fix list", () => { const out = renderTerminal(buildReport([namesNobody()], [])); - const fixFirst = out.slice(out.indexOf("FIX FIRST")); + // Guarded for the same reason as the slice above: an absent section makes `indexOf` return -1, + // `slice(-1)` yields the last character, and the negative assertion passes for the wrong reason. + const fixFirst = sliceFixFirst(out); expect(fixFirst).not.toContain("api.v1.silent.ts"); }); @@ -264,9 +281,48 @@ describe("collapsing the house-style finding", () => { it("still lists request-context when the entry fails something else too", () => { const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); - const fixFirst = out.slice(out.indexOf("FIX FIRST")); + const fixFirst = sliceFixFirst(out); expect(fixFirst).toContain("api.v1.auth.tokens.ts"); expect(fixFirst).toContain("request-context"); expect(fixFirst).toContain("error-classification"); }); }); + +// B6. A stderr warning is the minimum; the terminal report is where someone would notice. +describe("reporting a suppression that names no check", () => { + const typo = (fileName: string) => + scanFile( + fileName, + `// obs-map-disable eror-classification -- typo + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + it("names the file and the bad id", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("UNKNOWN SUPPRESSION"); + expect(out).toContain("api.v1.a.ts"); + expect(out).toContain("eror-classification"); + }); + + it("lists the ids that would have worked", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("error-classification, auth-boundary, request-context, audit-trail"); + }); + + it("does not report the finding as suppressed", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).not.toMatch(/SUPPRESSED\s+\d/); + }); + + it("reports one line per file rather than one for the run", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts"), typo("api.v1.b.ts")], [])); + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(2); + }); + + it("says nothing when every directive named a real check", () => { + expect(renderTerminal(report())).not.toContain("UNKNOWN SUPPRESSION"); + }); +}); From 3241352fc5542602cfcb8fd17ce9c80732b369a4 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:23:39 +0100 Subject: [PATCH 064/117] fix(observability-map): render measured state in the pr comment, and post only when it moves The changed-entries table compared raw scores. An entry no scored check applies to carries 100 as a placeholder, which the score itself excludes from every mean, so the table rendered a route refactored down to a trivial body as a 67-point improvement and a trivial route gaining real work as the pull request's worst regression. Both columns now say "not measured" for an unmeasured entry, the way the terminal gauge already does for a null mean, and the early-out compares measured state as well as score so a measured 100 becoming an unmeasured 100 still produces a row. A new entry that passes every check it was measured against no longer gets a row: its drop is 0, which by the drop doc comment's own logic means there is nothing to fix. A new entry nothing applied to is a different statement and still gets one. The job also stops commenting on a pull request that moves nothing. hasDelta is a pure function of the two reports, tested rather than written as shell logic, and it is true when there is no base, when the global moved, when an entry was added, removed, rescored or newly unmeasured, when a check fails at head that did not at base, and when the parse-failure or unknown-suppression warnings changed. With no delta and a comment already on the pull request, prCommentCli renders a short resolved state instead of going quiet and leaving findings that no longer exist standing. --- .../observability-map/src/report/prComment.ts | 121 +++++++++++-- .../src/report/prCommentCli.ts | 44 ++++- .../observability-map/test/prComment.test.ts | 171 +++++++++++++++++- .../test/prCommentCli.test.ts | 62 ++++++- 4 files changed, 376 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 8e857b8c6d9..80c0b4b70fe 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,6 +1,12 @@ import type { MapReport, ScoredEntry } from "../score.js"; import { SCORED_CHECK_IDS } from "../checks/index.js"; -import { auditLine, contextLine, contextOnly, scoredFailures } from "./terminal.js"; +import { + auditLine, + contextLine, + contextOnly, + scoredFailures, + unknownSuppressionLines, +} from "./terminal.js"; /** First line of every comment this job posts, so the upsert step can find its own comment again. */ export const MARKER = ""; @@ -32,17 +38,30 @@ function scoreLine(head: MapReport, base: MapReport | null): string { return `${headline} ${comparison}`; } +/** What goes in a score column: a figure, an absence, or no prior entry to compare against. */ +const NOT_MEASURED = "not measured"; + +/** + * `score` is 100 for an entry no scored check applied to, a placeholder the score itself excludes + * from every mean. Rendering that 100 as a figure turned a route refactored down to a trivial body + * into a 67-point improvement, and a trivial route gaining real work into the PR's worst + * regression. So the cell says what the terminal gauge says for a null mean instead. + */ +const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); + type ChangedRow = { routePath: string; sensitive: boolean; - baseScore: number | "new"; - headScore: number; + baseScore: number | string; + headScore: number | string; nowFailing: string[]; /** * How much the entry got worse, used to sort the table. A new entry has no base score to * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the * same as an existing one that dropped 40 points, which is the ordering "what needs fixing - * first" implies. + * first" implies. Zero whenever either side is unmeasured, because there is no arithmetic to do + * between a figure and an absence; such a row is in the table to disclose the transition, not to + * claim a size for it. */ drop: number; }; @@ -55,25 +74,31 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re for (const h of head.entries) { const b = baseByFile.get(h.fileName); if (!b) { + // A new entry that passes every check it was measured against has nothing to fix, which is + // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a + // different statement and still gets a row, since its 100 is a placeholder rather than a pass. + if (h.measured && h.score === 100) continue; rows.push({ routePath: h.routePath, sensitive: h.sensitive, baseScore: "new", - headScore: h.score, + headScore: scoreCell(h), nowFailing: failingIds(h), - drop: 100 - h.score, + drop: h.measured ? 100 - h.score : 0, }); continue; } - if (b.score === h.score) continue; + // Measured state is part of what changed: a measured-to-unmeasured transition can leave the + // score untouched at its placeholder value, and skipping on the number alone hid it. + if (b.measured === h.measured && b.score === h.score) continue; const baseFailing = new Set(failingIds(b)); rows.push({ routePath: h.routePath, sensitive: h.sensitive, - baseScore: b.score, - headScore: h.score, + baseScore: scoreCell(b), + headScore: scoreCell(h), nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), - drop: b.score - h.score, + drop: b.measured && h.measured ? b.score - h.score : 0, }); } @@ -148,6 +173,78 @@ function fixFirstSection(head: MapReport): string[] { return lines; } +/** + * Whether this pull request moves the report at all, so the job can stay quiet when it does not. + * + * True on: no base to compare against, a global score change, an entry added or removed, an + * entry's score or measured state changed, a check that fails at head and did not at base, a + * change in the parse failure count, or a change in the unknown suppression warnings. The last two + * are here because both render a line in the comment, so a run that gains one has moved the report + * even though no score did. + * + * The residual: adding a suppression whose check was already failing moves nothing this asks + * about. The score is capped at the pre-suppression ratio so it cannot move, and the finding + * leaves the fix list quietly. `SUPPRESSED` in the terminal report is where that shows up. + */ +export function hasDelta(head: MapReport, base: MapReport | null): boolean { + if (!base) return true; + if (head.global !== base.global) return true; + if (head.parseFailures.length !== base.parseFailures.length) return true; + if (JSON.stringify(head.unknownSuppressions) !== JSON.stringify(base.unknownSuppressions)) { + return true; + } + + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + if (base.entries.some((e) => !headFiles.has(e.fileName))) return true; + + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) return true; + if (b.measured !== h.measured || b.score !== h.score) return true; + const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); + if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; + } + return false; +} + +/** + * What replaces a comment whose findings a later push fixed. Going silent would leave the earlier + * comment standing with findings that no longer exist, which is worse than a redundant comment. + */ +export function renderResolvedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "Nothing in this pull request moves the report any more. The findings an earlier push " + + "reported are gone.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** + * What the job posts when the head scan did not produce a report. The alternative was a red x on + * a job that must never block a pull request, and the alternative to that was swallowing the + * failure so the only signal was a comment that never appeared. + */ +export function renderScanFailedComment(): string { + return [ + MARKER, + "", + "## Observability map", + "", + "The scan failed for this run, so there is no report. Anything above is from an earlier push " + + "and is stale. The workflow log has the error.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + /** * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the * two by `fileName`, the same identifier `renderJson` carries. @@ -162,7 +259,9 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string if (audit) lines.push(audit); const context = contextLine(head); if (context) lines.push(context); - if (audit || context) lines.push(""); + const unknown = unknownSuppressionLines(head); + lines.push(...unknown); + if (audit || context || unknown.length > 0) lines.push(""); lines.push( "Report only, nothing here gates the merge. The rules and their reasons: " + diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index 0283eb96591..a2aef9a3477 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -2,7 +2,12 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { MapReport } from "../score.js"; -import { renderPrComment } from "./prComment.js"; +import { + hasDelta, + renderPrComment, + renderResolvedComment, + renderScanFailedComment, +} from "./prComment.js"; /** Where output goes. Injectable so tests can read it without spawning a process. */ export type Io = { out: (s: string) => void; err: (s: string) => void }; @@ -28,15 +33,33 @@ function readReport(path: string, label: string): MapReport { } } -/** `-` or a missing second arg means no base: the CI job falls back to this when the base scan - * itself failed, so the comment still renders rather than the job going red. */ +/** + * `-` or a missing second arg means no base: the CI job falls back to this when the base scan + * itself failed, so the comment still renders rather than the job going red. + * + * Empty output means "post nothing". The job only comments when the pull request moves the report, + * and `--existing-comment` is how the workflow says a comment from an earlier push is already on + * the pull request: with the delta gone, that comment is replaced with a resolved state rather + * than left standing with findings that no longer exist. + * + * `--scan-failed` takes no report and prints the stale-report comment, for the case where the head + * scan produced nothing to read. + */ export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); - const headPath = args[0]; - const basePath = args[1]; + const scanFailed = args.includes("--scan-failed"); + const existingComment = args.includes("--existing-comment"); + const positional = args.filter((a) => !a.startsWith("--")); + const headPath = positional[0]; + const basePath = positional[1]; + + if (scanFailed) { + io.out(`${renderScanFailedComment()}\n`); + return 0; + } if (!headPath) { - io.err("usage: prCommentCli.ts [base.json|-]\n"); + io.err("usage: prCommentCli.ts [base.json|-] [--existing-comment]\n"); return 1; } @@ -50,8 +73,13 @@ export function main(argv: string[], io: Io = processIo): number { return 1; } - io.out(renderPrComment(head, base)); - io.out("\n"); + if (hasDelta(head, base)) { + io.out(`${renderPrComment(head, base)}\n`); + return 0; + } + if (existingComment) { + io.out(`${renderResolvedComment()}\n`); + } return 0; } diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 7f651d57dc2..4299b1ae3a7 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -1,4 +1,4 @@ -import { renderPrComment } from "../src/report/prComment.js"; +import { hasDelta, renderPrComment } from "../src/report/prComment.js"; import { buildReport } from "../src/score.js"; import { scanFile } from "../src/scan.js"; @@ -96,6 +96,94 @@ describe("renderPrComment", () => { expect(row).toMatch(/error-classification|auth-boundary|request-context/); }); + it("skips a new entry that passes every check it was measured against", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", cleanSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).not.toContain("/api/v1/new"); + expect(out).toContain("No entry point this PR touches changed its score."); + }); + + // B3. `score` is 100 for an entry no scored check applied to, and the table read that placeholder + // as a figure: a route refactored down to a trivial body rendered as a 67-point improvement. + describe("an unmeasured entry", () => { + const trivial = `export const loader = () => new Response("ok");`; + + it("renders as not measured rather than as 100 when the head stopped being measurable", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toContain("not measured"); + expect(row).not.toMatch(/\|\s*100\s*\|/); + }); + + it("renders as not measured in the base column when the head gained real work", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| not measured \| \d+ \|/); + }); + + // The early-out compared scores only, so a measured 100 turning into an unmeasured placeholder + // 100 produced no row at all: the table said nothing happened. + it("still produces a row when a measured 100 becomes an unmeasured placeholder 100", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + expect(base.entries[0]!.score).toBe(100); + expect(head.entries[0]!.score).toBe(100); + + const out = renderPrComment(head, base); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| 100 \| not measured \|/); + }); + + // Not the same statement as a new entry that passes everything, which is skipped above. + it("still gets a row when it is new, since its 100 is a placeholder and not a pass", () => { + const head = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", cleanSource)!, + scanFile("resources.health.ts", trivial)!, + ], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /resources/health |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| new \| not measured \|/); + }); + + it("does not sort an unmeasured transition above a real regression", () => { + const head = buildReport( + [scanFile("resources.gone.ts", trivial)!, scanFile("resources.busy.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("resources.gone.ts", brokenSource)!, scanFile("resources.busy.ts", cleanSource)!], + [] + ); + const out = renderPrComment(head, base); + + const busy = out.indexOf("/resources/busy"); + const gone = out.indexOf("/resources/gone"); + expect(busy).toBeGreaterThan(-1); + expect(gone).toBeGreaterThan(-1); + expect(busy).toBeLessThan(gone); + }); + }); + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; const sensitiveSmallDropHead = scanFile( @@ -185,3 +273,84 @@ describe("renderPrComment", () => { expect(out).toContain("internal-packages/observability-map/README.md"); }); }); + +// B4. The job posts only when the pull request moves the report, so the decision has to be a +// tested function of the two reports rather than shell logic in the workflow. +describe("hasDelta", () => { + const trivial = `export const loader = () => new Response("ok");`; + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + + it("is true when there is no base to compare against", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), null)).toBe(true); + }); + + it("is false for two identical reports", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), one("api.v1.a.ts", cleanSource))).toBe(false); + }); + + it("is true when the global score moved", () => { + expect(hasDelta(one("api.v1.a.ts", brokenSource), one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was added", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was removed", () => { + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(one("api.v1.a.ts", cleanSource), base)).toBe(true); + }); + + // The global is a mean over measured entries, so two entries moving in opposite directions can + // leave it where it was. The per-entry comparison is what catches that. + it("is true when an entry's score moved but the global mean did not", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + // audit-trail does not feed the score, so it can start failing without moving a single figure. + it("is true when an unscored check started failing and no score moved", () => { + const head = one("api.v1.envvars.ts", cleanSource); + const base = one( + "api.v1.envvars.ts", + `// obs-map-disable audit-trail -- no helper exists yet\n${cleanSource}` + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when an entry stopped being measured at the same placeholder score", () => { + const head = one("api.v1.a.ts", trivial); + const base = one("api.v1.a.ts", cleanSource); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a parse failure appeared, since the comment warns about it", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken.ts"]); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when a suppression names a check that does not exist", () => { + const head = one( + "api.v1.a.ts", + `// obs-map-disable eror-classification -- typo\n${cleanSource}` + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); +}); diff --git a/internal-packages/observability-map/test/prCommentCli.test.ts b/internal-packages/observability-map/test/prCommentCli.test.ts index b464725ea77..2b171f676db 100644 --- a/internal-packages/observability-map/test/prCommentCli.test.ts +++ b/internal-packages/observability-map/test/prCommentCli.test.ts @@ -19,15 +19,31 @@ const run = (...args: string[]) => { return { code, out: c.out(), err: c.err() }; }; +const swallows = `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +const handles = `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + describe("prCommentCli", () => { const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); const headPath = join(dir, "head.json"); const basePath = join(dir, "base.json"); + const unchangedPath = join(dir, "unchanged.json"); const source = `export const loader = () => new Response("ok");`; const report = buildReport([scanFile("resources.a.ts", source)!], []); - writeFileSync(headPath, renderJson(report)); - writeFileSync(basePath, renderJson(report)); + writeFileSync(headPath, renderJson(buildReport([scanFile("api.v1.t.ts", swallows)!], []))); + writeFileSync(basePath, renderJson(buildReport([scanFile("api.v1.t.ts", handles)!], []))); + writeFileSync(unchangedPath, renderJson(report)); afterAll(() => rmSync(dir, { recursive: true })); @@ -37,6 +53,40 @@ describe("prCommentCli", () => { expect(r.out.split("\n")[0]).toBe(""); }); + // B4. The job used to comment on every push, including one that moved nothing. + it("posts nothing when the report did not move and no comment exists yet", () => { + const r = run(unchangedPath, unchangedPath); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + }); + + it("replaces an existing comment with a resolved state when the delta has gone", () => { + const r = run(unchangedPath, unchangedPath, "--existing-comment"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + it("posts the full comment when there is a delta, existing comment or not", () => { + for (const args of [ + [headPath, basePath], + [headPath, basePath, "--existing-comment"], + ]) { + const r = run(...args); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("moves the report any more"); + } + }); + + it("prints the stale-report comment for --scan-failed without reading any file", () => { + const r = run("--scan-failed"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("The scan failed for this run"); + }); + it("treats '-' as no base", () => { const r = run(headPath, "-"); expect(r.code).toBe(0); @@ -49,6 +99,14 @@ describe("prCommentCli", () => { expect(r.out).toContain("Base comparison unavailable."); }); + // Without a base there is no delta to compute, so silence would be a guess. Posting is the + // honest answer even for a report identical to one nobody can see. + it("posts even for an unmoved report when the base is unavailable", () => { + const r = run(unchangedPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + it("exits 1 with a usage message when head.json is missing", () => { const r = run(); expect(r.code).toBe(1); From 5ed954495d40df7eb309bd51b47d19930e5359ee Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:04 +0100 Subject: [PATCH 065/117] ci(observability-map): stop the head scan reddening a pull request, and say when it failed The scan head step had no guard while render and upsert did, under a comment saying the job must never block a pull request. Swallowing it silently is not the answer either, because then nobody learns the report is stale. It is now guarded so it cannot fail the job, and a missing head report makes the render step post a comment saying the scan failed and the report is stale for this run. The head report is written to a temp path and moved on success, so a failed scan leaves no stale or partial file behind. The render step clears comment.md first and writes through a temp path too, so a render failure cannot leave the upsert posting a file that was never written, and it falls back to the same stale-report comment when the render itself fails on a malformed report. The upsert step posts nothing when comment.md is empty, which is how prCommentCli says the pull request does not move the report. The comment lookup moves ahead of the render step, because the render decision needs to know whether a comment already exists, and the upsert reuses the id rather than asking twice. A failed lookup is treated as "a comment exists": that can post a redundant resolved state, where the other guess leaves a stale comment standing. Also moves pull-requests: write from the workflow onto the one job that comments, which is what zizmor asks for. --- .github/workflows/observability-map.yml | 80 +++++++++++++++++++++---- 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 0406f1c3730..4d30b29cc5b 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -13,7 +13,6 @@ concurrency: permissions: contents: read - pull-requests: write jobs: # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, @@ -53,6 +52,10 @@ jobs: report: runs-on: warp-ubuntu-latest-x64-4x + # Only this job comments, so only this job gets the write. + permissions: + contents: read + pull-requests: write # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. if: github.event.pull_request.head.repo.full_name == github.repository steps: @@ -76,9 +79,18 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile + # Guarded rather than allowed to fail: this job must never block a pull request. The failure + # is not swallowed either, the render step below turns a missing head report into a comment + # saying so, because a swallowed failure with no comment is the outcome nobody wants. - name: 🔎 Scan head run: | - pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write > /tmp/head.json + if pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ + > /tmp/head.json.partial; then + mv /tmp/head.json.partial /tmp/head.json + else + rm -f /tmp/head.json /tmp/head.json.partial + echo "head scan failed; the comment will say the report is stale for this run" >&2 + fi - name: 🔎 Scan base with the head's scanner run: | @@ -91,16 +103,58 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi - # continue-on-error: this job must never block a PR. A malformed head.json or a rendering - # bug would otherwise turn the job red the same way a base-scan failure would not, since that - # step already falls back to "-" instead of failing. + # Looked up before the render step because the render step needs it: with no delta to report, + # a pull request that already has a comment gets a resolved state rather than being left with + # findings that no longer exist, and one that does not gets nothing at all. The upsert step + # reuses the id rather than asking again. + - name: 🔍 Look for a comment from an earlier push + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + rm -f /tmp/existing-comment-id + if gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty' \ + > /tmp/existing-comment-id.partial; then + mv /tmp/existing-comment-id.partial /tmp/existing-comment-id + else + rm -f /tmp/existing-comment-id.partial + echo "comment lookup failed; assuming a comment exists so a stale one still gets replaced" >&2 + fi + + # continue-on-error for the same reason as the scan: a rendering bug must not turn the job + # red. An empty /tmp/comment.md means there is nothing to post, which is a decision + # prCommentCli makes, not this shell. - name: 📝 Render comment continue-on-error: true run: | - if [ "$(cat /tmp/base.json)" = "-" ]; then - pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json - > /tmp/comment.md + rm -f /tmp/comment.md + render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + + if [ ! -s /tmp/head.json ]; then + render --scan-failed > /tmp/comment.md + exit 0 + fi + + base=/tmp/base.json + if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then + base="-" + fi + + # An absent file means the lookup itself failed, and assuming there is a comment is the + # safer half of that guess: it can post a redundant resolved state, not leave a stale one. + flags=() + if [ ! -f /tmp/existing-comment-id ] || [ -s /tmp/existing-comment-id ]; then + flags=(--existing-comment) + fi + + if render /tmp/head.json "$base" "${flags[@]}" > /tmp/comment.md.partial; then + mv /tmp/comment.md.partial /tmp/comment.md else - pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts /tmp/head.json /tmp/base.json > /tmp/comment.md + rm -f /tmp/comment.md.partial + echo "render failed; falling back to the stale-report comment" >&2 + render --scan-failed > /tmp/comment.md fi # continue-on-error for the same reason: a transient gh api failure (rate limit, network) @@ -111,8 +165,14 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | - existing=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty') + if [ ! -s /tmp/comment.md ]; then + echo "nothing to post: this pull request does not move the report" + exit 0 + fi + existing="" + if [ -f /tmp/existing-comment-id ]; then + existing=$(cat /tmp/existing-comment-id) + fi if [ -n "$existing" ]; then gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md else From 4d39fb4c735912b48fb02a2dcc358b36a81fb97b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:20 +0100 Subject: [PATCH 066/117] feat(observability-map): add --out, honour --json for one route, and declare the module type `map /api/v1/token --json` printed the text format and dropped the flag. It now prints the scored entry as json, with the unknown-suppression warning on stderr in both formats so it cannot land inside json a caller parses. `--out` names where the whole-tree report is written, defaulting to the repo root as before. It exists so a test can point the write at a temp directory: the only way to exercise the write path before was to let the tests delete and recreate observability-map.json in the monorepo root, which destroyed a developer's generated artifact on every run. package.json gains "type": "module", matching the sibling internal packages. The build emits ESM and cli.ts uses import.meta, while main and types advertised that to a manifest with no module type: node 24 loads it with a MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw an older node gives. The alternative, dropping main, types and build to be honest that this is a tool, also passes the suite; the convention the other internal packages already follow won. --- .../observability-map/package.json | 1 + .../observability-map/src/cli.ts | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json index 776ec458963..cd8c3fc14f4 100644 --- a/internal-packages/observability-map/package.json +++ b/internal-packages/observability-map/package.json @@ -2,6 +2,7 @@ "name": "@internal/observability-map", "private": true, "version": "0.0.1", + "type": "module", "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", "dependencies": { diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index cb9c9b6017a..6ef722d9afd 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -5,7 +5,11 @@ import type { EntryPoint } from "./types.js"; import { scanDirectory } from "./scan.js"; import { buildReport, scoreEntry } from "./score.js"; import { routePathOf } from "./adapters/remix.js"; -import { renderTerminal } from "./report/terminal.js"; +import { + renderTerminal, + unknownSuppressionLine, + unknownSuppressionLines, +} from "./report/terminal.js"; import { renderJson } from "./report/json.js"; const DEFAULT_ROUTES = "apps/webapp/app/routes"; @@ -56,6 +60,13 @@ function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { ); } +/** Value of a `--flag=value` argument, or null when the flag is absent. */ +function flagValue(args: string[], flag: string): string | null { + const prefix = `${flag}=`; + const found = args.find((a) => a.startsWith(prefix)); + return found === undefined ? null : found.slice(prefix.length); +} + export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); const asJson = args.includes("--json"); @@ -63,11 +74,11 @@ export function main(argv: string[], io: Io = processIo): number { const target = args.find((a) => !a.startsWith("--")); const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); - const routesFlag = args.find((a) => a.startsWith("--routes=")); + const routesFlag = flagValue(args, "--routes"); let routesDir: string; - if (routesFlag) { - routesDir = resolve(process.cwd(), routesFlag.slice("--routes=".length)); + if (routesFlag !== null) { + routesDir = resolve(process.cwd(), routesFlag); let isDir = false; try { isDir = statSync(routesDir).isDirectory(); @@ -99,6 +110,14 @@ export function main(argv: string[], io: Io = processIo): number { ); } const scored = scoreEntry(matches[0]!); + // On stderr in both formats: a warning on stdout would be inside the JSON a caller parses. + if (scored.unknownSuppressions.length > 0) { + io.err(`${unknownSuppressionLine(scored.fileName, scored.unknownSuppressions)}\n`); + } + if (asJson) { + io.out(`${JSON.stringify(scored, null, 2)}\n`); + return 0; + } const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; io.out( `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` @@ -111,10 +130,18 @@ export function main(argv: string[], io: Io = processIo): number { } const report = buildReport(entryPoints, parseFailures); + for (const line of unknownSuppressionLines(report)) io.err(`${line}\n`); io.out(asJson ? renderJson(report) : renderTerminal(report)); io.out("\n"); if (!noWrite) { - writeFileSync(resolve(repoRoot, "observability-map.json"), renderJson(report)); + // `--out` exists so a test can point the write somewhere disposable. Without it the only way + // to exercise the write path was to let the tests create and delete a file in the repo root. + const outFlag = flagValue(args, "--out"); + const outPath = + outFlag === null + ? resolve(repoRoot, "observability-map.json") + : resolve(process.cwd(), outFlag); + writeFileSync(outPath, renderJson(report)); } return 0; } From 6c54ec7465a65c782c8d41adbd4d92456f5ac439 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 22:24:38 +0100 Subject: [PATCH 067/117] test(observability-map): give the cli tests their own routes tree, and watch the live one in ci cli.test.ts asserted that /api/v1/token exists, that api.v1.runs.ts is line 2 of the output, and that /admin/api/v1/runs-replication matches several entries. Those are assertions about the webapp's contents, not about this package. Meanwhile the internal filter in pr_checks.yml did not cover apps/webapp, so a webapp-only pull request that renamed a route never ran this suite, merged green, and broke main or the next unrelated internal-packages pull request. Verified both halves: with api.v1.token.ts renamed away the old tests fail two cases and the new ones pass all fifteen. cli.test.ts now builds its own routes directory and points the cli at it with --routes. The one deliberate real-tree test stays in integration.test.ts and asserts only what survives churn: the scan does not crash, the count sits in a wide band, parse failures are zero. pr_checks.yml's internal filter gains apps/webapp/app/routes so that test cannot go stale unnoticed. countCandidates was a verbatim copy of the scanner's own walk, so the bound it guarded could not fail for any route shape both of them missed. It is a plain recursive readdir now, with a test proving the two disagree. Both real-tree tests opened with `if (!existsSync(ROUTES)) return;`, which turned them into green no-ops if this package ever moved relative to apps/webapp; a missing routes directory is a hard failure now. Adds the packaging invariant the module-type fix needs to stay fixed. --- .github/workflows/pr_checks.yml | 5 + .../observability-map/test/cli.test.ts | 149 +++++++++++++++--- .../test/integration.test.ts | 97 +++++++++--- 3 files changed, 211 insertions(+), 40 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b0fbd6ac040..89b5dc8c29b 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -74,6 +74,11 @@ jobs: internal: - 'internal-packages/**' - 'packages/**' + # @internal/observability-map has one deliberate test against the live route tree + # (internal-packages/observability-map/test/integration.test.ts). Without this the + # internal suite never ran for a webapp-only PR, so a renamed route merged green and + # broke main, or the next unrelated internal-packages PR. + - 'apps/webapp/app/routes/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-internal.yml' - '.configs/**' diff --git a/internal-packages/observability-map/test/cli.test.ts b/internal-packages/observability-map/test/cli.test.ts index 2fd28955d85..3feffc08eb3 100644 --- a/internal-packages/observability-map/test/cli.test.ts +++ b/internal-packages/observability-map/test/cli.test.ts @@ -1,9 +1,52 @@ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { main, type Io } from "../src/cli.js"; -const REPORT_FILE = resolve(__dirname, "../../../observability-map.json"); +/** + * A routes tree of this package's own making. These tests used to run against + * `apps/webapp/app/routes` and assert that `/api/v1/token` exists and that `api.v1.runs.ts` is line + * 2 of the output, which is an assertion about the webapp's contents rather than about this CLI. + * A webapp-only pull request renaming a route broke them, and `pr_checks.yml` did not run this + * suite for such a pull request, so the break landed on whoever pushed next. The one deliberate + * real-tree test lives in `integration.test.ts` and asserts only invariants that survive churn. + */ +const ROUTES = mkdtempSync(join(tmpdir(), "obs-map-fixture-routes-")); + +const withWork = (name: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.${name}.findMany(); } catch (e) { return null; } + }`; + +const FIXTURES: Record = { + // Exact target, and also the prefix of the two below it. + "api.v1.runs.ts": withWork("run"), + "api.v1.runs.$runId.ts": withWork("run"), + "api.v1.runs.$runId.cancel.ts": withWork("run"), + "api.v1.token.ts": withWork("token"), + // Five routes sharing a prefix that is not itself a route, for the ambiguity warning and for the + // "and N more" tail it grows past four matches. + "admin.api.v1.runs-replication.start.ts": withWork("replication"), + "admin.api.v1.runs-replication.stop.ts": withWork("replication"), + "admin.api.v1.runs-replication.status.ts": withWork("replication"), + "admin.api.v1.runs-replication.retry.ts": withWork("replication"), + "admin.api.v1.runs-replication.purge.ts": withWork("replication"), + // Nothing applicable: exercises the not-measured note. + "resources.health.ts": `export const loader = () => new Response("ok");`, + // A directive whose id names no check, for the warning it has to produce. + "api.v1.typo.ts": `// obs-map-disable eror-classification -- typo\n${withWork("thing")}`, +}; + +beforeAll(() => { + for (const [name, source] of Object.entries(FIXTURES)) { + writeFileSync(join(ROUTES, name), source); + } + // A directory route, so the fixture covers both shapes `scanDirectory` walks. + mkdirSync(join(ROUTES, "_app.orgs.$slug")); + writeFileSync(join(ROUTES, "_app.orgs.$slug", "route.tsx"), withWork("organization")); +}); + +afterAll(() => rmSync(ROUTES, { recursive: true, force: true })); const capture = () => { const out: string[] = []; @@ -14,7 +57,7 @@ const capture = () => { const run = (...args: string[]) => { const c = capture(); - const code = main(["node", "cli.js", ...args], c.io); + const code = main(["node", "cli.js", `--routes=${ROUTES}`, ...args], c.io); return { code, out: c.out(), err: c.err() }; }; @@ -34,6 +77,12 @@ describe("map ", () => { expect(r.out).toContain("api.v1.token.ts"); }); + it("accepts a directory route by the path its directory segment spells", () => { + const r = run("/_app/orgs/:slug"); + expect(r.code).toBe(0); + expect(r.out).toContain("_app.orgs.$slug/route.tsx"); + }); + it("exits 1 with a message when nothing matches", () => { const r = run("/api/v1/does-not-exist"); expect(r.code).toBe(1); @@ -43,31 +92,89 @@ describe("map ", () => { it("warns when a prefix matches more than one route rather than silently taking the first", () => { const r = run("/admin/api/v1/runs-replication"); expect(r.code).toBe(0); - expect(r.err).toMatch(/matches \d+ entry points, showing the first/); + expect(r.err).toMatch(/matches 5 entry points, showing the first/); expect(r.err).toContain("Others:"); + expect(r.err).toContain("and 1 more"); }); - // `/api/v1/runs` is a prefix of a dozen others, and also a route in its own right. + // `/api/v1/runs` is a prefix of two others in the fixture, and also a route in its own right. it("prefers an exact match over the routes it is a prefix of", () => { const r = run("/api/v1/runs"); expect(r.err).toBe(""); expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); }); + + it("says so rather than printing a bare 100 when nothing applied", () => { + const r = run("/resources/health"); + expect(r.code).toBe(0); + expect(r.out).toContain("not measured"); + }); + + // B7. `map /api/v1/token --json` printed the text format and dropped the flag on the floor. + it("honours --json for a single route instead of printing the text format", () => { + const r = run("/api/v1/token", "--json"); + expect(r.code).toBe(0); + expect(r.out).not.toContain("CHECKS"); + const parsed = JSON.parse(r.out); + expect(parsed.fileName).toBe("api.v1.token.ts"); + expect(parsed.routePath).toBe("/api/v1/token"); + expect(Array.isArray(parsed.checks)).toBe(true); + }); }); describe("map", () => { - // The flag is the only thing standing between a test run and a file written into the repo root, - // so the test has to check the file, not just the exit code. + // The flag is the only thing standing between a run and a written report, so the test has to + // check the file, not just the exit code. `--out` keeps that file in a temp directory: this test + // used to delete `observability-map.json` from the repo root and never put it back. it("renders the whole report without writing when asked not to", () => { - const existedBefore = existsSync(REPORT_FILE); - if (existedBefore) rmSync(REPORT_FILE); + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); - const r = run("--no-write"); + const r = run("--out=" + out, "--no-write"); expect(r.code).toBe(0); expect(r.out).toContain("COVERAGE"); expect(r.out).toContain("FIX FIRST"); - expect(existsSync(REPORT_FILE)).toBe(false); + expect(existsSync(out)).toBe(false); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("writes the report where --out names it when not asked to skip the write", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out); + + expect(r.code).toBe(0); + expect(existsSync(out)).toBe(true); + const parsed = JSON.parse(readFileSync(out, "utf8")); + expect(parsed.entries.length).toBe(Object.keys(FIXTURES).length + 1); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + +// B6. Stdout can be JSON a caller parses, so the warning goes to stderr in every mode, and the +// terminal report carries it too (see report.test.ts). +describe("warning about a suppression that names no check", () => { + it("names the file and the bad id on stderr for the whole report", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.err).toContain("api.v1.typo.ts"); + expect(r.err).toContain("eror-classification"); + }); + + it("warns for a single route without putting the warning in the json", () => { + const r = run("/api/v1/typo", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("eror-classification"); + expect(JSON.parse(r.out).fileName).toBe("api.v1.typo.ts"); + }); + + it("says nothing on stderr for a route whose directives all name a check", () => { + const r = run("/api/v1/token"); + expect(r.err).toBe(""); }); }); @@ -79,22 +186,24 @@ describe("map --routes=", () => { `export const loader = () => new Response("ok");` ); - const r = run("--routes=" + dir, "--json", "--no-write"); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir, "--json", "--no-write"], c.io); - expect(r.code).toBe(0); - const parsed = JSON.parse(r.out); + expect(code).toBe(0); + const parsed = JSON.parse(c.out()); expect(parsed.entries).toHaveLength(1); expect(parsed.entries[0].fileName).toBe("resources.only.ts"); - rmSync(dir, { recursive: true }); + rmSync(dir, { recursive: true, force: true }); }); it("exits 1 with a message when the directory does not exist", () => { const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); - const r = run("--routes=" + dir); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir], c.io); - expect(r.code).toBe(1); - expect(r.err).toContain("not a readable directory"); - expect(r.out).toBe(""); + expect(code).toBe(1); + expect(c.err()).toContain("not a readable directory"); + expect(c.out()).toBe(""); }); }); diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index 4db7dfc223a..ca4f22892c0 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -1,25 +1,42 @@ -import { existsSync, readdirSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { scanDirectory, scanFile } from "../src/scan.js"; import { buildReport } from "../src/score.js"; import { SCORED_CHECK_IDS } from "../src/checks/index.js"; +/** + * The one deliberate coupling to `apps/webapp/app/routes` in the suite. Everything else, including + * the CLI tests, runs against a fixture tree of this package's own making. + * + * The coupling is acceptable because nothing here names a route or a count: the scan must not + * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those + * survive routes being added, renamed and deleted, and they are the only things a fixture tree + * cannot tell us, since a fixture only contains shapes somebody thought to write down. `pr_checks. + * yml`'s `internal` filter watches `apps/webapp/app/routes/**` so a webapp pull request runs this + * rather than leaving the break for whoever pushes next. + */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); /** - * Counts route module candidates the same way `scanDirectory` walks the tree, without scanning - * their contents: a flat `.ts`/`.tsx` file, or one `route.ts`/`route.tsx` per directory. This is a - * structural upper bound, not a golden number - not every candidate exports a loader or action, so - * `entryPoints.length` must stay below it, and the route set is free to grow or shrink over time - * without breaking this test. + * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk, which + * looks at flat files and at one `route.ts`/`route.tsx` per directory: this used to be a verbatim + * copy of that walk, which made `entryPoints.length < countCandidates()` a tautology that could + * not fail for any route shape both of them missed. */ -function countCandidates(dir: string): number { +function countRouteModuleFiles(dir: string): number { let count = 0; for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.isDirectory()) { - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (child.isFile() && (child.name === "route.ts" || child.name === "route.tsx")) count++; - } + count += countRouteModuleFiles(join(dir, entry.name)); continue; } if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; @@ -27,19 +44,60 @@ function countCandidates(dir: string): number { return count; } +beforeAll(() => { + // A hard failure rather than the `if (!existsSync(ROUTES)) return;` these tests opened with: if + // this package moves relative to apps/webapp, the real-tree coverage must disappear loudly. + if (!existsSync(ROUTES)) { + throw new Error(`the webapp routes directory is missing: ${ROUTES}`); + } +}); + +// B7. The build emits ESM (`module: ESNext`, and `cli.ts` uses `import.meta`) while `main` and +// `types` advertised it to a package.json with no module type. On node 24 that loads with a +// MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw older nodes give, which +// is a warning about an artifact this package tells other packages to import. +describe("the package it advertises", () => { + const manifest = JSON.parse( + readFileSync(resolve(__dirname, "../package.json"), "utf8") + ) as Record; + + // Asserted flat rather than behind an `if (!advertised) return`, which is the silent skip this + // round removed from the tests below: the decision was to keep the entry point and declare the + // module type, so dropping the entry point later should have to edit this, not slip past it. + it("declares the module type its build emits alongside the entry point it advertises", () => { + expect(manifest.main).toBe("./dist/src/index.js"); + expect(manifest.types).toBe("./dist/src/index.d.ts"); + expect(manifest.type).toBe("module"); + }); +}); + +describe("counting candidates independently of the scanner", () => { + // The counter is only worth having if it disagrees with the scanner somewhere. It does: the + // scanner attributes nothing to a nested file that is not `route.ts`/`route.tsx`, and the + // counter counts every module file at every depth. + it("counts a nested non-route file the scanner does not attribute to any route", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-count-")); + mkdirSync(join(dir, "components")); + writeFileSync( + join(dir, "components", "helper.ts"), + `export const loader = () => new Response("ok");` + ); + + expect(countRouteModuleFiles(dir)).toBe(1); + expect(scanDirectory(dir).entryPoints).toHaveLength(0); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + describe("scanning the real webapp routes", () => { - it("parses every route file without crashing", () => { - if (!existsSync(ROUTES)) return; + it("parses every route file and produces a report inside a wide band", () => { const { entryPoints, parseFailures } = scanDirectory(ROUTES); - // Invariants, not exact numbers: the route set changes constantly. - expect(entryPoints.length).toBeGreaterThan(200); - expect(entryPoints.length).toBeLessThan(countCandidates(ROUTES)); + expect(parseFailures).toEqual([]); - }); + expect(entryPoints.length).toBeGreaterThan(100); + expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); - it("produces a report with a score in range", () => { - if (!existsSync(ROUTES)) return; - const { entryPoints, parseFailures } = scanDirectory(ROUTES); const report = buildReport(entryPoints, parseFailures); expect(report.global).toBeGreaterThanOrEqual(0); expect(report.global).toBeLessThanOrEqual(100); @@ -51,7 +109,6 @@ describe("scanning the real webapp routes", () => { // 176, because every entry whose only applicable checks were suppressed dropped out of the // mean. Measured must not move: every entry point that had something applicable still does. it("suppressing every scored check on every real route does not raise the global", () => { - if (!existsSync(ROUTES)) return; const { entryPoints, parseFailures } = scanDirectory(ROUTES); const before = buildReport(entryPoints, parseFailures); From 7273d128c1e2912bc47f3219040ec41034bb72dc Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:17:04 +0100 Subject: [PATCH 068/117] fix(observability-map): mark a suppression-caused row, and post when only a suppression moved Two halves of the same hole. hasDelta compared the score, the measured flag and newly failing checks, so a pull request that only suppressed findings moved none of them and posted nothing, while a MISTYPED directive did post because the unknown warnings were compared. It now also compares the per-entry suppression set, the audit gap and the context gap. All three are reachable with nothing else moving: an already-failing check being silenced, audit-trail going fail to pass (the webapp's first audit record), and the CONTEXT figure moving behind a suppression, which reads pre-suppression data. MapReport.suppressions is deliberately not compared, because its totals are summed from the same per-entry arrays the loop already walks. The changed-entries table now includes a row when the suppression set moved even if no score did, and marks any row a new suppression caused. Suppressing a check that was PASSING drops the score by round A's cap and rendered a row with an empty "now failing" column, sorted among the real regressions: _app.@.orgs.$organizationSlug.$.tsx renders 67 to 50 exactly that way on the real tree. The score movement is honest; the unmarked row was not. An earlier report claimed this class could not be constructed, which was wrong. Caps the unknown-suppression section of the comment at ten files. It was the one unbounded section, and one mistyped directive applied tree wide renders 87,938 characters against GitHub's 65,536 limit, for a 422 the job then swallows. --- .../observability-map/src/report/prComment.ts | 83 ++++++-- .../observability-map/test/prComment.test.ts | 183 ++++++++++++++++++ 2 files changed, 249 insertions(+), 17 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 80c0b4b70fe..58010e97efd 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -13,6 +13,14 @@ export const MARKER = ""; const MAX_CHANGED_ROWS = 15; +/** + * Every other section of this comment is bounded by construction. This one was not, and a single + * mistyped directive applied tree wide renders one line per file: 87,938 characters against + * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what + * stops the whole comment being lost to the section warning about a typo. + */ +const MAX_UNKNOWN_SUPPRESSION_LINES = 10; + // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. @@ -49,12 +57,25 @@ const NOT_MEASURED = "not measured"; */ const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); +/** Ids suppressed at head that were not suppressed at base. `[]` covers both "no change" and a + * suppression being removed, which shows up as the score going back up. */ +const newlySuppressed = (head: ScoredEntry, base: ScoredEntry | undefined): string[] => + head.suppressed.filter((id) => !(base?.suppressed ?? []).includes(id)); + type ChangedRow = { routePath: string; sensitive: boolean; baseScore: number | string; headScore: number | string; nowFailing: string[]; + /** + * Ids this pull request newly suppressed on the entry. Rendered on the route cell, because a + * suppression added to a check that was passing drops the score by round A's cap and produces a + * row with an empty "now failing" column, which is indistinguishable from a real regression: + * `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 that way. The score movement is honest, + * the row without this note was not. + */ + suppressed: string[]; /** * How much the entry got worse, used to sort the table. A new entry has no base score to * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the @@ -77,20 +98,25 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re // A new entry that passes every check it was measured against has nothing to fix, which is // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a // different statement and still gets a row, since its 100 is a placeholder rather than a pass. - if (h.measured && h.score === 100) continue; + if (h.measured && h.score === 100 && h.suppressed.length === 0) continue; rows.push({ routePath: h.routePath, sensitive: h.sensitive, baseScore: "new", headScore: scoreCell(h), nowFailing: failingIds(h), + suppressed: newlySuppressed(h, b), drop: h.measured ? 100 - h.score : 0, }); continue; } - // Measured state is part of what changed: a measured-to-unmeasured transition can leave the - // score untouched at its placeholder value, and skipping on the number alone hid it. - if (b.measured === h.measured && b.score === h.score) continue; + // Measured state and the suppression set are both part of what changed. A measured-to- + // unmeasured transition can leave the score at its placeholder value, and suppressing a check + // that was already failing moves no score at all, so skipping on the number alone hid both. A + // pull request whose whole purpose is to silence findings has to produce a row. + const suppressed = newlySuppressed(h, b); + const suppressionChanged = suppressed.length > 0 || h.suppressed.length !== b.suppressed.length; + if (b.measured === h.measured && b.score === h.score && !suppressionChanged) continue; const baseFailing = new Set(failingIds(b)); rows.push({ routePath: h.routePath, @@ -98,6 +124,7 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re baseScore: scoreCell(b), headScore: scoreCell(h), nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + suppressed, drop: b.measured && h.measured ? b.score - h.score : 0, }); } @@ -133,8 +160,9 @@ function whatChangedSection(head: MapReport, base: MapReport | null): string[] { lines.push("| route | base | head | now failing |"); lines.push("| --- | --- | --- | --- |"); for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + const note = row.suppressed.length > 0 ? ` (suppressed: ${row.suppressed.join(", ")})` : ""; lines.push( - `| ${row.routePath} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + `| ${row.routePath}${note} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` ); } if (rows.length > MAX_CHANGED_ROWS) { @@ -173,26 +201,43 @@ function fixFirstSection(head: MapReport): string[] { return lines; } +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + /** * Whether this pull request moves the report at all, so the job can stay quiet when it does not. * - * True on: no base to compare against, a global score change, an entry added or removed, an - * entry's score or measured state changed, a check that fails at head and did not at base, a - * change in the parse failure count, or a change in the unknown suppression warnings. The last two - * are here because both render a line in the comment, so a run that gains one has moved the report - * even though no score did. + * The rule this has to satisfy is that it must be true whenever `renderPrComment` would say + * something different, because anything it misses is a change the pull request silently does not + * report. So it covers every figure the comment renders, not only the score: the global, the + * per-entry score, measured state and suppression set, an entry added or removed, a check failing + * at head that did not at base, the parse failure count, the unknown suppression warnings, and the + * audit and context gaps. + * + * The per-entry suppression set and the two gaps are the half that was missing, and it ran the + * dangerous way. Suppressing an already-failing check moves no score, no measured flag and no new + * failure, so a pull request whose entire purpose was to silence findings posted nothing, while a + * mistyped directive did post because the unknown warnings were compared. `audit-trail` going from + * fail to pass, the first audit record in the webapp, was in the same hole, and so was the CONTEXT + * figure moving behind a suppression, since that figure reads pre-suppression data. + * + * The terms overlap on purpose, and what is defended is that their union is complete rather than + * that each one is load bearing. Four are individually reachable, each with a test that fails when + * only that term is removed: the parse failure count, the unknown suppression warnings, the audit + * gap and the context gap. The global, the removed-entry check and the per-entry score are each + * shadowed by another term today, and are kept because which term shadows which depends on the + * shape of the change rather than on anything stable. * - * The residual: adding a suppression whose check was already failing moves nothing this asks - * about. The score is capped at the pre-suppression ratio so it cannot move, and the finding - * leaves the fix list quietly. `SUPPRESSED` in the terminal report is where that shows up. + * `MapReport.suppressions` is the one term deliberately left out. Its two totals are summed from + * the very per-entry `suppressed` arrays the loop below compares one by one, so it cannot move + * without the loop moving. That is arithmetic rather than a happy overlap. */ export function hasDelta(head: MapReport, base: MapReport | null): boolean { if (!base) return true; if (head.global !== base.global) return true; if (head.parseFailures.length !== base.parseFailures.length) return true; - if (JSON.stringify(head.unknownSuppressions) !== JSON.stringify(base.unknownSuppressions)) { - return true; - } + if (!same(head.unknownSuppressions, base.unknownSuppressions)) return true; + if (!same(head.auditGap, base.auditGap)) return true; + if (!same(head.contextGap, base.contextGap)) return true; const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); const headFiles = new Set(head.entries.map((e) => e.fileName)); @@ -202,6 +247,7 @@ export function hasDelta(head: MapReport, base: MapReport | null): boolean { const b = baseByFile.get(h.fileName); if (!b) return true; if (b.measured !== h.measured || b.score !== h.score) return true; + if (!same(h.suppressed, b.suppressed)) return true; const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; } @@ -260,7 +306,10 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string const context = contextLine(head); if (context) lines.push(context); const unknown = unknownSuppressionLines(head); - lines.push(...unknown); + lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); + if (unknown.length > MAX_UNKNOWN_SUPPRESSION_LINES) { + lines.push(`and ${unknown.length - MAX_UNKNOWN_SUPPRESSION_LINES} more files with unknown ids`); + } if (audit || context || unknown.length > 0) lines.push(""); lines.push( diff --git a/internal-packages/observability-map/test/prComment.test.ts b/internal-packages/observability-map/test/prComment.test.ts index 4299b1ae3a7..8bea1e2edf3 100644 --- a/internal-packages/observability-map/test/prComment.test.ts +++ b/internal-packages/observability-map/test/prComment.test.ts @@ -184,6 +184,109 @@ describe("renderPrComment", () => { }); }); + // I4. A suppression added to a check that was PASSING drops the score by round A's cap and + // produces a row with an empty "now failing" column, sorted among the real regressions. On the + // real tree `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 exactly that way. + describe("a row a suppression caused", () => { + // Two of three applicable scored checks pass, so suppressing one of the passes takes the + // visible ratio from 2/3 to 1/2, which is the 67 to 50 the real tree renders on + // `_app.@.orgs.$organizationSlug.$.tsx`. The catch has to decide something for + // error-classification to apply at all, and nothing may name a tenant, or the ratio is 3/3. + const twoOfThree = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }`; + const silence = (id: string, source: string) => + `// obs-map-disable ${id} -- silenced\n${source}`; + + it("says so on the route, so it is not read as a regression", () => { + const base = buildReport([scanFile("api.v1.auth.tokens.ts", twoOfThree)!], []); + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", silence("error-classification", twoOfThree))!], + [] + ); + expect(base.entries[0]!.score).toBe(67); + expect(head.entries[0]!.score).toBe(50); + + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + // The column that would otherwise explain the drop is empty, which is the whole problem. + expect(row.split("|")[4]!.trim()).toBe(""); + }); + + // I3. This one moves no score at all, so before the suppression set was compared it produced + // no row and no comment: a pull request whose whole purpose is to silence findings was silent. + it("appears even when suppressing an already-failing check moved no score", () => { + const base = buildReport([scanFile("api.v1.t.ts", brokenSource)!], []); + const head = buildReport( + [scanFile("api.v1.t.ts", silence("error-classification", brokenSource))!], + [] + ); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.global).toBe(base.global); + + const out = renderPrComment(head, base); + expect(out).not.toContain("No entry point this PR touches changed its score."); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/t "))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + }); + + it("says nothing about suppression on a row that has none", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).not.toContain("suppressed:"); + }); + + it("gives a new entry that lands at 100 only because of a suppression a row", () => { + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const head = buildReport( + [ + scanFile("api.v1.a.ts", cleanSource)!, + scanFile("api.v1.new.ts", silence("request-context", cleanSource))!, + ], + [] + ); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/new"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: request-context)"); + }); + }); + + // M5. Every other section is bounded by construction; this one rendered one line per file, and a + // tree-wide typo took the comment past GitHub's 65,536 character limit for a 422 nobody sees. + it("caps the unknown-suppression lines instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push( + scanFile( + `api.v1.route${i}.ts`, + `// obs-map-disable eror-classification -- typo\n${brokenSource}` + )! + ); + } + const head = buildReport(entries, []); + const out = renderPrComment(head, null); + + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(10); + expect(out).toContain("and 30 more files with unknown ids"); + expect(out.length).toBeLessThan(65536); + }); + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; const sensitiveSmallDropHead = scanFile( @@ -346,6 +449,86 @@ describe("hasDelta", () => { expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); }); + // I3. These three are the half that was missing, and it ran the dangerous way: a pull request + // that only silences findings posted nothing, while a mistyped directive did post. + it("is true when a suppression was added to a check that was already failing", () => { + const base = one("api.v1.t.ts", brokenSource); + const head = one( + "api.v1.t.ts", + `// obs-map-disable error-classification -- silenced\n${brokenSource}` + ); + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.measured).toBe(base.measured); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when the audit gap closed, which no score reports", () => { + const audited = `import { auditLog } from "~/services/audit.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await auditLog("token.created", { tokenId: token.id }); + return json(token); + }`; + const unaudited = `import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + return json(token); + }`; + const head = one("api.v1.auth.tokens.ts", audited); + const base = one("api.v1.auth.tokens.ts", unaudited); + expect(head.auditGap).not.toEqual(base.auditGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // The CONTEXT line reads pre-suppression data, so with request-context suppressed its figure can + // move while the post-suppression checks, the score and the global all stay put. The comment + // says "0 of 1" and then "1 of 1"; nothing else in the report moves at all. + it("is true when the context figure moved behind a suppression", () => { + const silence = "// obs-map-disable request-context -- reported as a figure\n"; + const namesNobody = `${silence}import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.envVar.update({ where: {}, data: {} }); } catch (e) { return null; } + }`; + const namesTenant = `${silence}import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ params }) { + try { return await prisma.envVar.update({ where: {}, data: {} }); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); return null; } + }`; + const head = one("api.v1.envvars.ts", namesTenant); + const base = one("api.v1.envvars.ts", namesNobody); + + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.entries[0]!.suppressed).toEqual(base.entries[0]!.suppressed); + expect(head.contextGap).not.toEqual(base.contextGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // Suppressing a check that was not applicable anyway changes no status and no score, and moving + // that suppression between two entries leaves the report-level totals identical too. The + // per-entry suppression comparison is the only thing left that can see it. + it("is true when a suppression of an inapplicable check moved between entries", () => { + const silenced = `// obs-map-disable auth-boundary -- not a user-facing route\n${brokenSource}`; + const head = buildReport( + [scanFile("api.v1.a.ts", silenced)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", silenced)!], + [] + ); + expect(head.suppressions).toEqual(base.suppressions); + expect(head.global).toBe(base.global); + expect(head.entries.map((e) => e.score)).toEqual(base.entries.map((e) => e.score)); + const statuses = (r: typeof head) => + r.entries.map((e) => e.checks.map((c) => `${c.id}=${c.status}`).join(" ")); + expect(statuses(head)).toEqual(statuses(base)); + expect(hasDelta(head, base)).toBe(true); + }); + it("is true when a suppression names a check that does not exist", () => { const head = one( "api.v1.a.ts", From b0d6dc0cc528b5c9d88387bc7f587de495663196 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:17:24 +0100 Subject: [PATCH 069/117] ci(observability-map): make a failed comment lookup mean one thing, and run the tests here The render step read a missing /tmp/existing-comment-id as "a comment exists" and emitted the resolved state; the upsert step read it as "no id" and POSTed. So a transient lookup failure either added a second marker comment beside the stale one, or announced that an earlier push's findings were gone on a pull request that never had findings, which is the thing the post-only-on-delta change exists to prevent. The comment above it claimed the two agreed. The guess bought nothing in either direction: with no id, the render step's --existing-comment can only ever produce a body the upsert then POSTs. The lookup now retries three times, and on a failure that outlasts the retries it drops a sentinel both steps read, so the run posts nothing. Worst case is no comment this run, which the next push fixes. gh api --paginate runs the jq once per page, so a marker comment on two different pages yielded two ids: a newline in the PATCH url and a step that dies silently, or a second comment whose stale findings stand forever. The oldest id is taken, and a count above one is logged. The stale-report comment now goes through the same temp file dance as every other write, so a renderer that exits non-zero cannot leave a 0-byte comment.md for the upsert to skip in silence. Adds a blocking unit-tests job running this package alone. That is what protects the one test against the live route tree, in place of widening pr_checks.yml's internal filter, which ran all eighteen internal packages with postgres, clickhouse, redis and electric for the same protection. This workflow already triggers on exactly the paths that can break that test, and unlike the report job this one needs no token, so it runs for fork pull requests and is meant to fail. Verified by extracting the three comment step bodies and running them against a stubbed gh across seven cases: hard lookup failure, flaky lookup, no delta with and without an existing comment, two marker comments across pages, malformed head report, and a head scan that produced nothing. --- .github/workflows/observability-map.yml | 117 +++++++++++++++++++----- 1 file changed, 96 insertions(+), 21 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 4d30b29cc5b..3aa8f5018b5 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -50,6 +50,39 @@ jobs: pnpm --filter @internal/observability-map exec vitest run \ test/mutationCorpus.test.ts --testTimeout=120000 --disable-console-intercept + # The package has one deliberate test against the live route tree + # (internal-packages/observability-map/test/integration.test.ts), and this workflow already + # triggers on exactly the paths that can break it. Running just this package here is what keeps + # a webapp-only pull request from merging a renamed route green and leaving the failure for main + # or for the next unrelated internal-packages pull request. Unlike the report job below, this one + # is meant to be able to fail, and it runs for fork pull requests because it needs no token. + unit-tests: + name: 🧪 Package tests + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: ⎔ Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 🧪 Run the package tests + run: pnpm --filter @internal/observability-map run test + report: runs-on: warp-ubuntu-latest-x64-4x # Only this job comments, so only this job gets the write. @@ -103,25 +136,50 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi - # Looked up before the render step because the render step needs it: with no delta to report, - # a pull request that already has a comment gets a resolved state rather than being left with - # findings that no longer exist, and one that does not gets nothing at all. The upsert step - # reuses the id rather than asking again. + # Looked up before the render step because the render decision needs it: with no delta to + # report, a pull request that already has a comment gets a resolved state rather than being + # left with findings that no longer exist, and one that does not gets nothing at all. The + # upsert step reuses the id rather than asking twice. + # + # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than + # silence here: this step is the only thing that knows which comment to PATCH, so a guess of + # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either + # adds a second marker comment beside the stale one, or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. - name: 🔍 Look for a comment from an earlier push continue-on-error: true env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | - rm -f /tmp/existing-comment-id - if gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty' \ - > /tmp/existing-comment-id.partial; then - mv /tmp/existing-comment-id.partial /tmp/existing-comment-id - else - rm -f /tmp/existing-comment-id.partial - echo "comment lookup failed; assuming a comment exists so a stale one still gets replaced" >&2 + rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed + found="" + ok="" + for attempt in 1 2 3; do + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + touch /tmp/comment-lookup-failed + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 fi + printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id # continue-on-error for the same reason as the scan: a rendering bug must not turn the job # red. An empty /tmp/comment.md means there is nothing to post, which is a decision @@ -132,8 +190,24 @@ jobs: rm -f /tmp/comment.md render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte + # comment.md for the upsert to skip in silence. + emit() { + if render "$@" > /tmp/comment.md.partial; then + mv /tmp/comment.md.partial /tmp/comment.md + return 0 + fi + rm -f /tmp/comment.md.partial + return 1 + } + + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" >&2 + exit 0 + fi + if [ ! -s /tmp/head.json ]; then - render --scan-failed > /tmp/comment.md + emit --scan-failed || echo "could not render the stale-report comment either" >&2 exit 0 fi @@ -142,19 +216,14 @@ jobs: base="-" fi - # An absent file means the lookup itself failed, and assuming there is a comment is the - # safer half of that guess: it can post a redundant resolved state, not leave a stale one. flags=() - if [ ! -f /tmp/existing-comment-id ] || [ -s /tmp/existing-comment-id ]; then + if [ -s /tmp/existing-comment-id ]; then flags=(--existing-comment) fi - if render /tmp/head.json "$base" "${flags[@]}" > /tmp/comment.md.partial; then - mv /tmp/comment.md.partial /tmp/comment.md - else - rm -f /tmp/comment.md.partial + if ! emit /tmp/head.json "$base" "${flags[@]}"; then echo "render failed; falling back to the stale-report comment" >&2 - render --scan-failed > /tmp/comment.md + emit --scan-failed || echo "could not render the stale-report comment either" >&2 fi # continue-on-error for the same reason: a transient gh api failure (rate limit, network) @@ -165,6 +234,12 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | + # The same sentinel the render step reads, so the two cannot disagree about what a failed + # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs. + if [ -f /tmp/comment-lookup-failed ]; then + echo "the comment lookup failed, so this run posts nothing" + exit 0 + fi if [ ! -s /tmp/comment.md ]; then echo "nothing to post: this pull request does not move the report" exit 0 From bccaf9f457ce5357094bc4c6463ab0b3af9bf2a1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Sat, 1 Aug 2026 23:18:21 +0100 Subject: [PATCH 070/117] ci: revert the observability-map widening of the internal filter Widening pr_checks.yml's internal filter to apps/webapp/app/routes/** ran the whole eighteen package internal suite, twelve shards with postgres, clickhouse, redis and electric, for every pull request touching a route file, all to protect one test. The unit-tests job in observability-map.yml runs that package alone on the same paths and is the proportionate version. Adds the guard the docstring checker cannot give: it walks src/ only, so workflow prose goes unpoliced, and the lookup defect fixed alongside this was two steps disagreeing under a comment claiming they agreed. The new test is a text check over the workflow, and says so rather than selling itself as coverage of the file. --- .github/workflows/pr_checks.yml | 5 --- .../test/integration.test.ts | 44 +++++++++++++++++-- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 89b5dc8c29b..b0fbd6ac040 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -74,11 +74,6 @@ jobs: internal: - 'internal-packages/**' - 'packages/**' - # @internal/observability-map has one deliberate test against the live route tree - # (internal-packages/observability-map/test/integration.test.ts). Without this the - # internal suite never ran for a webapp-only PR, so a renamed route merged green and - # broke main, or the next unrelated internal-packages PR. - - 'apps/webapp/app/routes/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-internal.yml' - '.configs/**' diff --git a/internal-packages/observability-map/test/integration.test.ts b/internal-packages/observability-map/test/integration.test.ts index ca4f22892c0..f16c7b28b50 100644 --- a/internal-packages/observability-map/test/integration.test.ts +++ b/internal-packages/observability-map/test/integration.test.ts @@ -20,9 +20,13 @@ import { SCORED_CHECK_IDS } from "../src/checks/index.js"; * The coupling is acceptable because nothing here names a route or a count: the scan must not * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those * survive routes being added, renamed and deleted, and they are the only things a fixture tree - * cannot tell us, since a fixture only contains shapes somebody thought to write down. `pr_checks. - * yml`'s `internal` filter watches `apps/webapp/app/routes/**` so a webapp pull request runs this - * rather than leaving the break for whoever pushes next. + * cannot tell us, since a fixture only contains shapes somebody thought to write down. + * + * What runs this for a webapp pull request is the `unit-tests` job in + * `.github/workflows/observability-map.yml`, which already triggers on `apps/webapp/app/routes/**` + * and runs this package alone. Widening `pr_checks.yml`'s `internal` filter to those paths was + * tried and reverted: it ran all eighteen internal packages, twelve shards with postgres, + * clickhouse, redis and electric, to protect this one test. */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); @@ -71,6 +75,40 @@ describe("the package it advertises", () => { }); }); +/** + * The one thing the docstring checker cannot reach. It walks `src/` only, so workflow prose is + * unpoliced, and the C1 defect was exactly that: two steps disagreeing about what a missing + * `/tmp/existing-comment-id` meant, under a comment claiming they agreed. The render step read it + * as "a comment exists" and emitted the resolved state, the upsert step read it as "no id" and + * POSTed, so a transient lookup failure either added a second marker comment beside the stale one + * or announced that findings were gone on a pull request that never had any. + * + * This is a text check over the workflow, not a parse of its semantics, so it catches one shape of + * that class and no other. Named as such rather than sold as coverage of the file. + */ +describe("the report workflow's two readers of the comment lookup", () => { + const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); + + /** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ + function steps(): string[] { + if (!existsSync(WORKFLOW)) throw new Error(`the report workflow is missing: ${WORKFLOW}`); + const text = readFileSync(WORKFLOW, "utf8"); + return text.split(/^ {6}- name: /m).slice(1); + } + + it("both honour the same sentinel, so a failed lookup cannot mean two things", () => { + const readers = steps().filter((step) => step.includes("/tmp/existing-comment-id")); + expect(readers.length).toBeGreaterThanOrEqual(2); + expect(readers.filter((step) => !step.includes("/tmp/comment-lookup-failed"))).toEqual([]); + }); + + it("takes one id from a lookup that paginates rather than passing every line on", () => { + const lookup = steps().find((step) => step.includes('startswith(""; const MAX_CHANGED_ROWS = 15; /** - * Every other section of this comment is bounded by construction. This one was not, and a single - * mistyped directive applied tree wide renders one line per file: 87,938 characters against + * A mistyped directive applied tree wide renders one line per file: 87,938 characters against * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what * stops the whole comment being lost to the section warning about a typo. */ const MAX_UNKNOWN_SUPPRESSION_LINES = 10; +/** + * The same failure in the other section that grows with the size of the tree. `delegating` holds + * one file name per route whose body lives elsewhere, joined into a single line, and a codemod that + * moves route bodies into `.server.ts` modules is both the refactor this feature exists to notice + * and the one that makes the list tree-sized. The cap was claimed here before it was written: the + * note above used to open "every other section of this comment is bounded by construction", which + * was not true of this one. + * + * Fifteen matches the changed-entries table rather than the ten above, because a delegating file + * name is one comma-separated item rather than a line naming every known check. The bound that + * matters is the section's worst case: the longest route file name in the tree is 130 characters, + * so fifteen of those plus separators is under 2kB against GitHub's 65,536. + */ +const MAX_DELEGATED_ROUTES = 15; + // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. @@ -314,7 +328,7 @@ export function renderPrComment(head: MapReport, base: MapReport | null): string if (audit) lines.push(audit); const context = contextLine(head); if (context) lines.push(context); - const delegated = delegatedLines(head); + const delegated = delegatedLines(head, MAX_DELEGATED_ROUTES); lines.push(...delegated); const unknown = unknownSuppressionLines(head); lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); diff --git a/internal-packages/observability-map/src/report/terminal.test.ts b/internal-packages/observability-map/src/report/terminal.test.ts index 74221ddc7a5..61b7aa9c462 100644 --- a/internal-packages/observability-map/src/report/terminal.test.ts +++ b/internal-packages/observability-map/src/report/terminal.test.ts @@ -194,6 +194,24 @@ describe("rendering honestly when there is nothing to say", () => { expect(out).not.toContain("No audit helper exists"); }); + // Round E item 2. The other half of the branch, which the test above did not pin. A zero used to + // print "No audit helper exists in the webapp", which is false: `models/admin.server.ts` writes + // `prisma.impersonationAuditLog.create(...)` and `AUDIT_SYMBOLS` names the helpers that reach it. + // The full tree reads 3 of 49 so nobody sees the sentence today, and a scan of any subset with no + // impersonation route in it brings the sentence straight back. + it("does not claim the webapp has no audit helper when nothing reached one", () => { + const unaudited = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + return json(await prisma.token.create({ data: {} })); + }` + )!; + const out = renderTerminal(buildReport([unaudited], [])); + expect(out).toContain("AUDIT 0 of 1 sensitive mutations record an actor. 1 without one."); + expect(out).not.toContain("No audit helper exists"); + }); + it("says nothing about audit when no sensitive mutation was found", () => { const plain = scanFile( "resources.things.ts", diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 843501b68c4..dc59e205342 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -50,18 +50,29 @@ export function unknownSuppressionLines(report: MapReport): string[] { ); } -/** The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when - * there is nothing to report, i.e. no sensitive mutation exists. */ +/** + * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when + * there is nothing to report, i.e. no sensitive mutation exists. + * + * One shape for every count, and no branch on the count, because the branch is what carried the + * bug. A zero used to print "No audit helper exists in the webapp", which is false: the helper + * exists and `AUDIT_SYMBOLS` names it, `apps/webapp/app/models/admin.server.ts` writes + * `prisma.impersonationAuditLog.create(...)`, and `webappSymbols.test.ts` proves those symbols + * resolve. The count was already correct, so the sentence was the only wrong thing and it is gone + * rather than reworded. A zero here means nothing reached the helper, which is what + * "0 of N record an actor" already says. + * + * The full-tree scan reads 3 of 49 today, so the zero branch is not taken and nobody sees it. It is + * one `--routes=` away from being taken, and one reshaped impersonation route away on the full + * tree, which is why removing it beats leaving it unreachable. + */ export function auditLine(report: MapReport): string | null { const { sensitiveMutations, withAudit } = report.auditGap; if (sensitiveMutations === 0) return null; - // The closing sentence is a claim about the codebase, so it is only made when the figure in - // front of it supports it. It was printed unconditionally, including next to a non-zero count. - const gap = - withAudit === 0 - ? " No audit helper exists in the webapp." - : ` ${sensitiveMutations - withAudit} without one.`; - return `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor.${gap}`; + return ( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `${sensitiveMutations - withAudit} without one.` + ); } /** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ @@ -83,14 +94,21 @@ export function contextLine(report: MapReport): string | null { * The DELEGATED lines, shared with `prComment.ts`. Empty when every route's body is in its own * file. Worded as a shortfall rather than a note: these routes left the denominator and no check * looked at any of them. + * + * `limit` caps how many file names are named, for the caller that has a size limit to respect. The + * count in front of the list is always the full one, so a capped line still reports the real + * shortfall and only shortens the evidence. The terminal passes no limit and prints them all. */ -export function delegatedLines(report: MapReport): string[] { +export function delegatedLines(report: MapReport, limit = Infinity): string[] { if (report.delegating.length === 0) return []; const n = report.delegating.length; + const shown = report.delegating.slice(0, limit); + const tail = n > shown.length ? `, and ${n - shown.length} more` : ""; return [ `DELEGATED ${n} route${n === 1 ? "" : "s"} keep${n === 1 ? "s" : ""} the body in another ` + `module, so nothing here was checked and ${n === 1 ? "it is" : "they are"} out of the score: ` + - report.delegating.join(", "), + shown.join(", ") + + tail, ]; } diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 458c6328e81..70efae6eb33 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -2711,6 +2711,77 @@ describe("scanFile: the signals auth-scope reads", () => { expect(ep!.loaderScopesByCaller).toBe(false); }); + // Round E item 3. Neither of the two conditions above constrains the CALLEE, so a log line + // carrying the caller's id satisfied a tenant-scoping security check. Cheaper to write than the + // dead object, and unlike the dead object it survives review, because a log line is real code + // somebody wants. + it("does not read a caller id handed to a log call as a scope", () => { + const logger = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.error("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(logger!.loaderScopesByCaller).toBe(false); + + // A different logger family: `LOGGER_CALLEE` does not match `console.warn`, so this is the + // second sink rather than a restatement of the first. + const console_ = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + console.warn("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(console_!.loaderScopesByCaller).toBe(false); + }); + + // The refusal is made at the call, not at the property, so burying the id under the depth of + // nesting a real filter has does not get it past. + it("does not read a caller id nested inside a log call's payload as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.info("looking up", { where: { OR: [{ userId: user.id }] } }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The response body is the other sink that takes the same object and cannot narrow a read. + it("does not read a caller id handed to a response serializer as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return typedjson({ userId: user.id }); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The direction that matters more than any of the above: refusing the log line must not accuse a + // handler that also runs the query. This is the shape on the real tree today, + // `engine.v1.dev.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts`, which logs + // the environment id and reads with it, and which must keep its pass. + it("still reads a real query filter written beside a log call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + const run = await runStore.findRun({ runtimeEnvironmentId: authentication.environment.id }); + if (!run) logger.error("no run", { environmentId: authentication.environment.id }); + return json(run); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + // A resource's owner is not the caller. it("does not read another object's userId as a scope", () => { const ep = scanFile( diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 7cd8e5caaa5..c868fc142d1 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -96,21 +96,61 @@ const CALLER_ID_FIELD = /^(id|userId|user|memberId|orgMemberId|createdBy|createdByUserId|environmentId|runtimeEnvironmentId|organizationId|orgId|projectId)$/; /** - * Whether the object literal holding this property is handed to a call, through any depth of - * nesting: `findMany({ where: { members: { some: { userId } } } })` is, and + * Callees that are handed the caller's id and cannot narrow a read with it: the log line and the + * response body. Both take the very `{ userId: user.id }` object a query filter takes, so crediting + * them let one log statement clear `auth-scope` for a whole export. That is cheaper than the + * actor-argument residual `checks/authScope.ts` discloses, and it lands on the one check whose + * purpose is catching cross-org exposure. + * + * The shape is already in the tree rather than hypothetical: `engine.v1.dev.runs...attempts.start` + * writes `logger.error("...", { environmentId: authentication.environment.id })` beside the + * `runStore.findRun` that earns that export its credit honestly. Loggers account for 13 of the + * caller-id sites under `apps/webapp/app/routes` and the two response serializers for 2 more. + * + * A denylist of sinks rather than an allowlist of query callees, and that is a measurement rather + * than a preference. 72 distinct callees are handed a caller id across the route tree, running from + * `prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare + * `regenerateApiKey` and `resolveOrganizationForApiUser`. No name pattern separates those from + * `sendToPlain`, so an allowlist would accuse whichever route named its helper next, and a wrong + * accusation is the failure this check cannot afford. Refusing the sinks that are known not to + * scope shrinks the residual without pretending to close it: `someHelper({ userId: user.id })` that + * ignores its argument still credits, which needs types the scanner does not have. + */ +const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; + +/** + * Whether a callee could plausibly narrow a read with the object it is handed. A callee with no + * readable name of its own is credited: refusing it would ACCUSE the route, and under-crediting the + * constraint beats accusing a route that is fine. + */ +function couldScopeAQuery(callee: ts.Expression): boolean { + const text = calleeText(callee); + if (text === null) return true; + return !LOGGER_CALLEE.test(text) && !NON_SCOPING_CALLEE.test(text); +} + +/** + * Whether the object literal holding this property is handed to a call that could scope a query, + * through any depth of nesting: `findMany({ where: { members: { some: { userId } } } })` is, and * `const unused = { userId };` is not. Arrays count, so `{ OR: [{ userId }] }` still reaches its * call. * - * What this refuses is a filter built and dropped. What it does NOT refuse is a filter built and - * handed to a call that ignores it: `String({ userId: user.id });` reads as scoping, the same way - * `try { String(0); }` reads as error handling, and for the same reason. Knowing whether the callee - * uses the argument needs types the scanner does not have. + * Two things are refused. A filter built and dropped, which is the dead-object shape + * `dead-caller-scope-object` and `dead-caller-scope-userid` cover. And a filter handed to a callee + * that provably cannot read with it, which is `NON_SCOPING_CALLEE` and which the corpus entry + * `log-caller-scope-userid` covers at tree scale. + * + * What this does NOT refuse is a filter handed to a named call that ignores it: + * `String({ userId: user.id });` reads as scoping, the same way `try { String(0); }` reads as error + * handling, and for the same reason. Knowing whether an arbitrary callee uses the argument needs + * types the scanner does not have. */ -function isHandedToACall(property: ts.PropertyAssignment): boolean { +function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { let node: ts.Node = property; for (let parent = node.parent; parent; node = parent, parent = node.parent) { if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { - return parent.arguments?.some((a) => a === node) === true; + if (parent.arguments?.some((a) => a === node) !== true) return false; + return couldScopeAQuery(parent.expression); } if ( !ts.isObjectLiteralExpression(parent) && @@ -147,7 +187,9 @@ function isHandedToACall(property: ts.PropertyAssignment): boolean { * halves of that shape. * * So the property NAME has to be an identity field, and the object it sits in has to be handed to a - * call. See `isHandedToACall` for what that does and does not refuse. + * call that could scope a query. The third condition is what stops `logger.error("create failed", + * { userId: user.id })`, written anywhere in a builder-wrapped handler, clearing the check for that + * export. See `isHandedToAScopingCall` for what that does and does not refuse. */ function scopesByCallerIn(fns: Iterable): boolean { let found = false; @@ -159,7 +201,7 @@ function scopesByCallerIn(fns: Iterable): boolean { CALLER_ID_FIELD.test(node.name.text) ) { const path = propertyPath(node.initializer); - if (path !== null && CALLER_ID_PATH.test(path) && isHandedToACall(node)) { + if (path !== null && CALLER_ID_PATH.test(path) && isHandedToAScopingCall(node)) { found = true; return; } From 32c301db3a3bd76ff6afc89d5f4c628e04fc847b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:22:11 +0100 Subject: [PATCH 097/117] ci(observability-map): stop the nightly corpus depending on the paths filter `needs: changes` carries an implicit success() that outranks the event test in the `if`, so a failed or skipped filter job silently skipped the nightly mutation corpus and the tree-drift scan stopped without saying so. Adding a status-check function to the `if` drops the implicit success() and lets the event test decide alone. The filter job now runs only for pull requests, the sole path that reads its output. Pull request gating is unchanged. --- .github/workflows/observability-map.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 48508374231..cb4a82a4ab9 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -26,6 +26,10 @@ jobs: # evaluates it once per workflow. This narrows it again for the corpus job alone. changes: name: 🔍 Which paths moved + # Only the pull request path reads this job's output. On a schedule the action has no base to + # diff, warns that `before` is missing and reports the files in the last commit on main, which + # nothing then consults. Skipping it there keeps the nightly off a job it does not need. + if: github.event_name == 'pull_request' runs-on: warp-ubuntu-latest-x64-2x outputs: package: ${{ steps.filter.outputs.package }} @@ -63,7 +67,20 @@ jobs: mutation-corpus: name: 🧬 Mutation corpus needs: changes - if: github.event_name != 'pull_request' || needs.changes.outputs.package == 'true' + # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit + # success() on the job it names, and that implicit test outranks the `||` below: with a plain + # condition, a `changes` job that failed or was skipped skips this one, so the nightly would + # stop scanning for tree drift and report nothing about having stopped. A status-check function + # in the `if` is what drops the implicit success(), so the event test below decides alone. + # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a + # superseded run should not finish this job. + # + # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves + # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop + # for that, which is the same trade the paths gate already makes for routes-only pull requests. + if: >- + !cancelled() && + (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') runs-on: warp-ubuntu-latest-x64-4x steps: - name: ⬇️ Checkout repo From f06e723eab9fd4b6ae45482aee2e8e877853659d Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:22:33 +0100 Subject: [PATCH 098/117] chore(observability-map): declare the ES2020 lib the tests already need The tests call String.prototype.matchAll, an ES2020 library feature, while `lib` said ES2019. Typecheck passed anyway because @types/node v24 declares `/// `, so the program already contained es2020. No behaviour changes; the declaration now states the requirement instead of resting on a transitive reference. --- internal-packages/observability-map/tsconfig.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json index 61669556f76..ea5663ae806 100644 --- a/internal-packages/observability-map/tsconfig.json +++ b/internal-packages/observability-map/tsconfig.json @@ -1,7 +1,11 @@ { "compilerOptions": { "target": "ES2019", - "lib": ["ES2019"], + // ES2020 rather than the ES2019 most sibling packages use, because the tests call + // String.prototype.matchAll. It already resolved: @types/node carries a + // `/// `, so the program had es2020 whatever this line said. Stating + // it here stops the requirement resting on a transitive reference from a types package. + "lib": ["ES2020"], "module": "ESNext", "moduleResolution": "Bundler", "esModuleInterop": true, From 5ad60264f39e15fa19bd3dfa120c32774701c398 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:56:57 +0100 Subject: [PATCH 099/117] fix(observability-map): keep the test task out of the turbo cache The suite scans apps/webapp/app, packages/plugins/src, internal-packages/rbac/src and four files under .github/workflows, none of which turbo hashes for this package, so turbo run test replayed a pass recorded before those trees changed. Measured rather than argued: a route file with a syntax error fails the suite under vitest, and the same tree came back FULL TURBO in 301ms with the failure cached away as a success. inputs was tried and rejected rather than assumed unworkable. Turbo 1.x does accept .. in an input glob, and ../../apps/webapp/app/** did bust the cache on a route change, but it replaces the default file set instead of adding to it, so the same config silently dropped this package's own vitest.config.ts from the hash. The $TURBO_DEFAULT$ token that would add rather than replace is turbo 2.x only and matches nothing on 1.10.3. Costs about 23s per run and no CI job pays it: the dedicated workflow calls vitest without turbo, and unit-tests-internal.yml runs cold. Reported by Devin on #4455. --- .../observability-map/src/integration.test.ts | 31 +++++++++++++++++++ .../observability-map/turbo.json | 31 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 internal-packages/observability-map/turbo.json diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 03e8ca78b2c..5c673a6632b 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -213,6 +213,37 @@ describe("the package's tests are wired into the gate", () => { }); }); +/** + * The third road into this suite, after the two workflows above: `turbo run test`, which is what + * `pnpm run test` and `pnpm run test:internal` reach it by. + * + * Turbo keys a task's cache on the package's own files. This suite's real inputs are mostly not + * its own files, they are `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` + * and the workflow files read above, so turbo happily replayed a pass recorded before a route + * changed. Measured rather than argued: a route file with a syntax error in it makes + * `parses every route file and produces a report inside a wide band` fail under vitest, and the + * same tree came back FULL TURBO in 301ms with the failure cached away as a success. + * + * So the task is uncacheable, and this asserts that, because the config is one line and reads like + * a performance oversight to anyone who does not know what the suite reads. + * + * What this does not assert is the rejected alternative. `inputs` can name `../../apps/webapp/...` + * and does bust the cache, but it replaces turbo 1.x's default file set instead of adding to it, + * so it drops the package's own files from the hash unless every one of them is listed too; that + * was measured the same way, by editing `vitest.config.ts` and getting FULL TURBO back. The + * reasoning lives in `turbo.json` next to the config it explains. + */ +describe("the third road in, turbo", () => { + it("keeps its test task out of the turbo cache", () => { + const config = readFileSync(resolve(__dirname, "../turbo.json"), "utf8"); + // Comments are legal in turbo.json and this one carries the reasoning, so strip them to parse. + const pipeline = JSON.parse(config.replace(/^\s*\/\/.*$/gm, "")) as { + pipeline?: { test?: { cache?: boolean } }; + }; + expect(pipeline.pipeline?.test?.cache).toBe(false); + }); +}); + describe("counting candidates independently of the scanner", () => { // The counter is only worth having if it disagrees with the scanner somewhere. It does: the // scanner attributes nothing to a nested file that is not `route.ts`/`route.tsx`, and the diff --git a/internal-packages/observability-map/turbo.json b/internal-packages/observability-map/turbo.json new file mode 100644 index 00000000000..42fa065651d --- /dev/null +++ b/internal-packages/observability-map/turbo.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://turborepo.org/schema.json", + "extends": ["//"], + "pipeline": { + // Uncacheable on purpose. This suite's real inputs live outside the package: it scans + // `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and four files under + // `.github/workflows`. The root `test` task keys the cache on this package's own files, so a + // cached pass replayed after a route change broke the scan, which is a guard that stops + // guarding while still reading green. + // + // `inputs` was tried and rejected rather than assumed unworkable. Turbo 1.x does accept `..` + // in an input glob, and `../../apps/webapp/app/**` did bust the cache on a route change. What + // it also does is replace the default file set rather than add to it, so the same config + // silently dropped this package's own `vitest.config.ts` from the hash: editing it replayed a + // cached pass. The `$TURBO_DEFAULT$` token that adds rather than replaces is turbo 2.x only + // and matches nothing on the 1.10.3 here. Trading a stale-on-routes hole for a + // stale-on-own-config hole is not a fix, and an inputs list mirroring what the tests read is + // one more thing that drifts out of sync without saying so. + // + // Cost is about 23s per run, and no CI job pays it: the dedicated workflow calls vitest + // without turbo, and `unit-tests-internal.yml` runs cold. + // + // `it("keeps its test task out of the turbo cache")` in `src/integration.test.ts` fails if + // this is removed. + "test": { + "dependsOn": ["^build"], + "outputs": [], + "cache": false + } + } +} From 00b78cb63761d8c40440f6fc17817ca9eb437ba3 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 02:57:29 +0100 Subject: [PATCH 100/117] ci(observability-map): write scan reports with --out instead of capturing stdout Both scan steps redirected pnpm --filter ... exec stdout into files the renderer JSON.parses. pnpm takes its recursive path under --filter and some versions announce 'Scope: N of M workspace projects' on it; one such line in head.json fails the parse and degrades every run to the stale-report comment, which is a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 the workflow pins, which was checked, so this closes the class rather than a reproduction: the scanner writes its own file and stdout is left to be log output. The -s guard keeps the partial dance honest now the redirect no longer creates the file, so a scanner that exits 0 without writing takes the stale-report branch instead of failing the mv and turning the job red. The render step still captures stdout, since prCommentCli has no --out and a banner there puts a stray line in a markdown comment rather than breaking a parse. Reported by Devin on #4455. --- .github/workflows/observability-map.yml | 23 ++++++++++--- .../observability-map/src/integration.test.ts | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index cb4a82a4ab9..6043ce15976 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -149,10 +149,24 @@ jobs: # Guarded rather than allowed to fail: this job must never block a pull request. The failure # is not swallowed either, the render step below turns a missing head report into a comment # saying so, because a swallowed failure with no comment is the outcome nobody wants. + # + # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside + # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some + # versions announce `Scope: N of M workspace projects` on the way; that line landing in + # head.json would fail the parse and degrade every run to the stale-report comment, which is + # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 + # pinned above, so this closes the class rather than a reproduction: the file is written by + # the process that owns it and stdout is left to be log output. Held by + # `it("let the scanner write its own report rather than capturing stdout")` in + # `internal-packages/observability-map/src/integration.test.ts`. + # + # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner + # that exits 0 without writing takes the else branch and the stale-report comment, instead of + # failing the `mv` and turning the job red. - name: 🔎 Scan head run: | - if pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ - > /tmp/head.json.partial; then + if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then mv /tmp/head.json.partial /tmp/head.json else rm -f /tmp/head.json /tmp/head.json.partial @@ -168,8 +182,9 @@ jobs: - name: 🔎 Scan base with the head's scanner run: | if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ - && pnpm --filter @internal/observability-map exec tsx src/cli.ts --json --no-write \ - --routes=/tmp/base-tree/apps/webapp/app/routes > /tmp/base.json; then + && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \ + && [ -s /tmp/base.json ]; then : else echo "-" > /tmp/base.json || true diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 5c673a6632b..4be9d22ecc7 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -117,6 +117,40 @@ describe("the report workflow's two readers of the comment lookup", () => { }); }); +/** + * Both scan steps used to capture the scanner's stdout with a shell redirect, into files the + * renderer then `JSON.parse`s. Anything else reaching stdout therefore corrupted the report: + * `pnpm --filter` takes its recursive path, and some versions of pnpm announce + * `Scope: N of M workspace projects` on it. A single line of that in head.json fails the parse, + * and the workflow degrades to the stale-report comment on every run, quietly and permanently. + * + * It does not reproduce on the 10.33.2 the workflow pins, which was checked. What is asserted here + * is the shape that cannot have the bug at all rather than the version that happens not to: the + * scanner writes its own file through `--out`, so stdout carries log output and nothing else. + * The same is not yet true of the render step, which has no `--out` to reach for; a banner there + * puts a stray line in a markdown comment instead of breaking a parse, so it is left alone. + */ +describe("the report workflow's two scan steps", () => { + const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); + + it("let the scanner write its own report rather than capturing stdout", () => { + const scans = readFileSync(WORKFLOW, "utf8") + .split(/^ {6}- name: /m) + .slice(1) + .filter((step) => step.startsWith("🔎 Scan")); + expect(scans).toHaveLength(2); + + for (const step of scans) { + // The `if ...; then` condition only, which is where the scanner runs. The else branch writes + // `echo "-" > /tmp/base.json`, a redirect of the workflow's own making that has nothing to + // do with capturing the scanner, and an earlier version of this test failed on it. + const command = step.split("; then")[0]!; + expect(command).toMatch(/--out=\S+\.json/); + expect(command).not.toMatch(/>\s*\S*\.json/); + } + }); +}); + /** * The gating half of the same problem. A test job that nothing waits for is decoration, and the * first attempt at this was exactly that: a job inside `observability-map.yml`, which reads well From 26c405967e01dd234530281ef8ba4e10041affee Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 03:31:34 +0100 Subject: [PATCH 101/117] ci(observability-map): widen the obsmap filter to what the suite reads The filter watched apps/webapp/app/routes only, which was narrower than the suite's actual coupling. webappSymbols.test.ts walks all of apps/webapp/app and fails when a guard, sensitive or audit symbol stops resolving, so renaming e.g. requireUserId in app/services/session.server.ts matched the webapp filter and nothing else: no job ran this suite and the break landed on main, or on the next unrelated internal-packages PR. integration.test.ts also asserts on the text of observability-map.yml, which no filter watched at all, so editing the report workflow alone ran nothing. Derived the coupling set from the code rather than from the comment. Outside its own directory the suite reads apps/webapp/app (whole tree for symbols, the route subtree for the scan), packages/plugins/src, internal-packages/rbac/src, and four workflow files. packages/plugins/src and internal-packages/rbac/src stay out: internal already matches packages/** and internal-packages/**, and unit-tests-internal.yml runs the same suite, so listing them here would run it twice. A new test pins that reasoning. Cost, over the last 400 commits on main: 31% touch routes, 52% touch apps/webapp/app, so the job fires on roughly half of PRs instead of roughly a third. It is the cheap one, a single 4x runner with no containers and no database. Reported by Devin on #4455. --- .github/workflows/pr_checks.yml | 33 +++++++++---- .../unit-tests-observability-map.yml | 5 +- .../observability-map/src/integration.test.ts | 49 +++++++++++++++---- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index ddedf3559f2..e3df9b416f2 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -82,20 +82,33 @@ jobs: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - 'turbo.json' - # Routes only, and that is the whole reason this filter exists: one test in - # @internal/observability-map scans the live route tree, and without this a webapp-only - # PR that renames a route merges green and the break lands on main, or on the next - # unrelated internal-packages PR. + # The whole webapp app tree, not just its routes, and that is the whole reason this + # filter exists. Two tests in @internal/observability-map read it: integration.test.ts + # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and + # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this + # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts + # matched `webapp` and nothing else, so no job ran the suite and the break landed on + # main, or on the next unrelated internal-packages PR. # - # The package's own paths are deliberately NOT here. `internal` above already matches - # `internal-packages/**`, and `unit-tests-internal.yml` runs `turbo run test --filter - # "@internal/*"`, which picks up @internal/observability-map and runs the same vitest - # suite including that route-tree test. Listing the package here as well ran the suite - # twice on every PR touching it, which was this filter's own doing. + # The cost of the wider set, measured over the last 400 commits on main: 31% touch + # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of + # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no + # database, no prisma generate -- which is what makes that affordable. + # + # observability-map.yml is here because integration.test.ts asserts on its text and no + # other filter watches it, so editing the report workflow alone ran nothing at all. + # + # Deliberately NOT here: this package's own paths, and packages/plugins/src and + # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads. + # `internal` above already matches `internal-packages/**` and `packages/**`, and + # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up + # @internal/observability-map and runs the same vitest suite. Listing them here as well + # ran the suite twice on every PR touching them, which was this filter's own doing. obsmap: - - 'apps/webapp/app/routes/**' + - 'apps/webapp/app/**' - '.github/workflows/pr_checks.yml' - '.github/workflows/unit-tests-observability-map.yml' + - '.github/workflows/observability-map.yml' - 'package.json' - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' diff --git a/.github/workflows/unit-tests-observability-map.yml b/.github/workflows/unit-tests-observability-map.yml index d71aecf985e..eebf1ef971c 100644 --- a/.github/workflows/unit-tests-observability-map.yml +++ b/.github/workflows/unit-tests-observability-map.yml @@ -36,7 +36,8 @@ jobs: - name: 📥 Download deps run: pnpm install --frozen-lockfile - # One test in this suite scans apps/webapp/app/routes, which is why the filter that gates - # this workflow watches those paths as well as the package's own. + # This suite reads apps/webapp/app (the route tree for the scan, the whole app tree for the + # symbol check) and the report workflow's text, which is why the filter that gates this + # workflow watches all of those and not only the routes folder. - name: 🧪 Run tests run: pnpm --filter @internal/observability-map run test diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 4be9d22ecc7..8351dc333dc 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -14,8 +14,12 @@ import { buildReport } from "./score.js"; import { SCORED_CHECK_IDS } from "./checks/index.js"; /** - * The one deliberate coupling to `apps/webapp/app/routes` in the suite. Everything else, including - * the CLI tests, runs against a fixture tree of this package's own making. + * This file's deliberate coupling to `apps/webapp/app/routes`. It is not the suite's only one, and + * saying it was is what let the paths filter be written for this file alone: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, `packages/plugins/src` and + * `internal-packages/rbac/src`, and `mutationCorpus.test.ts` scans the route tree behind an env + * gate. Everything else, including the CLI tests, runs against a fixture tree of this package's own + * making. * * The coupling is acceptable because nothing here names a route or a count: the scan must not * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those @@ -23,12 +27,16 @@ import { SCORED_CHECK_IDS } from "./checks/index.js"; * cannot tell us, since a fixture only contains shapes somebody thought to write down. * * What runs this for a webapp pull request is `.github/workflows/unit-tests-observability-map.yml`, - * called from `pr_checks.yml` behind an `obsmap` paths filter covering `apps/webapp/app/routes/**`, - * and listed in the `all-checks` aggregate so it actually gates. A pull request touching this - * PACKAGE reaches the same test by the other road: `internal` already matches - * `internal-packages/**`, and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"` - * over this package too. So both directions are gated, and neither is gated twice; the `obsmap` - * filter used to name the package as well, which ran this suite twice on every PR touching it. + * called from `pr_checks.yml` behind an `obsmap` paths filter covering the whole of + * `apps/webapp/app` plus the report workflow, and listed in the `all-checks` aggregate so it + * actually gates. The filter is wider than this file's own coupling because the suite's is: + * `webappSymbols.test.ts` walks all of `apps/webapp/app`, and the describes below read + * `observability-map.yml`. A pull request touching this PACKAGE, or `packages/plugins/src` or + * `internal-packages/rbac/src`, reaches the same test by the other road: `internal` already matches + * `internal-packages/**` and `packages/**`, and `unit-tests-internal.yml` runs `turbo run test + * --filter "@internal/*"` over this package too. So every direction is gated and none is gated + * twice; the `obsmap` filter used to name the package as well, which ran this suite twice on every + * PR touching it. * * Two shapes were tried and rejected on the way here. Widening `pr_checks.yml`'s `internal` filter * to the route paths ran all eighteen internal packages, twelve shards with postgres, clickhouse, @@ -179,9 +187,30 @@ describe("the package's tests are wired into the gate", () => { expect(read(REUSABLE)).toContain("workflow_call"); }); - it("watches the live route tree, which is the only thing the internal filter misses", () => { + // Round 5. The filter watched `apps/webapp/app/routes/**` while the suite reads more than that, + // so a rename outside the routes folder matched only `webapp`, ran no job that runs this suite, + // and broke the build for whoever pushed next. Asserted as the whole set the suite reads and no + // other filter covers, rather than as the one path that prompted the filter, because the routes + // entry looked complete right up until it wasn't. + it("watches every webapp path the internal filter misses, not just the routes folder", () => { const filter = read(PR_CHECKS).split(" obsmap:")[1]!.split(" cli:")[0]!; - expect(filter).toContain("'apps/webapp/app/routes/**'"); + // webappSymbols.test.ts walks all of apps/webapp/app, not just routes. + expect(filter).toContain("'apps/webapp/app/**'"); + // The report workflow, whose text the two describes above assert on. No other filter names it. + expect(filter).toContain("'.github/workflows/observability-map.yml'"); + }); + + // The other two trees webappSymbols.test.ts reads. They belong to `internal`, not here, and this + // pins the reason so the obvious-looking addition has to argue with a test first. + it("leaves the two non-webapp roots it reads to the internal filter", () => { + const text = read(PR_CHECKS); + const obsmap = text.split(" obsmap:")[1]!.split(" cli:")[0]!; + expect(obsmap).not.toContain("packages/plugins"); + expect(obsmap).not.toContain("internal-packages/rbac"); + + const internal = text.split(" internal:")[1]!.split(" # ")[0]!; + expect(internal).toContain("'packages/**'"); + expect(internal).toContain("'internal-packages/**'"); }); // Round E item 6. `internal` matches `internal-packages/**` and `unit-tests-internal.yml` runs From 4ea20b03f1f20362eb5347b75b33ad1a6b5eb842 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 03:32:28 +0100 Subject: [PATCH 102/117] test(observability-map): size the real-tree timeouts for shard contention The 30s and 60s per-test timeouts on the two real-tree tests were chosen on an idle machine, and the suite also runs inside unit-tests-internal.yml, which executes turbo run test --filter "@internal/*" as twelve concurrent shard processes on one runner. The 30s one does flake under that. Measured on an 8-core box. This file alone at load average 0.9: 6.3-6.4s for the scan, 10.8-11.2s for the sweep, both well above the 1.6-2.6s the old comment claimed. Two batches of twelve concurrent copies on those same 8 cores: 24.2-34.0s for the scan and 27.6-39.7s for the sweep, with one of the first twelve dying on "Test timed out in 30000ms". Twelve processes over 8 cores is 1.5 per core where the 32-vCPU runner is 0.375, so the reproduction is harsher than CI, which is why it is the thing to size against. Both now use one 120s constant, which is 3x the worst contended run measured. 60s was the other candidate and is not enough: the sweep already reached 39.7s. Neither test asserts anything about elapsed time, so the number is a hang detector rather than a performance budget, and the docstring says so. Reported by Devin on #4455. --- .../observability-map/src/integration.test.ts | 91 ++++++++++++------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 8351dc333dc..593ff311d69 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -326,41 +326,70 @@ describe("counting candidates independently of the scanner", () => { }); }); +/** + * Timeout for the two real-tree tests, which do not fit the suite's 10s default: the first runs a + * ts.Program per route file for the parse diagnostics and walks the tree a second time to count + * candidates, the second scans the tree twice and re-scans every source with the suppression + * directive prepended. + * + * It is a hang detector and nothing else. Neither test asserts anything about how long the scan + * takes, so a number tight enough to be a performance budget would only be a way to fail on a busy + * runner, and a performance budget that flakes gets the whole suite marked unreliable. + * + * The old 30s and 60s were chosen on an idle machine and the 30s one does flake. Measured on an + * 8-core box, this file alone at load average 0.9: 6.3-6.4s for the scan, 10.8-11.2s for the sweep. + * Twenty-four runs of it as two batches of twelve concurrent copies on those same 8 cores: + * 24.2-34.0s for the scan and 27.6-39.7s for the sweep, with one of the first twelve dying on + * "Test timed out in 30000ms". That contention is not hypothetical: + * `.github/workflows/unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"` as + * twelve concurrent shard processes on one runner, and this file executes inside one of them. + * + * The local reproduction is deliberately harsher than CI, which is why it is the thing to size + * against: twelve processes over 8 cores is 1.5 per core where the 32-vCPU runner is 0.375. 120s is + * 3x the worst contended run measured and about 11x the idle sweep. 60s was the other candidate and + * is not enough on those numbers: the sweep already reached 39.7s, which is 1.5x, and a margin that + * thin on a machine nobody controls is how the 30s got here. + */ +const TREE_SCAN_TIMEOUT = 120_000; + describe("scanning the real webapp routes", () => { - it("parses every route file and produces a report inside a wide band", () => { - const { entryPoints, parseFailures } = scanDirectory(ROUTES); - - expect(parseFailures).toEqual([]); - expect(entryPoints.length).toBeGreaterThan(100); - expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); - - const report = buildReport(entryPoints, parseFailures); - expect(report.global).toBeGreaterThanOrEqual(0); - expect(report.global).toBeLessThanOrEqual(100); - expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); - // A full scan of the real tree runs a `ts.Program` per file for the parse diagnostics, and - // `countRouteModuleFiles` walks the tree a second time. That is 1.6 to 2.6 seconds on an idle - // machine and it flaked past the suite's 10s default under parallel load. Budgeted rather than - // left marginal, the same way the exhaustive sweep below is. - }, 30_000); + it( + "parses every route file and produces a report inside a wide band", + () => { + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + + expect(parseFailures).toEqual([]); + expect(entryPoints.length).toBeGreaterThan(100); + expect(entryPoints.length).toBeLessThan(countRouteModuleFiles(ROUTES)); + + const report = buildReport(entryPoints, parseFailures); + expect(report.global).toBeGreaterThanOrEqual(0); + expect(report.global).toBeLessThanOrEqual(100); + expect(Object.keys(report.byFamily).length).toBeGreaterThan(1); + }, + TREE_SCAN_TIMEOUT + ); // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to // 176, because every entry whose only applicable checks were suppressed dropped out of the // mean. Measured must not move: every entry point that had something applicable still does. - it("suppressing every scored check on every real route does not raise the global", () => { - const { entryPoints, parseFailures } = scanDirectory(ROUTES); - const before = buildReport(entryPoints, parseFailures); - - const directive = SCORED_CHECK_IDS.map( - (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` - ).join(""); - const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); - const after = buildReport(suppressed, parseFailures); - - expect(after.measured).toBe(before.measured); - expect(after.unmeasured).toBe(before.unmeasured); - expect(after.global).not.toBeGreaterThan(before.global!); - // Two full tree scans plus a re-scan of every source, which does not fit the suite default. - }, 60_000); + it( + "suppressing every scored check on every real route does not raise the global", + () => { + const { entryPoints, parseFailures } = scanDirectory(ROUTES); + const before = buildReport(entryPoints, parseFailures); + + const directive = SCORED_CHECK_IDS.map( + (id) => `// obs-map-disable ${id} -- exhaustive sweep\n` + ).join(""); + const suppressed = entryPoints.map((ep) => scanFile(ep.fileName, directive + ep.source)!); + const after = buildReport(suppressed, parseFailures); + + expect(after.measured).toBe(before.measured); + expect(after.unmeasured).toBe(before.unmeasured); + expect(after.global).not.toBeGreaterThan(before.global!); + }, + TREE_SCAN_TIMEOUT + ); }); From 30fdb02384f38f2550f29a9ece028395b41d0e37 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:45:46 +0100 Subject: [PATCH 103/117] fix(observability-map): attribute auth-boundary guards per export Every input auth-boundary read was entry-point-wide, so one guarded export spoke for the whole file: calleeNames is the union of both bodies, checkedCallees was too, and usesBuilder was an OR over both initializer callees. A file whose loader called requireUser and whose action called nothing read as guarded in the body, and a createLoaderApiRoute loader authenticated a hand-written action beside it. This is the same defect auth-scope was fixed for a round earlier, in its sibling check. scanFile now splits calleeNames, calleeTexts, checkedCallees, statementCount and hasTryCatch per export, filled from one push site each so the union and the split cannot drift apart. usesBuilder had no other caller and is gone. routeExports is the single enumeration of a file's exports, shared with auth-scope, which had grown its own [loader, action] literal. Triviality had to follow, or the fix trades a false pass for a false accusation: naive per-export attribution moved auth.github.ts and auth.google.ts to fail, both being a one-line redirect-stub loader beside a guarded action that the entry-point-wide rule called non-trivial. isTrivial is now one rule over two views. The per-export view matches the side-effect hints against that export's own callee paths: the whole file is defeatable (the corpus's log-caller-scope-userid puts the word logger in the file and un-excuses the untouched loader) and nothing at all guts the check (five fixtures go from fail to not-applicable, because calleeNames keeps only a call's last segment and prisma.x.findMany reads as findMany). login.mfa's action verifies a TOTP or recovery code, which is a login-surface proof of possession like the verify* guards already listed, so it joins them rather than being accused once its loader stops speaking for it. Real tree unmoved: global 19, 62 auth-boundary applicable, 59 passing, no route changing any check. scan.ts also picks up routeModuleFiles here, shared with the corpus harness, because it sits in the same hunk as the per-export return shape. --- .../src/checks/authBoundary.ts | 91 ++++++-- .../observability-map/src/checks/authScope.ts | 37 ++-- .../src/checks/errorClassification.ts | 7 - .../src/checks/index.test.ts | 193 ++++++++++++++++ .../observability-map/src/integration.test.ts | 35 ++- .../observability-map/src/routeExports.ts | 71 ++++++ .../observability-map/src/scan.test.ts | 104 ++++++++- .../observability-map/src/scan.ts | 207 +++++++++++++----- .../observability-map/src/triviality.ts | 98 +++++++-- .../observability-map/src/types.ts | 55 ++++- 10 files changed, 766 insertions(+), 132 deletions(-) create mode 100644 internal-packages/observability-map/src/routeExports.ts diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 13327f582fb..4abbeb7dc73 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -1,14 +1,16 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; -import { isTrivial } from "../triviality.js"; -import { usesBuilder } from "./errorClassification.js"; +import { routeExports, type ExportName, type RouteExport } from "../routeExports.js"; +import { isTrivialExport } from "../triviality.js"; +import { BUILDERS } from "./errorClassification.js"; const ID = "auth-boundary"; /** - * The guard helpers this webapp actually has, matched against `calleeNames`, which is scoped to the - * loader/action bodies and follows one hop into a same-file helper. A guard the route only imports - * and never calls does not count. + * The guard helpers this webapp actually has, matched against the calling export's own + * `loaderCalleeNames`/`actionCalleeNames`, each scoped to that export's handlers and following one + * hop into a same-file helper. A guard the route only imports and never calls does not count, and + * neither does one the OTHER export calls. * * A name list rather than the three patterns it replaces, because all three over-matched and this * is the one check where a false pass hides a security gap: @@ -66,16 +68,22 @@ export const GUARDS = new Set([ // Local helpers, each declared inside the one route that uses it. "authenticateAdmin", "authenticatePlainRequest", - // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC. + // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC, + // and a login-surface second factor is authenticated by checking the code presented. + // `login.mfa`'s action is the second half of a login, so like `authenticate` above it establishes + // identity from the credential rather than requiring an already authenticated caller. It reached + // this list when per-export attribution stopped its loader's `isAuthenticated` speaking for it. "verifyHttpCallbackHash", "verifyWebhook", "verifyUserActorToken", + "verifyTotpForLogin", + "verifyRecoveryCodeForLogin", ]); /** * Guards that answer with null instead of throwing. Calling one is not evidence of a boundary, - * because the route is free to ignore the answer, so these are only credited when the body - * demonstrably reads what they returned (`EntryPoint.checkedCallees`). + * because the route is free to ignore the answer, so these are only credited when THAT EXPORT's + * handlers demonstrably read what they returned (`EntryPoint.loaderCheckedCallees`). * * The distinction is the whole reason this set is separate from `GUARDS`. `requireUserId` redirects * on its own, so calling it IS the boundary; `getUserId` hands back `string | null` and a route @@ -85,10 +93,43 @@ export const GUARDS = new Set([ * rather than something a hand-read established once. * * What it still cannot see is whether the test that reads the answer guards anything. See - * `EntryPoint.checkedCallees` for the exact shape of that residual. + * `EntryPoint.loaderCheckedCallees` for the exact shape of that residual. */ export const SOFT_GUARDS = new Set(["getUser", "getUserId"]); +type GuardedExport = { name: ExportName; guarded: boolean; how: string; export: RouteExport }; + +/** + * The exports this file declares, each with its own verdict. + * + * Per export, because the exposure is per export, and this is the same defect `auth-scope` was + * fixed for one round earlier. Every input here was entry-point-wide: `calleeNames` is the union of + * both bodies, `checkedCallees` was too, and `usesBuilder` was an OR over the two initializer + * callees. So a file whose loader called `requireUser` and whose action called nothing read as + * "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` credited its + * hand-written action with the builder's authentication. Three inputs, one bug, and it is a false + * PASS on the one check where that hides a security gap. + * + * `routeExports` lists only the exports the file actually declares, so an export that calls nothing + * at all is judged rather than skipped: an empty body is exactly the unguarded case. It is shared + * with `auth-scope`, which grew its own copy of the same `[loader, action]` literal. + */ +function guardedExports(ep: EntryPoint): GuardedExport[] { + return routeExports(ep).map((e) => { + const verdict = (guarded: boolean, how: string) => ({ name: e.name, guarded, how, export: e }); + if (e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) { + return verdict(true, "authenticated by the builder"); + } + if (e.calleeNames.some((n) => GUARDS.has(n))) { + return verdict(true, "guarded in the body"); + } + if (e.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { + return verdict(true, "resolves the caller and reads the answer"); + } + return verdict(false, ""); + }); +} + /** * Whether a route that handles credentials, tokens or money checks who is asking. * @@ -123,26 +164,30 @@ export const authBoundary = { if (!sensitivity.sensitive) { return { id: ID, status: "not-applicable", detail: "not sensitive" }; } - if (usesBuilder(ep)) { - return { id: ID, status: "pass", detail: "authenticated by the builder" }; - } - if (ep.calleeNames.some((n) => GUARDS.has(n))) { - return { id: ID, status: "pass", detail: "guarded in the body" }; - } - if (ep.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { - return { id: ID, status: "pass", detail: "resolves the caller and reads the answer" }; - } - if (isTrivial(ep)) { + // Never empty: `scanFile` returns null unless the file declares a loader or an action. + const exports = guardedExports(ep); + const guarded = exports.filter((e) => e.guarded); + // Triviality excuses per export, matching the attribution: the reasoning below is about one + // body being the place a guard would have to be, and reading it entry-point-wide let a busy + // action make a redirect-stub loader answerable for a guard it has nothing to guard. + const accused = exports.filter((e) => !e.guarded && !isTrivialExport(e.export)); + if (accused.length > 0) { return { id: ID, - status: "not-applicable", - detail: "cannot verify: no privileged work in the body, any guard is behind an import", + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body: ${accused + .map((e) => e.name) + .join(", ")}`, }; } + if (guarded.length > 0) { + const how = [...new Set(guarded.map((e) => e.how))].join(" and "); + return { id: ID, status: "pass", detail: how }; + } return { id: ID, - status: "fail", - detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body`, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", }; }, }; diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts index 2aa4a74dc6d..832928fa620 100644 --- a/internal-packages/observability-map/src/checks/authScope.ts +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -1,5 +1,6 @@ import type { CheckResult, EntryPoint } from "../types.js"; import { classifySensitivity } from "../sensitivity.js"; +import { routeExports } from "../routeExports.js"; import { BUILDERS } from "./errorClassification.js"; const ID = "auth-scope"; @@ -12,30 +13,22 @@ type BuilderExport = { name: string; callee: string; scoped: boolean; why: strin * Per export, because the exposure is per export. `authorization` is declared on the builder call * one export made, and a caller filter is written in the handler one export runs, so neither says * anything about the other half of the file. + * + * The enumeration itself is `routeExports`, shared with `auth-boundary`, which had to be given the + * same per-export treatment a round later and wrote a second copy of this literal to get it. */ function builderExports(ep: EntryPoint): BuilderExport[] { - const all = [ - { - name: "loader", - callee: ep.loaderInitializerCallee, - authorization: ep.loaderBuilderOptions.includes("authorization"), - filters: ep.loaderScopesByCaller, - }, - { - name: "action", - callee: ep.actionInitializerCallee, - authorization: ep.actionBuilderOptions.includes("authorization"), - filters: ep.actionScopesByCaller, - }, - ]; - return all - .filter((e) => e.callee !== null && BUILDERS.has(e.callee)) - .map((e) => ({ - name: e.name, - callee: e.callee!, - scoped: e.authorization || e.filters, - why: e.authorization ? "an authorization gate" : "a filter on the caller's identity", - })); + return routeExports(ep) + .filter((e) => e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) + .map((e) => { + const authorization = e.builderOptions.includes("authorization"); + return { + name: e.name, + callee: e.initializerCallee!, + scoped: authorization || e.scopesByCaller, + why: authorization ? "an authorization gate" : "a filter on the caller's identity", + }; + }); } /** diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 91d86c15088..34cfce33d34 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -118,13 +118,6 @@ function swallows(clause: CatchEvidence): boolean { return !decides(clause) && !inert(clause); } -export function usesBuilder(ep: EntryPoint): boolean { - return ( - (ep.loaderInitializerCallee !== null && BUILDERS.has(ep.loaderInitializerCallee)) || - (ep.actionInitializerCallee !== null && BUILDERS.has(ep.actionInitializerCallee)) - ); -} - /** * Who decides what a failure means, and on what evidence. * diff --git a/internal-packages/observability-map/src/checks/index.test.ts b/internal-packages/observability-map/src/checks/index.test.ts index 84b330b180f..f0e80af528d 100644 --- a/internal-packages/observability-map/src/checks/index.test.ts +++ b/internal-packages/observability-map/src/checks/index.test.ts @@ -1833,6 +1833,199 @@ describe("auth-boundary: the guard accept-list", () => { }); }); +/** + * Per-export attribution. Every input `auth-boundary` reads used to be entry-point-wide, so one + * guarded export spoke for the whole file. Each `it` here goes green on the entry-point-wide + * version of exactly one of those inputs, which is why they are separate cases rather than one. + */ +describe("auth-boundary: a guard credits only the export that calls it", () => { + const TOKENS = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server";`; + + it("fails an unguarded action beside a loader that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("fails an unguarded loader beside an action that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ params }) { + const tokens = await prisma.token.findMany({ where: { orgId: params.orgId } }); + return json({ tokens }); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); + + // The soft-guard arm reads its own export's checked-callee list for the same reason. + it("fails an unguarded action beside a loader that reads what getUserId returned", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (!userId) return redirect("/login"); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + // The builder arm. `usesBuilder` was an OR over both initializer callees, so a builder on one + // export authenticated a hand-written handler on the other. + it("fails a hand-written action beside a builder-wrapped loader", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async ({ authentication }) => { + return json(await prisma.token.findMany({ where: { userId: authentication.userId } })); + }); + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("passes when both exports call a guard of their own", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // One handler serving both exports guards both, which is the dominant API-route shape. + it("passes a shared builder handler that both exports resolve to", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json({ userId: authentication.userId }); + });` + ); + expect(r.status).toBe("pass"); + }); + + /** + * The damper on the attribution, and the reason it is not simply "accuse every unguarded export". + * `auth.github.ts` and `auth.google.ts` are this shape: per export the loader is unguarded, and + * an entry-point-wide triviality rule calls the file non-trivial because the ACTION is not. Both + * routes went pass to fail on the real tree until `isTrivialExport` existed. + */ + it("reports not-applicable for a redirect-stub loader beside a guarded action", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + const url = new URL(request.url); + const safeRedirect = sanitizeRedirectPath(url.searchParams.get("redirectTo"), "/"); + return await authenticator.authenticate("github", request, { + successRedirect: safeRedirect, + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("guarded in the body"); + }); + + /** + * The per-export excuse must read the export's own body and not the file's text. It read + * `ep.source` first, and `log-caller-scope-userid` in the mutation corpus, which prepends + * `logger.error(...)` to every body, put the word `logger` in this file and turned the untouched + * loader from excused into accused. A five-minute corpus run is the wrong place to catch that. + */ + it("does not un-excuse a redirect-stub loader because the file mentions a logger", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { logger } from "~/services/logger.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + logger.error("obs-map", { userId: request.userId }); + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + }); + + it("fails an export whose own body does real work unguarded", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { prisma } from "~/db.server"; + export let loader = async ({ params }) => { + const org = await prisma.organization.findFirst({ where: { slug: params.slug } }); + const members = await prisma.orgMember.findMany({ where: { orgId: org.id } }); + return json({ org, members }); + }; + export let action = async ({ request }) => { + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); +}); + // C1b. A builder authenticates the request; it does not necessarily scope it. `authorization` is // optional on every one of them and the RBAC gate only runs when it is declared. describe("auth-scope", () => { diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 593ff311d69..99b6f5ed851 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -9,7 +9,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { scanDirectory, scanFile } from "./scan.js"; +import { isScannableFile, scanDirectory, scanFile } from "./scan.js"; import { buildReport } from "./score.js"; import { SCORED_CHECK_IDS } from "./checks/index.js"; @@ -59,7 +59,7 @@ function countRouteModuleFiles(dir: string): number { count += countRouteModuleFiles(join(dir, entry.name)); continue; } - if (entry.isFile() && /\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) count++; + if (entry.isFile() && isScannableFile(entry.name)) count++; } return count; } @@ -370,6 +370,37 @@ describe("scanning the real webapp routes", () => { TREE_SCAN_TIMEOUT ); + /** + * The per-export split against the entry-point-wide union it came from, on every real route. + * `scan.test.ts` pins the same property on fixtures; this is the version that sees the shapes + * nobody thought to write down, and the two representations only stay honest while both hold. + */ + it( + "every callee name is attributed to an export that exists", + () => { + const { entryPoints } = scanDirectory(ROUTES); + const orphaned: string[] = []; + const unattributed: string[] = []; + + for (const ep of entryPoints) { + const attributed = new Set([...ep.loaderCalleeNames, ...ep.actionCalleeNames]); + for (const name of new Set(ep.calleeNames)) { + if (!attributed.has(name)) unattributed.push(`${ep.fileName}: ${name}`); + } + const union = new Set(ep.calleeNames); + for (const name of attributed) { + if (!union.has(name)) orphaned.push(`${ep.fileName}: ${name}`); + } + if (!ep.hasLoader) expect(ep.loaderCalleeNames).toEqual([]); + if (!ep.hasAction) expect(ep.actionCalleeNames).toEqual([]); + } + + expect(unattributed).toEqual([]); + expect(orphaned).toEqual([]); + }, + TREE_SCAN_TIMEOUT + ); + // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to // 176, because every entry whose only applicable checks were suppressed dropped out of the diff --git a/internal-packages/observability-map/src/routeExports.ts b/internal-packages/observability-map/src/routeExports.ts new file mode 100644 index 00000000000..a53b2ccf979 --- /dev/null +++ b/internal-packages/observability-map/src/routeExports.ts @@ -0,0 +1,71 @@ +import type { EntryPoint } from "./types.js"; + +export type ExportName = "loader" | "action"; + +/** + * One export of a route file, carrying that export's own evidence and nothing from the other one. + * + * Every field here has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the + * mistake this type exists to make hard. `calleeNames` is the union of both bodies, `hasTryCatch` + * is true if either has one, and `statementCount` counts both. + */ +export type RouteExport = { + name: ExportName; + /** Callee of the initializer call this export is assigned from, if any. */ + initializerCallee: string | null; + /** Top-level keys of the object literal passed to that call. */ + builderOptions: string[]; + /** Callees inside this export's handlers, and inside same-file helpers they call. */ + calleeNames: string[]; + /** The same calls as whole dotted paths, so `prisma.thing.findFirst` keeps its receiver. */ + calleeTexts: string[]; + /** Callees whose answer those handlers demonstrably read. */ + checkedCallees: string[]; + /** Whether those handlers narrow a query by the caller's own id. */ + scopesByCaller: boolean; + statementCount: number; + hasTryCatch: boolean; +}; + +/** + * The exports this route file declares, in `loader`, `action` order. + * + * One enumeration for the whole package, because two checks asking the same per-export question + * each grew their own. `auth-scope`'s `builderExports` and `auth-boundary`'s `guardedExports` were + * hand-maintained `[loader, action]` literals in adjacent files, reading the same six + * `loaderX`/`actionX` field pairs, with different tests for whether an export was there at all: + * one used `hasLoader`/`hasAction` and the other inferred it from a non-null initializer callee. + * Adding a seventh per-export fact meant editing both, and this whole branch is a record of what + * happens when a rule lives in two places and only one gets the fix. + * + * Absent exports are not returned, so a caller never has to remember to filter them: the shape of + * the bug in `auth-boundary` was crediting an export for something the other one did, and a list + * that only contains real exports is one fewer way to write it. + */ +export function routeExports(ep: EntryPoint): RouteExport[] { + const all: RouteExport[] = [ + { + name: "loader", + initializerCallee: ep.loaderInitializerCallee, + builderOptions: ep.loaderBuilderOptions, + calleeNames: ep.loaderCalleeNames, + calleeTexts: ep.loaderCalleeTexts, + checkedCallees: ep.loaderCheckedCallees, + scopesByCaller: ep.loaderScopesByCaller, + statementCount: ep.loaderStatementCount, + hasTryCatch: ep.loaderHasTryCatch, + }, + { + name: "action", + initializerCallee: ep.actionInitializerCallee, + builderOptions: ep.actionBuilderOptions, + calleeNames: ep.actionCalleeNames, + calleeTexts: ep.actionCalleeTexts, + checkedCallees: ep.actionCheckedCallees, + scopesByCaller: ep.actionScopesByCaller, + statementCount: ep.actionStatementCount, + hasTryCatch: ep.actionHasTryCatch, + }, + ]; + return all.filter((e) => (e.name === "loader" ? ep.hasLoader : ep.hasAction)); +} diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 70efae6eb33..32f9d224d7e 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -2795,6 +2795,96 @@ describe("scanFile: the signals auth-scope reads", () => { }); }); +/** + * The per-export split of `calleeNames`. `auth-boundary` reads it, so a name landing in the wrong + * half is a wrong verdict on the one check where a false pass hides a security gap. + */ +describe("scanFile: callee names attributed per export", () => { + it("keeps each export's callees out of the other's list", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + await deleteEverything(request); + return json({ ok: true }); + }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.loaderCalleeNames).not.toContain("deleteEverything"); + expect(ep!.actionCalleeNames).toContain("deleteEverything"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a same-file helper to the export that calls it", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); } + export async function action({ request }) { return json(await request.json()); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a helper both exports call to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function guarded(request) { return requireUserId(request); } + export async function loader({ request }) { return json(await guarded(request)); } + export async function action({ request }) { return json(await guarded(request)); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).toContain("requireUserId"); + }); + + // One handler, both exports: `const { loader, action } = createActionApiRoute({ handler })`. + it("attributes a handler serving both exports to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json(await authenticateApiRequest(authentication)); + });` + ); + expect(ep!.loaderCalleeNames).toContain("authenticateApiRequest"); + expect(ep!.actionCalleeNames).toContain("authenticateApiRequest"); + }); + + // The union the split came from. `calleeNames` stays entry-point-wide for `sensitivity.ts`, + // `triviality.ts` and `audit-trail`, so the two representations have to agree. + it("every callee name is attributed to an export that exists", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function shared(request) { return audit(request); } + export async function loader({ request }) { return json(await shared(request)); } + export async function action({ request }) { return json(await mutate(request)); }` + ); + const attributed = new Set([...ep!.loaderCalleeNames, ...ep!.actionCalleeNames]); + expect([...new Set(ep!.calleeNames)].filter((n) => !attributed.has(n))).toEqual([]); + for (const name of attributed) expect(ep!.calleeNames).toContain(name); + }); + + it("counts statements and a try per export as well as entry-point-wide", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const loader = () => redirect("/login"); + export async function action({ request }) { + try { return json(await request.json()); } catch (e) { throw e; } + }` + ); + expect(ep!.loaderStatementCount).toBe(1); + expect(ep!.loaderHasTryCatch).toBe(false); + expect(ep!.actionHasTryCatch).toBe(true); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.statementCount).toBe(ep!.loaderStatementCount + ep!.actionStatementCount); + }); +}); + // Round C ruling 2. A guard that answers with null instead of throwing is only a boundary if the // route reads the answer, so the scan records which callees' results a condition looked at. describe("scanFile: callees whose answer the body read", () => { @@ -2808,7 +2898,7 @@ describe("scanFile: callees whose answer the body read", () => { return json({ email: user.email }); }` ); - expect(ep!.checkedCallees).toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("getUser"); }); it("records one whose result a plain if tests", () => { @@ -2821,7 +2911,7 @@ describe("scanFile: callees whose answer the body read", () => { return typedjson({}); }` ); - expect(ep!.checkedCallees).toContain("getUserId"); + expect(ep!.loaderCheckedCallees).toContain("getUserId"); }); it("records one whose result a ternary tests", () => { @@ -2832,7 +2922,7 @@ describe("scanFile: callees whose answer the body read", () => { return user ? json({ ok: true }) : redirect("/login"); }` ); - expect(ep!.checkedCallees).toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("getUser"); }); it("does not record a result that is bound and never tested", () => { @@ -2843,7 +2933,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(await prisma.invite.findMany({ where: { email: user.email } })); }` ); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); it("does not record a result that is dropped entirely", () => { @@ -2854,7 +2944,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(await prisma.invite.findMany()); }` ); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); it("does not record a callee just because a same-named local is tested elsewhere", () => { @@ -2866,7 +2956,7 @@ describe("scanFile: callees whose answer the body read", () => { return json(rows); }` ); - expect(ep!.checkedCallees).toContain("findMany"); - expect(ep!.checkedCallees).not.toContain("getUser"); + expect(ep!.loaderCheckedCallees).toContain("findMany"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); }); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index c868fc142d1..2067e60203a 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -668,13 +668,11 @@ function containsExit(node: ts.Node): boolean { */ function literalTruth(expr: ts.Expression): boolean | null { const target = unwrap(expr); - if (target.kind === ts.SyntaxKind.TrueKeyword) return true; - if (target.kind === ts.SyntaxKind.FalseKeyword) return false; - if (target.kind === ts.SyntaxKind.NullKeyword) return false; - if (ts.isStringLiteral(target) || ts.isNoSubstitutionTemplateLiteral(target)) { - return target.text !== ""; - } - if (ts.isNumericLiteral(target)) return Number(target.text) !== 0; + // The five bare-literal kinds are `literalValue`'s list, not a second copy of it: this was the + // same five node-kind tests written out three lines above the function that already had them, + // differing only in returning the truthiness rather than the value. + const literal = literalValue(target); + if (literal !== undefined) return Boolean(literal); if (ts.isPrefixUnaryExpression(target) && target.operator === ts.SyntaxKind.ExclamationToken) { const inner = literalTruth(target.operand); return inner === null ? null : !inner; @@ -1395,6 +1393,51 @@ function countFunctionStatements(fn: EntryFunction): number { return countStatements(fn.body.statements); } +type ExportName = "loader" | "action"; + +/** + * The call-site facts a body walk accumulates, kept in one shape so the entry-point-wide totals and + * each export's own totals are filled by the same code rather than by two similar loops. + */ +type BodyFacts = { + calleeNames: string[]; + /** + * The same calls as `calleeNames`, each as its whole dotted path. `calleeName` keeps only the + * last segment, so `prisma.organization.findFirst` arrives in `calleeNames` as `findFirst` and + * the receiver that says WHAT is being called is gone. The per-export triviality rule needs it + * back: `prisma` in the path is how a short body is known to touch the datastore. + */ + calleeTexts: string[]; + /** Locals initialised from a call, by local name. */ + declaredFrom: Map; + /** Every identifier read by an `if`, `while`, `switch` or conditional condition. */ + testedNames: Set; + statementCount: number; + hasTryCatch: boolean; +}; + +function newBodyFacts(): BodyFacts { + return { + calleeNames: [], + calleeTexts: [], + declaredFrom: new Map(), + testedNames: new Set(), + statementCount: 0, + hasTryCatch: false, + }; +} + +/** Callees whose answer these bodies looked at: declared from a call AND read by a condition. */ +function checkedCalleesOf(facts: BodyFacts): string[] { + return [ + ...new Set( + [...facts.declaredFrom] + .filter(([local]) => facts.testedNames.has(local)) + .flatMap(([, callees]) => callees) + ), + ]; +} + type EntryTarget = { hasLoader: boolean; hasAction: boolean; @@ -1616,31 +1659,42 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (!target.hasLoader && !target.hasAction) return null; - let statementCount = 0; - let hasTryCatch = false; const callbackCatches: CatchEvidence[] = []; const catches: CatchEvidence[] = []; - const calleeNames: string[] = []; const logCalls: LogCall[] = []; - // Locals initialised from a call, and the names any condition in the body reads. Their - // intersection is `checkedCallees`: callees whose answer the route demonstrably looked at. - const declaredFrom = new Map(); - const testedNames = new Set(); - const collectTested = (node: ts.Node) => { - if (ts.isIdentifier(node)) testedNames.add(node.text); - ts.forEachChild(node, collectTested); + const wholeEntry = newBodyFacts(); + const byExport: Record = { + loader: newBodyFacts(), + action: newBodyFacts(), }; const localFunctions = collectLocalFunctions(sf); // A body that delegates to a same-file helper does the work in that helper, so the helper's // statements, try/catch and callees belong to the entry point. One hop only: a helper's own // helpers are not followed, and the visited set stops a cycle and any double counting. + // + // The helper's callees belong to whichever EXPORTS reach it, too, which is what `helperOwners` + // carries. A helper called from both halves of the file is owned by both; the union is taken on + // the second discovery rather than dropped, because `visited` has already queued it by then. const visited = new Set(target.functions); const helpers: EntryFunction[] = []; + const helperOwners = new Map>(); + + const walkBody = (fn: EntryFunction, followHelpers: boolean, owners: ReadonlySet) => { + // One push site feeds the entry-point-wide list and each owning export's list, so + // `calleeNames` and the per-export lists cannot drift apart. See `EntryPoint.loaderCalleeNames`. + const sinks: BodyFacts[] = [wholeEntry]; + for (const owner of owners) sinks.push(byExport[owner]); + const collectTested = (node: ts.Node) => { + if (ts.isIdentifier(node)) for (const s of sinks) s.testedNames.add(node.text); + ts.forEachChild(node, collectTested); + }; - const walkBody = (fn: EntryFunction, followHelpers: boolean) => { - statementCount += countFunctionStatements(fn); + const addStatements = (n: number) => { + for (const sink of sinks) sink.statementCount += n; + }; + addStatements(countFunctionStatements(fn)); if (!fn.body) return; // `inCallback` is true once the walk has entered a per-item iteration callback @@ -1665,13 +1719,13 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { // the mutation corpus is that shape. const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { - if (isEntryFunction(node)) statementCount += countFunctionStatements(node); + if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); const entersIterationCallback = inCallback || isIterationCallback(node); ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); return; } if (ts.isTryStatement(node)) { - hasTryCatch = true; + for (const sink of sinks) sink.hasTryCatch = true; if (node.catchClause) { // Built the same way for a refused catch as for an own one, so the dead-code defence // and the walk's guaranteed-execution rules apply to both. Which list it lands in is @@ -1699,9 +1753,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (ts.isCallExpression(initializer)) { const cn = calleeName(initializer.expression); if (cn) { - const existing = declaredFrom.get(node.name.text); - if (existing) existing.push(cn); - else declaredFrom.set(node.name.text, [cn]); + for (const sink of sinks) { + const existing = sink.declaredFrom.get(node.name.text); + if (existing) existing.push(cn); + else sink.declaredFrom.set(node.name.text, [cn]); + } } } } @@ -1715,7 +1771,10 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const cn = calleeName(node.expression); if (cn) { const text = calleeText(node.expression) ?? cn; - calleeNames.push(cn); + for (const sink of sinks) { + sink.calleeNames.push(cn); + sink.calleeTexts.push(text); + } if (LOGGER_CALLEE.test(text)) { logCalls.push({ @@ -1733,6 +1792,10 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { if (helper && !visited.has(helper)) { visited.add(helper); helpers.push(helper); + helperOwners.set(helper, new Set(owners)); + } else if (helper) { + const already = helperOwners.get(helper); + if (already) for (const owner of owners) already.add(owner); } } } @@ -1742,8 +1805,16 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { visit(fn.body, false, false); }; - for (const fn of target.functions) walkBody(fn, true); - for (const helper of helpers) walkBody(helper, false); + // Every handler is walked exactly once, whichever exports own it, so the entry-point-wide + // statement count and catch list stay single while the per-export lists see it from both sides. + const ownersOf = (fn: EntryFunction): Set => { + const owners = new Set(); + if (target.loaderFunctions.has(fn)) owners.add("loader"); + if (target.actionFunctions.has(fn)) owners.add("action"); + return owners; + }; + for (const fn of target.functions) walkBody(fn, true, ownersOf(fn)); + for (const helper of helpers) walkBody(helper, false, helperOwners.get(helper) ?? new Set()); return { fileName, @@ -1762,31 +1833,76 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { target.actionInitializerCallee === null, loaderScopesByCaller: scopesByCallerIn(target.loaderFunctions), actionScopesByCaller: scopesByCallerIn(target.actionFunctions), - checkedCallees: [ - ...new Set( - [...declaredFrom] - .filter(([local]) => testedNames.has(local)) - .flatMap(([, callees]) => callees) - ), - ], + loaderCheckedCallees: checkedCalleesOf(byExport.loader), + actionCheckedCallees: checkedCalleesOf(byExport.action), importedNames, - calleeNames, - hasTryCatch, + calleeNames: wholeEntry.calleeNames, + loaderCalleeNames: byExport.loader.calleeNames, + actionCalleeNames: byExport.action.calleeNames, + loaderCalleeTexts: byExport.loader.calleeTexts, + actionCalleeTexts: byExport.action.calleeTexts, + hasTryCatch: wholeEntry.hasTryCatch, + loaderHasTryCatch: byExport.loader.hasTryCatch, + actionHasTryCatch: byExport.action.hasTryCatch, catches, callbackCatches, logCalls, - statementCount, + statementCount: wholeEntry.statementCount, + loaderStatementCount: byExport.loader.statementCount, + actionStatementCount: byExport.action.statementCount, }; } const SOURCE_FILE = /\.tsx?$/; -/** Exported so `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads. Its - * anti-vacuity thresholds count files and sites the scanner never saw if the two predicates drift. */ +/** + * Whether a file name is one the scanner reads at all. + * + * Exported because three other places ask the same question and each had written its own copy: + * `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads, and + * `integration.test.ts` and `webappSymbols.test.ts` walk trees of their own. The corpus's + * anti-vacuity thresholds count files and sites the scanner never saw if those predicates drift, + * and `integration.test.ts`'s `entryPoints.length < countRouteModuleFiles(ROUTES)` stops meaning + * anything if its denominator counts files the scanner skips. + */ export function isScannableFile(fileName: string): boolean { return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); } +/** One route module: where to read it, and the name the report and the scan record it under. */ +export type RouteModuleFile = { absolutePath: string; relativeName: string }; + +/** + * The route modules under `dir`: every scannable flat file, plus the `route.ts`/`route.tsx` of each + * immediate subdirectory. + * + * Exported so `mutationCorpus.test.ts` can materialize exactly this set rather than re-deriving it. + * Its `readTree` was a verbatim copy of the walk below; the FILE half of that copy was later + * replaced by a call to `isScannableFile` while the DIRECTORY half stayed duplicated, which is the + * usual way this package's duplicates half-die. A corpus that enumerates a different tree from the + * scanner reports file and site counts for files the scan never reads, and those counts are the + * only thing standing between a mutation that reaches nothing and a green test. + */ +export function routeModuleFiles(dir: string): RouteModuleFile[] { + const files: RouteModuleFile[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + files.push({ + absolutePath: join(dir, entry.name, child.name), + relativeName: `${entry.name}/${child.name}`, + }); + } + continue; + } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + files.push({ absolutePath: join(dir, entry.name), relativeName: entry.name }); + } + return files; +} + export function scanDirectory(dir: string): { entryPoints: EntryPoint[]; parseFailures: string[]; @@ -1810,18 +1926,7 @@ export function scanDirectory(dir: string): { if (ep) entryPoints.push(ep); }; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - if (entry.isDirectory()) { - // Flat-route directories hold the route module in `route.ts`/`route.tsx`. - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; - scan(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); - } - continue; - } - if (!entry.isFile() || !isScannableFile(entry.name)) continue; - scan(join(dir, entry.name), entry.name); - } + for (const file of routeModuleFiles(dir)) scan(file.absolutePath, file.relativeName); return { entryPoints, parseFailures }; } diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts index e36b80dc9b0..15515d1ec36 100644 --- a/internal-packages/observability-map/src/triviality.ts +++ b/internal-packages/observability-map/src/triviality.ts @@ -1,9 +1,10 @@ +import type { RouteExport } from "./routeExports.js"; import type { EntryPoint } from "./types.js"; /** - * Substrings that say the route touches a service, a datastore or the network. Matched against the - * callee names and the whole file, so an import of `prisma` disqualifies the file even when the - * query itself sits somewhere the scanner does not walk. + * Substrings that say the route touches a service, a datastore or the network. Always matched + * against the callee names, and additionally against `TrivialityView.hintText`, which is the whole + * file for the entry-point-wide view and empty for a per-export one. */ const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; @@ -23,6 +24,42 @@ const MAX_CALLS = 3; */ const MAX_STATEMENTS = 3; +/** + * What the rule reads, so the entry-point-wide answer and a single export's answer are the same + * rule over different bodies rather than two rules that can drift. + */ +type TrivialityView = { + statementCount: number; + calleeNames: string[]; + hasTryCatch: boolean; + /** Every builder call in scope of this view. A view with one is never trivial. */ + initializerCallees: (string | null)[]; + /** + * Text to match the side-effect hints against besides the callee names. + * + * The whole file for the entry-point-wide view, so an import of `prisma` disqualifies it even + * when the query sits somewhere the scanner does not walk. For a per-export view it is that + * export's own callee PATHS instead, and the difference is not a convenience: + * + * - The file's text is a fact about the file, so reading it into one export's verdict is the + * per-file-for-per-export substitution this rule exists to damp. It is also defeatable. + * `log-caller-scope-userid` in the mutation corpus prepends `logger.error(...)` to every body; + * with this term file-wide that put the word `logger` in `auth.github.ts` and turned its + * untouched one-line redirect loader from excused into accused, on a rewrite that changed + * nothing the loader does. + * - Emptying it instead is not the answer either, and that was measured: `calleeNames` keeps only + * a call's last segment, so `prisma.orgMember.findMany` reads as `findMany` and a + * three-statement body that queries the datastore matches no hint at all. Five existing + * `auth-boundary` fixtures went from `fail` to `not-applicable`, which is the check being + * switched off rather than fixed. + * + * The callee paths are body-scoped like the first option wants and name the receiver like the + * second needs. Comments and imports are not in them, which is deliberate: everything in this + * view is something the export actually does. + */ + hintText: string; +}; + /** * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed * response, or hands off in a single call. Checks report not-applicable for these rather than @@ -31,22 +68,55 @@ const MAX_STATEMENTS = 3; * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the * report again, so every signal that the body might be doing real work rules triviality out: * - * - `statementCount` does not descend into inline callbacks, so a two-statement body can still hold - * a pile of work. `calleeNames` does descend, so the call count catches what the statement count - * misses. + * - `statementCount` counts a nested function's statements but `calleeNames` descends further, into + * the callee of every call at any depth, so the call count still catches bodies the statement + * count reads as short. * - An initializer callee means the route is wrapped in a builder, and the config passed to that * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is * not the whole route, so we cannot claim it is trivial. * - A try/catch is exactly what the error-classification check reads, so a body with one has an * error path worth reporting on however short it is. */ +function isTrivialView(view: TrivialityView): boolean { + if (view.statementCount > MAX_STATEMENTS) return false; + if (view.calleeNames.length > MAX_CALLS) return false; + if (view.hasTryCatch) return false; + if (view.initializerCallees.some((c) => c !== null)) return false; + + const callees = view.calleeNames.join(" ").toLowerCase(); + const hints = view.hintText.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || hints.includes(h)); +} + +/** The rule over everything the entry point does, both exports and their same-file helpers. */ export function isTrivial(ep: EntryPoint): boolean { - if (ep.statementCount > MAX_STATEMENTS) return false; - if (ep.calleeNames.length > MAX_CALLS) return false; - if (ep.hasTryCatch) return false; - if (ep.loaderInitializerCallee !== null || ep.actionInitializerCallee !== null) return false; - - const callees = ep.calleeNames.join(" ").toLowerCase(); - const source = ep.source.toLowerCase(); - return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || source.includes(h)); + return isTrivialView({ + statementCount: ep.statementCount, + calleeNames: ep.calleeNames, + hasTryCatch: ep.hasTryCatch, + initializerCallees: [ep.loaderInitializerCallee, ep.actionInitializerCallee], + hintText: ep.source, + }); +} + +/** + * The same rule over ONE export's handlers. + * + * Needed because a per-export verdict judged against an entry-point-wide triviality rule accuses + * the wrong half of a file. `auth.github.ts` and `auth.google.ts` are + * `export let loader = () => redirect("/login")` beside an action that calls + * `authenticator.authenticate`: per export the loader is unguarded, and the entry-point-wide rule + * calls the file non-trivial because the ACTION is not, so `auth-boundary` accused a one-line + * redirect stub of missing an auth guard. `checks/index.test.ts` pins both directions of that + * ("reports not-applicable for a redirect-stub loader beside a guarded action" and "fails an export + * whose own body does real work unguarded"). + */ +export function isTrivialExport(e: RouteExport): boolean { + return isTrivialView({ + statementCount: e.statementCount, + calleeNames: e.calleeNames, + hasTryCatch: e.hasTryCatch, + initializerCallees: [e.initializerCallee], + hintText: e.calleeTexts.join(" "), + }); } diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index b75e4714476..f11fb4a4a1f 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -129,31 +129,67 @@ export type EntryPoint = { loaderScopesByCaller: boolean; actionScopesByCaller: boolean; /** - * Callees whose answer the body demonstrably looked at: the call's result was bound to a local - * and some `if`, `while`, `switch` or conditional in the same bodies reads that local. - * `const user = await getUser(request); if (!user) return redirect("/login");` puts `getUser` - * here; a call whose result is dropped, or bound and never tested, does not appear. + * Callees whose answer THIS export's handlers demonstrably looked at: the call's result was bound + * to a local and some `if`, `while`, `switch` or conditional in the same handlers reads that + * local. `const user = await getUser(request); if (!user) return redirect("/login");` puts + * `getUser` here; a call whose result is dropped, or bound and never tested, does not appear. * * Read by `auth-boundary` for the guards that answer with null instead of throwing, where being * called is not evidence that the route acted on the answer. * + * Split per export for the same reason `loaderScopesByCaller` is, and there is no entry-point-wide + * version on purpose: a loader that reads what `getUser` returned says nothing about the action + * beside it, so the union is not a fact any check should be able to reach for. + * * Deliberately coarse. It does not check that the test guards anything, that the local is the one * tested rather than a same-named one in another scope, or that the branch exits: a route * that writes `if (!user) { logger.warn("anonymous"); }` and carries on is credited. It separates * "looked at the answer" from "ignored it", which is the distinction the check needs, and not * "acted correctly on the answer", which it cannot see. */ - checkedCallees: string[]; + loaderCheckedCallees: string[]; + actionCheckedCallees: string[]; /** Named and default imports, file-wide. */ importedNames: string[]; - /** Names of functions called inside the loader/action bodies, or in a same-file helper they call. */ + /** + * Names of functions called inside the loader/action bodies, or in a same-file helper they call. + * + * Entry-point-wide, and read only by the questions that are themselves entry-point-wide: + * `sensitivity.ts` asks what the file touches, `triviality.ts` counts how much the file does, + * `audit-trail` asks whether the file records anything. A question about ONE export's exposure + * must read `loaderCalleeNames`/`actionCalleeNames` instead. `auth-boundary` read this and + * credited a file whose loader called a guard for an action that called none. + */ calleeNames: string[]; + /** + * The same callee names attributed to the export whose handlers made the call. A handler serving + * both exports (`const { loader, action } = createActionApiRoute({ handler })`) contributes to + * both, and a same-file helper contributes to whichever exports reach it. + * + * Every name here appears in `calleeNames` and every name in `calleeNames` appears in at least one + * of these, because all three are filled from one push in `scanFile`. `scan.test.ts` pins that + * ("every callee name is attributed to an export that exists") and `integration.test.ts` pins it + * again across the real route tree, so the split cannot drift away from the union it came from. + */ + loaderCalleeNames: string[]; + actionCalleeNames: string[]; + /** + * The same calls as `loaderCalleeNames`/`actionCalleeNames`, each as its whole dotted path + * (`prisma.organization.findFirst` rather than `findFirst`). Read by the per-export triviality + * rule, which has to know that a three-statement body reaches the datastore; the bare name that + * `auth-boundary` matches guards against throws that receiver away. + */ + loaderCalleeTexts: string[]; + actionCalleeTexts: string[]; /** * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. */ hasTryCatch: boolean; + /** The same fact for one export's handlers alone. Read by the per-export triviality rule. */ + loaderHasTryCatch: boolean; + actionHasTryCatch: boolean; /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; /** @@ -176,4 +212,11 @@ export type EntryPoint = { * one hop only: work in a helper's own helpers, or in an imported module, is not counted. */ statementCount: number; + /** + * The same count for one export's handlers alone, counted by the same walk. A handler serving both + * exports is counted once in `statementCount` and once in each of these, so the two do not sum to + * the entry point's total and must not be used as though they did. + */ + loaderStatementCount: number; + actionStatementCount: number; }; From 4f80d58267e100d55a4d17e6aa53b4b59fb7f684 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:46:11 +0100 Subject: [PATCH 104/117] fix(observability-map): widen the corpus helper to every export form mutations.ts entryBodies collected exported function declarations and exported const identifiers only, so it missed the object binding pattern (export const { action, loader } = createActionApiRoute(...)), the export clause (const { action } = builder(...); export { action }), and export const action = route.action. That is 36 of the tree's 427 entry points, all of them API routes: every whole-body corpus entry skipped them while the file count suggested otherwise. The scanner has read all four forms since early on, so this was the harness lagging it. No assertion could have noticed. A mutation that reaches fewer routes lowers the score rather than raising it, which is exactly how the suppress-every-check omission hid, so the answer is the same: assert the population. admin.tsx is the one exclusion, named rather than counted, because its handler is a concise arrow with no block for a block wrapper to wrap. wrap-body-in-rethrow goes from 391 files with 36 entry points missed to 426 files, 1020 sites, 1 missed. Widening changes no entry's verdict: the full corpus is 55 passed and 1 expected fail either way, and with the narrow population the only failure is the new population assertion itself. readTree now calls routeModuleFiles rather than keeping its own copy of the directory walk. isScannableFile had already replaced the file half of that copy; the directory half survived. --- .../src/mutationCorpus.test.ts | 74 ++++++--- .../observability-map/src/mutations.ts | 140 ++++++++++++++++-- 2 files changed, 178 insertions(+), 36 deletions(-) diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index cc8128c3750..658ce32eca9 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -1,7 +1,7 @@ -import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { isScannableFile, scanDirectory } from "./scan.js"; +import { routeModuleFiles, scanDirectory } from "./scan.js"; import { buildReport } from "./score.js"; import { ADDITIVE_IDS, MUTATIONS, type Mutation } from "./mutations.js"; import { CHECKS } from "./checks/index.js"; @@ -62,27 +62,20 @@ const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); type SourceFile = { relativeName: string; source: string }; -/** Route modules exactly as `scanDirectory` enumerates them: flat files, plus one `route.ts(x)` per - * directory. Read once; every mutation rewrites this list rather than the tree on disk. */ +/** + * Route modules exactly as `scanDirectory` enumerates them, because it is the same enumeration and + * no longer a copy of it. `isScannableFile` had already replaced the file half of the copy; the + * directory half survived, so "one `route.ts(x)` per immediate subdirectory" was still written + * twice. A harness that reads a different tree from the scanner reports files and sites the scan + * never saw, and those counts are what the thresholds below rest on. + * + * Read once; every mutation rewrites this list rather than the tree on disk. + */ function readTree(dir: string): SourceFile[] { - const files: SourceFile[] = []; - const take = (absolutePath: string, relativeName: string) => { - files.push({ relativeName, source: readFileSync(absolutePath, "utf8") }); - }; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - if (entry.isDirectory()) { - for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { - if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; - take(join(dir, entry.name, child.name), `${entry.name}/${child.name}`); - } - continue; - } - // `scanDirectory`'s own predicate rather than a copy of it. A copy that drifts lets a mutation - // report files and sites the scanner never read, which is what the thresholds below are for. - if (!entry.isFile() || !isScannableFile(entry.name)) continue; - take(join(dir, entry.name), entry.name); - } - return files; + return routeModuleFiles(dir).map((file) => ({ + relativeName: file.relativeName, + source: readFileSync(file.absolutePath, "utf8"), + })); } function materialize(files: SourceFile[]): string { @@ -318,6 +311,43 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME baseline = measure(files); }, ENTRY_TIMEOUT_MS); + /** + * The corpus's own population, against the scanner's. + * + * The whole-body entries wrap what `entryBodies` finds, and that helper read two of the four + * export forms `scan.ts` reads. It missed `export const { action, loader } = builder(...)`, + * `const { action } = builder(...); export { action };` and `export const action = route.action`, + * which is 36 of the tree's entry points: the corpus was testing less than its entry count + * implied, and no assertion could notice, because a mutation that reaches fewer routes lowers the + * score rather than raising it. Same failure mode as the `suppress-every-check` omission above, + * so the answer is the same: assert the population rather than wait for a verdict to move. + * + * `admin.tsx` is the one documented exclusion. Its handler is a concise arrow + * (`async ({ user }) => typedjson({ user })`) with no block for a block wrapper to wrap, which is + * a limit of the rewrite rather than a gap in the enumeration. It is named rather than counted so + * a second one cannot appear silently. + */ + const CONCISE_ARROW_BODIES = new Set(["admin.tsx"]); + + it("wraps a body in every non-delegating entry point the scanner finds", () => { + const wrap = MUTATIONS.find((m) => m.id === "wrap-body-in-rethrow")!; + const root = materialize(files); + let entryPoints; + try { + ({ entryPoints } = scanDirectory(root)); + } finally { + rmSync(root, { recursive: true, force: true }); + } + + const byName = new Map(files.map((f) => [f.relativeName, f.source])); + const untouched = entryPoints + .filter((ep) => !ep.delegating && !CONCISE_ARROW_BODIES.has(ep.fileName)) + .filter((ep) => wrap.apply(ep.fileName, byName.get(ep.fileName)!) === null) + .map((ep) => ep.fileName); + + expect(untouched).toEqual([]); + }); + it("has a baseline worth mutating", () => { expect(baseline).not.toBeNull(); expect(baseline!.entryPoints).toBeGreaterThan(300); diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index d273728ec03..64d6a07f9ad 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -179,18 +179,44 @@ function rootCall(call: ts.CallExpression): ts.CallExpression { } } -function fromInitializer(expr: ts.Expression, out: EntryFunction[]): void { +/** + * The handler functions an export's initializer resolves to. + * + * `locals` is consulted for the two indirect spellings, both of which `scan.ts` resolves and + * neither of which this reached: `export const action = route.action` beside + * `const route = createActionApiRoute(...)`, which is 7 of the tree's API routes, and + * `export const action = handleThing` naming a local. `seen` stops `const a = b; const b = a`. + */ +function fromInitializer( + expr: ts.Expression, + out: EntryFunction[], + locals: LocalDeclarations, + seen: Set = new Set() +): void { const target = unwrap(expr); if (isEntryFunction(target)) { out.push(target); return; } - if (!ts.isCallExpression(target)) return; - for (const arg of rootCall(target).arguments) { - const unwrapped = unwrap(arg); - if (isEntryFunction(unwrapped)) out.push(unwrapped); - else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + if (ts.isCallExpression(target)) { + for (const arg of rootCall(target).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } + return; } + // `route.action`, and any longer chain, is resolved from whatever declared its root identifier. + let root: ts.Expression = target; + while (ts.isPropertyAccessExpression(root) || ts.isElementAccessExpression(root)) { + root = unwrap(root.expression); + } + if (!ts.isIdentifier(root) || seen.has(root.text)) return; + const declaration = locals.get(root.text); + if (declaration === undefined) return; + seen.add(root.text); + if (ts.isFunctionDeclaration(declaration)) out.push(declaration); + else fromInitializer(declaration, out, locals, seen); } const ENTRY_NAMES = new Set(["loader", "action"]); @@ -202,23 +228,109 @@ function isExported(node: ts.Node): boolean { ); } -/** Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. */ -function entryBodies(sf: ts.SourceFile): ts.Block[] { - const functions: EntryFunction[] = []; +/** + * Top-level declarations by binding name, so a named export clause (`export { action }`) resolves + * back to the initializer it came from. Object binding patterns are read element by element, which + * is what makes `const { action, loader } = createActionApiRoute(...)` resolvable. + */ +type LocalDeclarations = Map; + +function localDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); for (const statement of sf.statements) { - if (!isExported(statement)) continue; if (ts.isFunctionDeclaration(statement) && statement.name) { - if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + locals.set(statement.name.text, statement); continue; } if (!ts.isVariableStatement(statement)) continue; for (const decl of statement.declarationList.declarations) { - if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; - if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions); + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } } } + return locals; +} + +/** + * Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. + * + * Reads the same four export forms `scan.ts` reads: an exported function declaration, an exported + * `const`, an exported object binding pattern, and a named export clause resolved back through a + * local. It read only the first two, which is the shape of every API route in the tree + * (`const { action, loader } = createActionApiRoute(...); export { action, loader };` and the + * direct `export const { action } = ...`), so `wrapEveryBody` and the other whole-body entries + * silently skipped 36 of the 427 entry points while reporting a file count that suggested + * otherwise. `mutationCorpus.test.ts` pins the population now ("wraps a body in every + * non-delegating entry point the scanner finds"), so the harness cannot lag the scanner here again + * without going red. + * + * This is NOT a retreat from the deliberate independence `collectNamedHandlers` documents. That + * independence is about disagreeing over where a HANDLER sits inside a builder's argument, which is + * a judgement the corpus has to be able to make for itself. Which exports exist is not a judgement, + * and the harness was simply behind. + */ +function entryBodies(sf: ts.SourceFile): ts.Block[] { + const functions: EntryFunction[] = []; + const locals = localDeclarations(sf); + const fromLocal = (name: string) => { + const decl = locals.get(name); + if (decl === undefined) return; + if (ts.isFunctionDeclaration(decl)) functions.push(decl); + else fromInitializer(decl, functions, locals); + }; + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + if (ENTRY_NAMES.has(statement.name.text)) functions.push(statement); + continue; + } + + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + if (ENTRY_NAMES.has(decl.name.text)) fromInitializer(decl.initializer, functions, locals); + continue; + } + if (!ts.isObjectBindingPattern(decl.name)) continue; + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name) && ENTRY_NAMES.has(element.name.text)) { + fromInitializer(decl.initializer, functions, locals); + } + } + } + continue; + } + + // A re-export (`export { loader } from "./x"`) has no local binding to resolve, and a namespace + // clause cannot name a loader or an action. + if ( + ts.isExportDeclaration(statement) && + statement.exportClause && + !statement.moduleSpecifier && + ts.isNamedExports(statement.exportClause) + ) { + for (const element of statement.exportClause.elements) { + if (!ENTRY_NAMES.has(element.name.text)) continue; + fromLocal(element.propertyName?.text ?? element.name.text); + } + } + } + + // One handler can serve both exports, and both reach it by their own road: the loader and the + // action of `const { loader, action } = createActionApiRoute({ handler })` resolve to the same + // node. Wrapping it twice would splice the same text in twice at the same offset, because + // `applyEdits` treats two zero-width inserts at one position as non-overlapping. const bodies: ts.Block[] = []; - for (const fn of functions) { + for (const fn of new Set(functions)) { if (fn.body && ts.isBlock(fn.body)) bodies.push(fn.body); } return bodies; From 2de22e4e031b0cd8c3e07aa8b80223fdce2fb754 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 04:46:35 +0100 Subject: [PATCH 105/117] refactor(observability-map): share or pin the rest of the duplicated rules Sweeping the package for the defect behind the two review threads: one question answered by two pieces of code, where only one copy gets fixed. Shared: - the scannable-file predicate, copied into integration.test.ts and webappSymbols.test.ts after it was exported to stop mutationCorpus.test.ts copying it - the FIX FIRST filter and sort, byte-identical in terminal.ts and prComment.ts, which already imports five helpers from it; failingIds is now scoredFailures plus a map - normalizeSegment, in the test that validates SENSITIVE_SEGMENTS against the real tree. It was splitting segments with /_+$/, the regex that function's own comment says not to use - the five bare-literal node kinds, written out in literalTruth three lines above the literalValue that already had them Pinned: - contextGap and auditGap, which redo by hand what checkContributions computes generically, on two headline figures with nothing saying they had to agree. Reverting either to a different denominator or numerator now goes red Left alone, with reasons recorded in the sweep report: canRaise vs tryBlockMayThrow, the two exact true-keyword folds, the two comment extractors, the three means, ratio vs globalWithout, and the eight AST helpers mutations.ts keeps its own copies of so the corpus can disagree with the scanner. --- .../observability-map/src/report/prComment.ts | 15 +---- .../observability-map/src/report/terminal.ts | 28 +++++--- .../observability-map/src/score.test.ts | 66 +++++++++++++++++++ .../observability-map/src/sensitivity.ts | 2 +- .../src/webappSymbols.test.ts | 15 ++++- 5 files changed, 102 insertions(+), 24 deletions(-) diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 03872cb7c80..3a5d5842aee 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -1,10 +1,9 @@ import type { MapReport, ScoredEntry } from "../score.js"; -import { SCORED_CHECK_IDS } from "../checks/index.js"; import { auditLine, checkContributionLines, contextLine, - contextOnly, + fixFirst, delegatedLines, scoredFailures, unknownSuppressionLines, @@ -40,8 +39,7 @@ const MAX_DELEGATED_ROUTES = 15; // Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost // every sensitive mutation today, so listing it per route would nag with something unfixable // instead of surfacing the route-specific gaps this column exists for. -const failingIds = (e: ScoredEntry) => - e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail").map((c) => c.id); +const failingIds = (e: ScoredEntry) => scoredFailures(e).map((c) => c.id); function scoreLine(head: MapReport, base: MapReport | null): string { const headline = @@ -197,14 +195,7 @@ function whatChangedSection(head: MapReport, base: MapReport | null): string[] { function fixFirstSection(head: MapReport): string[] { const lines = ["FIX FIRST"]; - const worst = head.entries - .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) - .sort( - (a, b) => - Number(b.sensitive) - Number(a.sensitive) || - a.score - b.score || - a.fileName.localeCompare(b.fileName) - ); + const worst = fixFirst(head.entries); for (const e of worst.slice(0, 3)) { const marks = e.sensitive ? " (sensitive)" : ""; diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index dc59e205342..0bb904bae50 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -27,6 +27,25 @@ export const scoredFailures = (e: ScoredEntry) => * An entry that fails something else as well stays in the list with all of its findings, so a * route like `/account/tokens` still shows the request-context gap alongside the rest. */ +/** + * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, + * then by name. Exported because `prComment.ts` renders the same list with different bullets and + * had a byte-identical copy of this filter and sort, in a file that already imports + * `scoredFailures` and `contextOnly` from here. + * + * `contextOnly` routes are left out because `request-context` fails almost everything, so a list + * headed by three of them tells a reader nothing they cannot read off the gap figure. + */ +export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => + entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + export const contextOnly = (e: ScoredEntry) => { const failures = scoredFailures(e); return failures.length === 1 && failures[0]!.id === "request-context"; @@ -197,14 +216,7 @@ export function renderTerminal(report: MapReport): string { ); } - const worst = report.entries - .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) - .sort( - (a, b) => - Number(b.sensitive) - Number(a.sensitive) || - a.score - b.score || - a.fileName.localeCompare(b.fileName) - ); + const worst = fixFirst(report.entries); lines.push(""); lines.push("FIX FIRST"); diff --git a/internal-packages/observability-map/src/score.test.ts b/internal-packages/observability-map/src/score.test.ts index a8f38a7f4ce..a8d6943d0e0 100644 --- a/internal-packages/observability-map/src/score.test.ts +++ b/internal-packages/observability-map/src/score.test.ts @@ -561,3 +561,69 @@ export async function action() { expect(after.unmeasured).toBe(0); }); }); + +/** + * `contextGap` and `auditGap` are the same arithmetic `checkContributions` already does for every + * check, written out again by hand for two named ids: `map(find).filter(status)` for the context + * figure, `filter(some)` for the audit one, and a third spelling of "passed" for each. Three + * implementations of "applicable, and how many of those passed", and nothing said they had to + * agree, on the two figures the report puts in front of a reader as headline numbers. + * + * Pinned rather than shared. Collapsing them would mean the gap figures reading their check's row + * out of `checkContributions`, which is a fine refactor and a wider blast radius than the property + * is worth: what matters is that they cannot disagree, and an assertion says that without moving + * any code the renderers read. + */ +describe("the hand-rolled gap figures agree with the per-check contributions", () => { + const SOURCE = `import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +export async function action({ params }) { + try { + return await prisma.apiKey.create({ data: { orgId: params.orgId } }); + } catch (e) { + logger.error("failed", { orgId: params.orgId }); + return null; + } +}`; + + // A sensitive mutation that DOES record an audit event, so `withAudit` is not simply + // `sensitiveMutations`. Without it the audit assertion held whatever the numerator counted. + const AUDITED = `import { prisma } from "~/db.server"; +import { startImpersonation } from "~/models/admin.server"; +export async function action({ request, params }) { + const session = await startImpersonation(request, params.userId); + await prisma.apiKey.create({ data: { orgId: params.orgId } }); + return redirect("/", { headers: session }); +}`; + + const report = buildReport( + [ + scanFile("api.v1.orgs.$orgId.apikeys.ts", SOURCE)!, + scanFile("api.v1.tokens.ts", SOURCE)!, + scanFile("resources.impersonation.ts", AUDITED)!, + scanFile("healthcheck.ts", `export const loader = () => new Response("ok");`)!, + ], + [] + ); + + const contribution = (id: string) => report.checkContributions.find((c) => c.id === id)!; + + it("reports the same request-context denominator and numerator", () => { + expect(report.contextGap.applicable).toBe(contribution("request-context").applicable); + expect(report.contextGap.naming).toBe(contribution("request-context").passed); + }); + + it("reports the same audit-trail denominator and numerator", () => { + expect(report.auditGap.sensitiveMutations).toBe(contribution("audit-trail").applicable); + expect(report.auditGap.withAudit).toBe(contribution("audit-trail").passed); + }); + + // A denominator of zero would make both assertions above hold vacuously. + // Both assertions above hold vacuously on a zero denominator, and the audit one holds vacuously + // whenever every applicable route fails, since the two counts coincide. + it("measured something for both of them, with the audit numerator strictly between", () => { + expect(report.contextGap.applicable).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeLessThan(report.auditGap.sensitiveMutations); + }); +}); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index 760d7cd189e..ba13454fe17 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -122,7 +122,7 @@ export const SENSITIVE_SEGMENTS = [ * written the way a reader would say it. A trailing underscore opts a route out of its parent * layout (`resources.impersonation_.view-as.ts`) and changes nothing about what the route does. */ -function normalizeSegment(segment: string): string { +export function normalizeSegment(segment: string): string { // Trimmed by hand rather than with /_+$/, which backtracks polynomially on a run of underscores // and trips CodeQL. Nothing here is attacker-controlled (the input is a filename read off disk), // so this is about not spending a reviewer's attention on the alert. diff --git a/internal-packages/observability-map/src/webappSymbols.test.ts b/internal-packages/observability-map/src/webappSymbols.test.ts index f9542a0261e..3a709d2d913 100644 --- a/internal-packages/observability-map/src/webappSymbols.test.ts +++ b/internal-packages/observability-map/src/webappSymbols.test.ts @@ -3,7 +3,13 @@ import { readdirSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { AUDIT_SYMBOLS } from "./checks/auditTrail.js"; import { GUARDS, SOFT_GUARDS } from "./checks/authBoundary.js"; -import { ANTICIPATED_SEGMENTS, SENSITIVE_SEGMENTS, SENSITIVE_SYMBOLS } from "./sensitivity.js"; +import { isScannableFile } from "./scan.js"; +import { + ANTICIPATED_SEGMENTS, + normalizeSegment, + SENSITIVE_SEGMENTS, + SENSITIVE_SYMBOLS, +} from "./sensitivity.js"; /** * Every name and every path segment the tool matches on must exist in the codebase it is pointed @@ -82,7 +88,7 @@ function walkFiles(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir, { withFileTypes: true })) { const path = join(dir, entry.name); if (entry.isDirectory()) walkFiles(path, out); - else if (/\.tsx?$/.test(entry.name) && !entry.name.endsWith(".d.ts")) out.push(path); + else if (isScannableFile(entry.name)) out.push(path); } return out; } @@ -137,7 +143,10 @@ function routeSegments(): Set { const segments = new Set(); for (const entry of readdirSync(ROUTES, { withFileTypes: true })) { for (const part of entry.name.replace(/\.tsx?$/, "").split(".")) { - segments.add(part.replace(/_+$/, "")); + // `sensitivity.ts`'s own normalizer. This validates the vocabulary that file matches on, so + // a segment has to be trimmed here exactly as it is trimmed there; the local `/_+$/` was + // also the regex `normalizeSegment`'s own comment says not to use. + segments.add(normalizeSegment(part)); } } return segments; From 4b0b24364a2868b27ae00411bd33234fd12e630b Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 05:38:26 +0100 Subject: [PATCH 106/117] fix(observability-map): read a branch arm's exit through the liveness fold selectsADistinctPath decided whether an if or a switch in a catch clause made a real classification decision by asking containsExit, a plain containment walk. Containment is true of an exit that can never run, so catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, { status: 500 }); } read as a decision while the same clause without the if read as a swallow: 50 points a route for a behaviour-preserving mechanical edit. Measured over apps/webapp/app/routes, that shape took the tree from 19 to 27 and raised 80 of 412 routes. An earlier wave had already moved catchClauseEvidence's exited flag onto containsLiveExit for the same eleven dead spellings. The branch predicate 120 lines below it kept the containment read, so this is one rule fixed in one place and left in its sibling. The three exit reads now go through containsLiveExit and containsExit is deleted, so there is one exit read in the file. The property that makes one helper safe for two callers reading it for opposite purposes is now written down on containsLiveWhere: it is strictly subtractive against containment, so it only ever un-blinds the exited flag and only ever withholds a branch grant. Conservatism is a property of the helper plus what the caller does with a true, and auditing it at the definition is how this was missed. Adds dead-armed-instanceof-if to the mutation corpus, additive class, and dead-conjunction-instanceof-if under KNOWN_GAPS. The second is the sibling this fix does not close: folding the arm does not fold a dead condition, and if (e instanceof Error && false) reaches the same grant for the same 80 routes. literalTruth treats && and || as always null on purpose, so closing it means widening that fold, a different rule with its own measurement. Recorded and running rather than left to be rediscovered. Requiring the arm to definitelyExits was measured and rejected: it accuses admin.api.v1.orgs.$organizationId.environments.staging.ts, which classifies Prisma's P2002 and rethrows everything else, of taking one way out regardless of what was thrown. Pinned by 'still credits an arm guarded by a condition that does not fold'. The real tree does not move: every route's score and every check's status are byte-identical before and after, global 19 either way. --- .../src/mutationCorpus.test.ts | 13 ++- .../observability-map/src/mutations.ts | 30 +++++++ .../observability-map/src/scan.test.ts | 84 ++++++++++++++++++- .../observability-map/src/scan.ts | 59 ++++++++----- 4 files changed, 164 insertions(+), 22 deletions(-) diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index 658ce32eca9..ef2094e0aee 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -57,8 +57,19 @@ const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; * flag raised before each statement's own branch check, which makes every deciding statement refuse * itself. Raising it after is byte-identical on the real tree and closes the shape, so the entry is * defended now and the `if (true)` family needed no condition folding after all. + * + * `dead-conjunction-instanceof-if` is the sibling of `dead-armed-instanceof-if` that the arm-liveness + * fix does not close. `selectsADistinctPath` now folds a dead ARM; a dead CONDITION still reaches + * the grant, and `e instanceof Error && false` is exactly that, a guard that references the caught + * binding and can never be true. No fold in `scan.ts` can see it, because `literalTruth` treats + * `&&` and `||` as always null on purpose so that a live guard can never be read as dead. Widening + * that fold is a different rule from the one this round fixed and needs its own measurement, so the + * shape is recorded and running rather than left for the next person to rediscover. */ -const KNOWN_GAPS = new Set(["dead-classifying-try-with-call"]); +const KNOWN_GAPS = new Set([ + "dead-classifying-try-with-call", + "dead-conjunction-instanceof-if", +]); type SourceFile = { relativeName: string; source: string }; diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index 64d6a07f9ad..4273223f881 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -843,6 +843,34 @@ export const MUTATIONS: Mutation[] = [ (e) => `if (false) { if (${e} instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }` ), + // The sibling of `empty-instanceof-if`. That entry's arm is empty; this one's arm holds an exit + // that can never run, which is the same no-op written so that a containment read cannot tell the + // difference. `selectsADistinctPath` asked a plain containment question, true of + // `if (false) { return null; }`, so the test read as a real classification decision and turned a + // swallowing catch into a passing one: 80 routes and the tree from 19 to 27 when measured. + // Additive: it plants fake signal. `catchClauseEvidence`'s own `exited` flag had already been + // moved onto `containsLiveExit` for exactly this reason and the branch predicate beside it was + // left behind, which is why the shape is spelled with the corpus's own `if (false)` and not + // something exotic. + prependToEveryCatch( + "dead-armed-instanceof-if", + "preserving", + "splice if (e instanceof Error) { if (false) { return null; } } into every catch", + (e) => `if (${e} instanceof Error) { if (false) { return null; } }` + ), + // The sibling the entry above does NOT close, found while closing it and filed here rather than + // fixed. Moving the ARM's exit read onto `containsLiveExit` folds a dead arm; it does not fold a + // dead CONDITION, and a condition that both references the caught binding and is provably false + // reaches the same grant. `literalTruth` cannot see it: `&&` and `||` are documented there as + // always null, deliberately, so `e instanceof Error && false` is an undecidable guard to every + // fold in the file. Closing it means widening that fold, which is a different rule with its own + // measurement, so this runs as a `KNOWN_GAPS` expected failure instead of sitting unrecorded. + prependToEveryCatch( + "dead-conjunction-instanceof-if", + "preserving", + "splice if (e instanceof Error && false) { return null; } into every catch", + (e) => `if (${e} instanceof Error && false) { return null; }` + ), // A finally that leaves itself by `break` cancels the try's completion, so nothing hosted in // that tryBlock ever escapes the clause: the whole statement is a no-op. The walk's // catchless-try entry credited it anyway, minting a branch from the hosted classifier on 80 @@ -1087,6 +1115,8 @@ export const ADDITIVE_IDS = [ "wrap-body-in-rethrow", "wrap-body-in-same-arms-throw-ternary", "empty-instanceof-if", + "dead-armed-instanceof-if", + "dead-conjunction-instanceof-if", "dead-classifier-one-arm", "dead-throw-in-cancelled-try", "dead-deciding-map", diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 32f9d224d7e..458fab79ac4 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -759,7 +759,7 @@ describe("scanFile: catch clause evidence", () => { }); // The mirror of the family above. Each dead spelling earns nothing, and it must also COST - // nothing: `containsExit` was true of the dead statement itself, so prepending one raised the + // nothing: a plain containment read was true of the dead statement itself, so prepending one raised the // `exited` flag and blinded the walk to the real classification below it, turning a pass into a // swallow verdict on 78 real routes. `containsLiveExit` folds the literal guard and sees no live // exit, so the deciding statements keep their credit. The spellings are the CORPUS spellings @@ -2470,6 +2470,88 @@ describe("a ternary on the error has to send its arms somewhere different", () = }); }); +// The liveness gap in the branch predicate. `selectsADistinctPath` asked a plain containment +// question, so an arm holding an exit that can never run read as an arm that takes the error +// somewhere. `catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, +// { status: 500 }); }` is the same swallow as the clause without the `if`, and it was worth 50 +// points a route. The same eleven dead spellings had already been folded out of +// `catchClauseEvidence`'s `exited` flag by `containsLiveExit`, and this predicate beside it kept +// the containment read. `dead-armed-instanceof-if` in the mutation corpus is the tree-scale +// version: global 19 -> 27 and 80 routes raised, before the fix. +describe("an arm whose only exit is dead decides nothing", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return json({ error: "generic" }, { status: 500 }); + } + } + `; + + it("reads the unmutated clause as a swallow, as a baseline", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The reported shape, plus three further spellings of the same no-op reaching the same + // predicate by different routes: a dead loop body, a dead else arm, and a switch clause. + const DEAD_ARMS: Array<[string, string]> = [ + ["an if (false) arm", "if (error instanceof Error) { if (false) { return null; } }"], + ["a while (false) body", "if (error instanceof Error) { while (false) { throw error; } }"], + [ + "a dead else arm beside an arm that goes nowhere", + "if (error instanceof Error) { doThing(); } else { for (const k in {}) { return null; } }", + ], + [ + "a switch clause whose exit is dead", + "switch (error.code) { case 'x': if (1 === 2) { throw error; } }", + ], + ]; + + for (const [label, mutation] of DEAD_ARMS) { + it(`does not credit ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // The positive controls. The fold is subtractive against containment, so anything it cannot + // prove dead reads exactly as it did, including a guard whose truth is not decidable from the + // token alone. Without these the fix could be "always false" and the four cases above would + // still pass. + // + // `an arm guarded by a condition that does not fold` is also the pin on the alternative that was + // measured and rejected: asking the arm to `definitelyExits` rather than to hold a live exit. + // That is the "guaranteed" reading, it refuses all four shapes above, and it accuses + // `admin.api.v1.orgs.$organizationId.environments.staging.ts` on the real tree, taking the global + // from 19 to 18. That clause recognises Prisma's P2002, re-reads the conflicting row and returns + // `{ status: "updated" }`, rethrowing everything else: a textbook classification whose arm + // happens to fall through to the rethrow when the re-read finds nothing. Accusing it of taking + // one way out regardless of what was thrown is simply false, and a new false accusation is the + // direction that gets the tool switched off. + const LIVE_ARMS: Array<[string, string]> = [ + ["a plain returning arm", "if (error instanceof Error) { return badRequest(); }"], + [ + "an arm guarded by a condition that does not fold", + "if (error instanceof Error) { if (error.code === 'P2002') { return conflict(); } }", + ], + [ + "a live exit in the else arm only", + "if (error instanceof Error) { doThing(); } else { return badRequest(); }", + ], + ["a switch clause that returns", "switch (error.code) { case 'P2002': return conflict(); }"], + ]; + + for (const [label, mutation] of LIVE_ARMS) { + it(`still credits ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } +}); + // C4a. `export const { action, loader } = createActionApiRoute(...)` produced no entry point at // all: `scanFile` skipped a non-identifier binding name at the export site, so the route was // absent from the denominator rather than parsed, failed or unmeasured. The two-step spelling diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 2067e60203a..78b48d54e08 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -650,15 +650,6 @@ function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolea return ts.forEachChild(node, (child) => containsEscapingJump(child, jumps)) === true; } -/** Whether the tree rooted at `node` contains a `return` or a `throw` of its own, not counting one - * inside a nested function. What separates an arm that takes the error somewhere from an arm that - * runs and falls back into the clause's single common exit. */ -function containsExit(node: ts.Node): boolean { - if (ts.isFunctionLike(node)) return false; - if (ts.isReturnStatement(node) || ts.isThrowStatement(node)) return true; - return ts.forEachChild(node, containsExit) === true; -} - /** * Literal truthiness of a guard expression: true, false, or null when not decidable from the * token alone. Only literal tokens fold; an identifier, call, bigint, `&&`, `||` or a template @@ -713,13 +704,25 @@ function tryBlockMayThrow(block: ts.Block): boolean { } /** - * `containsExit`, minus exits that sit in a provably-untaken branch. `if (false) { throw e; }` - * contains an exit and can never run one; treating it as an exit is what let a dead statement - * blind the walk to the real classification below it, prepending one to a deciding clause turned - * its pass into a swallow verdict on 78 real routes. Folds literal guards only, so an unknown - * condition keeps the containsExit answer, which is the direction that refuses credit rather than - * inventing it. The mirror twins under `dead and deferred code prepended to a deciding catch does - * not blind it` hold the recovered half; the `BRANCH_EXITED` family holds the refusing half. + * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch + * does not already rule out. A plain containment walk, minus the hits it can prove never run: + * `if (false) { throw e; }` contains a throw and can never run one. + * + * Folds literal guards only, so wherever `literalTruth` cannot decide, every hit the plain walk + * would have found is still found. That makes this strictly subtractive against containment, which + * is what lets both of its callers read it for opposite purposes: + * + * - `catchClauseEvidence`'s `exited` flag, where a hit BLINDS the walk to whatever follows. + * Containment blinded it on a dead statement, so prepending one to a deciding clause turned its + * pass into a swallow verdict on 78 real routes. Subtracting dead hits only ever un-blinds. + * - `selectsADistinctPath`, where a hit GRANTS a branch. Containment granted one for an arm whose + * only exit was dead, which is `dead-armed-instanceof-if` in the mutation corpus, measured at 80 + * routes and the tree from 19 to 27. Subtracting dead hits only ever withholds. + * + * The `exited` half is pinned by the mirror twins under `dead and deferred code prepended to a + * deciding catch does not blind it` (recovered) and the `BRANCH_EXITED` family (refusing). The + * `selectsADistinctPath` half is pinned by `an arm whose only exit is dead decides nothing` and its + * siblings, plus the corpus entry. */ function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean { const walk = (node: ts.Node): boolean => { @@ -809,6 +812,16 @@ function containsLiveReturn(node: ts.Node): boolean { * An `if`/`else` whose two arms are textually identical does not count, the same comparison * `selectsAnErrorPath` makes of a ternary's arms. * + * The exit an arm is credited for has to be a LIVE one, `containsLiveExit` and never a plain + * containment read. `if (e instanceof Error) { if (false) { return null; } }` contains an exit that + * can never run, so under containment it read as a real decision and took a swallowing catch to a + * pass for the price of a mechanical edit: 80 routes and the tree from 19 to 27 when measured. The + * same liveness rule had already been put on `catchClauseEvidence`'s `exited` flag, for the same + * eleven dead spellings, and this predicate beside it kept the containment read. `an arm whose only + * exit is dead decides nothing` and its siblings are the unit pins; `dead-armed-instanceof-if` in + * the mutation corpus is the tree-scale version. Being subtractive against containment, the fold + * can only ever withhold a branch, never invent one, so a live arm reads exactly as it did. + * * The residual both branch tests share, stated here once for both: two arms that produce the same * outcome by different spellings still read as a real decision. * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides @@ -822,11 +835,17 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b const otherwise = statement.elseStatement; if (otherwise !== undefined) { if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; - return containsExit(statement.thenStatement) || containsExit(otherwise); + return containsLiveExit(statement.thenStatement) || containsLiveExit(otherwise); } - return containsExit(statement.thenStatement); + return containsLiveExit(statement.thenStatement); } - return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsExit)); + // Per clause statement rather than over the whole switch, so a live exit in any clause counts + // whatever the discriminant is. Reading the switch as one node would hand `containsLiveWhere`'s + // discriminant fold a `switch (e.code)` it cannot decide, which changes nothing, and a + // `switch (1)` it can, which is not this predicate's business: an unreachable CLAUSE is caught + // by the same fold one level down, and the statement is only reached at all when its condition + // references the caught binding. + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsLiveExit)); } /** @@ -893,7 +912,7 @@ function catchClauseEvidence(clause: ts.CatchClause): { // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written // after something that already exited`. // - // Raised off `containsLiveExit`, never `containsExit`. The containment read is true of + // Raised off `containsLiveExit`, never a plain containment read. Containment is true of // `if (false) { throw e; }` itself, so a provably dead statement raised the flag and blinded the // walk to the real classification below it: prepending one to a deciding clause turned its pass // into a swallow verdict on 78 real routes, the same false accusation for all eleven dead From 042c9c372edce638387971065abe00827de589c9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 13:27:40 +0100 Subject: [PATCH 107/117] fix(observability-map): reconcile a comment the paths no longer reach The report workflow carried a `paths:` filter, and GitHub evaluates one of those per workflow, so a pull request whose diff stopped matching never started the workflow at all: the resolved state could not fire and a comment from an earlier push stood for ever showing findings that had left the diff. Verified on a throwaway pull request whose only route change was reverted. The case that matters is the one touching a route and other files whose author reverts only the route change, which still has a diff and still does not match. The workflow now runs on every pull request and the gating is internal. The cheap path-detection job also looks for the marker comment, so the report job starts only when the watched paths moved or that comment already exists, and in the second case it reconciles the comment to its resolved state without scanning anything. A pull request with neither pays for that one cheap job. The lookup moving there also retires the sentinel pair the render and upsert steps shared: the job cannot start unless the lookup finished cleanly, and both steps read the id from one job output. Every comment now names the head commit it was rendered for, as a link to the pull request's compare range, because a sticky comment edited in place across pushes otherwise says nothing about which push it reflects. The sha and the URL are passed through the CLI as data, so the renderers stay pure and a local run without them still renders. --- .github/workflows/observability-map.yml | 195 ++++++++++++------ internal-packages/observability-map/README.md | 15 +- .../observability-map/src/integration.test.ts | 154 ++++++++++---- .../src/report/prComment.test.ts | 57 ++++- .../observability-map/src/report/prComment.ts | 45 +++- .../src/report/prCommentCli.test.ts | 66 ++++++ .../src/report/prCommentCli.ts | 49 ++++- 7 files changed, 463 insertions(+), 118 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 6043ce15976..a2545ad1a63 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -1,11 +1,15 @@ name: 🗺️ Observability Map on: + # No paths filter, deliberately. GitHub evaluates one per workflow, so a pull request whose diff + # stops matching does not start the workflow at all: the resolved state cannot fire and a comment + # from an earlier push stands for ever showing findings that are no longer in the diff. Verified on + # a throwaway pull request whose only route change was reverted, and the realistic case is worse + # than that empty diff, because a pull request touching a route and other files, whose author + # reverts the route change and keeps the rest, still has a non-empty diff that no longer matches. + # The gating moved into the jobs below instead, where it can read whether a comment exists. pull_request: types: [opened, synchronize, reopened] - paths: - - "apps/webapp/app/routes/**" - - "internal-packages/observability-map/**" # The corpus job below is gated to this package's own paths, so a scheduled run is what still # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a # known laundering shape start paying, but that is a property of the tree accumulating, not of any @@ -22,17 +26,34 @@ permissions: contents: read jobs: - # The workflow's paths filter is the union of what the two jobs below want, because GitHub - # evaluates it once per workflow. This narrows it again for the corpus job alone. + # The whole cost of a pull request that touches nothing this workflow watches: a checkout, a paths + # filter and one comment lookup. Everything expensive is gated on this job's outputs, and the + # lookup is here rather than in the report job so that gate can read it and the report job need + # never start. changes: - name: 🔍 Which paths moved + name: 🔍 What moved # Only the pull request path reads this job's output. On a schedule the action has no base to # diff, warns that `before` is missing and reports the files in the last commit on main, which # nothing then consults. Skipping it there keeps the nightly off a job it does not need. if: github.event_name == 'pull_request' runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + # Reading the pull request's comments, to find one an earlier push left. Read only: the write + # stays on the report job, which is the only job that posts. + pull-requests: read outputs: + # The corpus job's gate. Narrower than the report's on purpose: what the corpus measures is + # the tool's resistance to laundering, which only an edit to the tool can weaken. package: ${{ steps.filter.outputs.package }} + # The report job's gate, the union: a route change moves the report as well. + report: ${{ steps.filter.outputs.package == 'true' || steps.filter.outputs.routes == 'true' }} + # The id of a marker comment an earlier push left, empty if there is none, and the one source + # both the render and upsert steps read it from. + comment: ${{ steps.comment.outputs.id }} + # Set only by a lookup that finished cleanly, so anything else, retries exhausted or the step + # dying somewhere unforeseen, reads as "do not touch this pull request's comments". + lookup: ${{ steps.comment.outputs.ok }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -45,6 +66,54 @@ jobs: package: - 'internal-packages/observability-map/**' - '.github/workflows/observability-map.yml' + routes: + - 'apps/webapp/app/routes/**' + + # Looked up here because the report job's gate needs it: with the watched paths unmoved, a + # pull request that already has a comment gets a resolved state rather than being left with + # findings that no longer exist, and one that does not gets no job at all. + # + # On a failure that outlasts the retries this reports nothing, and the report job's gate reads + # that as "post nothing this run". Guessing is worse than silence: this step is the only thing + # that knows which comment to PATCH, so a guess of "no comment exists" POSTs, which either + # adds a second marker comment beside the stale one or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. + - name: 🔍 Look for a comment from an earlier push + id: comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + found="" + ok="" + for attempt in 1 2 3; do + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 + fi + { + echo "id=$(printf '%s\n' "$found" | awk 'NF { print $1; exit }')" + echo "ok=ok" + } >> "$GITHUB_OUTPUT" # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, # asserting the score does not rise. Roughly four and a half minutes for 45 entries, which is why @@ -114,6 +183,7 @@ jobs: # workflow the all-checks aggregate can see, so a job in this file would report a result nobody # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter. report: + needs: changes runs-on: warp-ubuntu-latest-x64-4x # Only this job comments, so only this job gets the write. permissions: @@ -122,9 +192,25 @@ jobs: # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. # The event test is what keeps this job off the nightly, which has no pull request to comment on # and only exists for the corpus job above. + # + # The two output tests are what the workflow-level paths filter used to do, plus the thing it + # could not do. The report has to run when the watched paths moved, and ALSO when they did not + # but a marker comment is already on the pull request, because that comment is the one showing + # findings that have left the diff. Reconciling it needs no scan, so the steps below are gated + # again on the same output. + # + # `needs` carries an implicit success() and that is wanted here: a `changes` job that failed + # knows neither which paths moved nor whether a comment exists, and a report job that ran anyway + # could only guess. Same reason the lookup test is positive rather than a check for a failure + # sentinel: retries exhausted, or the lookup step dying anywhere unforeseen, both leave the + # output unset and both mean the same thing, so neither can be read as "no comment exists" by + # one step and "a comment exists" by another. That disagreement is what the sentinel pair this + # replaces got wrong once already. if: >- github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository + github.event.pull_request.head.repo.full_name == github.repository && + needs.changes.outputs.lookup == 'ok' && + (needs.changes.outputs.report == 'true' || needs.changes.outputs.comment != '') steps: - name: ⬇️ Checkout repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -163,7 +249,13 @@ jobs: # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner # that exits 0 without writing takes the else branch and the stale-report comment, instead of # failing the `mv` and turning the job red. + # + # Gated: this is the expensive half, and the reconcile run has nothing to compare. The steps + # above it are not gated because the renderer is TypeScript in this repo, so reconciling still + # needs the checkout and the install. That is the cost of the reconcile run and it is paid only + # by a pull request that has a comment and no longer matches the paths. - name: 🔎 Scan head + if: needs.changes.outputs.report == 'true' run: | if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then @@ -180,6 +272,7 @@ jobs: # request's own work. A merge base would leave the intervening base-branch commits in the head # tree and out of the base tree, and blame the pull request for all of them. - name: 🔎 Scan base with the head's scanner + if: needs.changes.outputs.report == 'true' run: | if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ @@ -191,59 +284,26 @@ jobs: echo "base scan failed or the worktree could not be added; falling back to no base" >&2 fi - # Looked up before the render step because the render decision needs it: with no delta to - # report, a pull request that already has a comment gets a resolved state rather than being - # left with findings that no longer exist, and one that does not gets nothing at all. The - # upsert step reuses the id rather than asking twice. - # - # On a failure that outlasts the retries, both steps below do nothing. Guessing is worse than - # silence here: this step is the only thing that knows which comment to PATCH, so a guess of - # "a comment exists" still reaches an upsert with no id to patch, which POSTs. That either - # adds a second marker comment beside the stale one, or says "the findings an earlier push - # reported are gone" on a pull request that never had findings. Worst case now is no comment - # this run, which the next push fixes. - - name: 🔍 Look for a comment from an earlier push - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - rm -f /tmp/existing-comment-id /tmp/comment-lookup-failed - found="" - ok="" - for attempt in 1 2 3; do - if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then - ok=1 - break - fi - echo "comment lookup attempt ${attempt} failed" >&2 - sleep $((attempt * 5)) - done - - if [ -z "$ok" ]; then - touch /tmp/comment-lookup-failed - echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 - exit 0 - fi - - # --paginate runs the jq once per page, so a marker comment on more than one page yields - # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under - # continue-on-error. The oldest wins: it is the one the upsert has been updating. - count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) - if [ "$count" -gt 1 ]; then - echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 - fi - printf '%s\n' "$found" | awk 'NF { print $1; exit }' > /tmp/existing-comment-id - # continue-on-error for the same reason as the scan: a rendering bug must not turn the job # red. An empty /tmp/comment.md means there is nothing to post, which is a decision # prCommentCli makes, not this shell. + # + # Both shas are forwarded so every comment this job posts says which commit it was rendered + # for, which a sticky comment edited in place across pushes otherwise never tells you. They go + # through the CLI as data: the renderer builds no URL and reads no environment. - name: 📝 Render comment continue-on-error: true + env: + SCANNED: ${{ needs.changes.outputs.report }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + COMPARE_URL: ${{ github.server_url }}/${{ github.repository }}/compare/${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} run: | rm -f /tmp/comment.md - render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts "$@"; } + render() { + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts \ + --commit-sha="$HEAD_SHA" --commit-url="$COMPARE_URL" "$@" + } # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte # comment.md for the upsert to skip in silence. @@ -256,8 +316,12 @@ jobs: return 1 } - if [ -f /tmp/comment-lookup-failed ]; then - echo "the comment lookup failed, so this run posts nothing" >&2 + # Nothing this workflow watches moved, so nothing was scanned and there is no delta to + # compute. The job's gate only lets that case through when a comment from an earlier push + # is on the pull request, so there is exactly one thing left to say: what it shows is not + # in this diff any more. + if [ "$SCANNED" != "true" ]; then + emit --resolved || echo "could not render the resolved comment" >&2 exit 0 fi @@ -272,7 +336,7 @@ jobs: fi flags=() - if [ -s /tmp/existing-comment-id ]; then + if [ -n "$EXISTING_COMMENT" ]; then flags=(--existing-comment) fi @@ -283,28 +347,23 @@ jobs: # continue-on-error for the same reason: a transient gh api failure (rate limit, network) # must not fail the job either. Worst case, the PR gets no comment this run. + # + # The id comes from the same job output the render step read, so the two cannot disagree about + # whether a comment exists. A lookup that did not finish cleanly never reaches either of them: + # the job's gate stops it. - name: 💬 Upsert PR comment continue-on-error: true env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} run: | - # The same sentinel the render step reads, so the two cannot disagree about what a failed - # lookup means. Without it this step reads a missing id as "no comment exists" and POSTs. - if [ -f /tmp/comment-lookup-failed ]; then - echo "the comment lookup failed, so this run posts nothing" - exit 0 - fi if [ ! -s /tmp/comment.md ]; then echo "nothing to post: this pull request does not move the report" exit 0 fi - existing="" - if [ -f /tmp/existing-comment-id ]; then - existing=$(cat /tmp/existing-comment-id) - fi - if [ -n "$existing" ]; then - gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" -F body=@/tmp/comment.md + if [ -n "$EXISTING_COMMENT" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING_COMMENT}" -F body=@/tmp/comment.md else gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md fi diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index c428c2741a9..a4dca165d7e 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -29,10 +29,21 @@ prefix matches more than one route rather than silently taking the first`). ## CI A PR that touches `apps/webapp/app/routes` or this package gets a sticky comment scanning head -against the tip of the base branch, with the score, what changed, and the current fix list. It is -report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a +against the tip of the base branch, with the score, what changed, and the current fix list. Every +comment names the head commit it was rendered for, as a link to the PR's compare range, because the +comment is edited in place across pushes and otherwise says nothing about which push it reflects. It +is report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a later phase decides to add one. See `.github/workflows/observability-map.yml`. +The workflow itself runs on every PR, and the paths above are a gate inside it rather than a `paths:` +filter on the trigger. GitHub evaluates one of those per workflow, so a PR whose diff stops matching +does not start the workflow at all, and the comment an earlier push left then stands for ever showing +findings that are no longer in the diff. The case that matters is not a PR reverted to nothing: it is +one touching a route and other files whose author reverts the route change and keeps the rest, which +still has a diff and still does not match. So a PR with a comment and nothing left to compare gets +the comment reconciled to its resolved state without scanning anything, and a PR with neither pays +for one cheap job that reads the paths and looks for a comment. + This paragraph used to say "merge base", and two reviewers read that against `github.event.pull_request.base.sha` and reported the workflow as the thing that was wrong. It is the other way round. `actions/checkout` on a `pull_request` event checks out GitHub's test merge diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 99b6f5ed851..978d7964488 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -91,6 +91,31 @@ describe("the package it advertises", () => { }); }); +const WORKFLOWS = resolve(__dirname, "../../../.github/workflows"); +const REPORT = resolve(WORKFLOWS, "observability-map.yml"); + +function read(path: string): string { + if (!existsSync(path)) throw new Error(`workflow is missing: ${path}`); + return readFileSync(path, "utf8"); +} + +/** Comment lines dropped, for an assertion about what the YAML says rather than what its prose + * happens to mention. */ +const withoutComments = (text: string) => text.replace(/^\s*#.*$/gm, ""); + +/** One job's block, from its key to the next key at job indent. */ +function job(name: string): string { + const parts = read(REPORT).split(new RegExp(`^ {2}${name}:$`, "m")); + expect(parts).toHaveLength(2); + return parts[1]!.split(/^ {2}[a-z][a-z-]*:$/m)[0]!; +} + +/** A job's condition and everything else it declares before its steps. */ +const gate = (name: string) => job(name).split(" steps:")[0]!; + +/** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ +const steps = (block: string) => block.split(/^ {6}- name: /m).slice(1); + /** * The one thing the docstring checker cannot reach. It walks `src/` only, so workflow prose is * unpoliced, and the C1 defect was exactly that: two steps disagreeing about what a missing @@ -99,32 +124,100 @@ describe("the package it advertises", () => { * POSTed, so a transient lookup failure either added a second marker comment beside the stale one * or announced that findings were gone on a pull request that never had any. * - * This is a text check over the workflow, not a parse of its semantics, so it catches one shape of - * that class and no other. Named as such rather than sold as coverage of the file. + * The sentinel pair those two shared is gone with the lookup, which moved into the `changes` job so + * the report job's own gate could read it. What replaces it is structural rather than agreed: the + * report job does not start unless the lookup finished cleanly, and the id it then uses is one job + * output that both steps read, so there is nothing left for two readers to disagree about. + * + * These are text checks over the workflow, not a parse of its semantics, so they catch the wiring + * coming apart and nothing about whether GitHub agrees. Named as such rather than sold as coverage + * of the file. */ -describe("the report workflow's two readers of the comment lookup", () => { - const WORKFLOW = resolve(__dirname, "../../../.github/workflows/observability-map.yml"); - - /** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ - function steps(): string[] { - if (!existsSync(WORKFLOW)) throw new Error(`the report workflow is missing: ${WORKFLOW}`); - const text = readFileSync(WORKFLOW, "utf8"); - return text.split(/^ {6}- name: /m).slice(1); - } +describe("the report workflow's one source of the comment id", () => { + it("does not start the report job at all unless the lookup finished cleanly", () => { + expect(gate("report")).toContain("needs.changes.outputs.lookup == 'ok'"); + }); - it("both honour the same sentinel, so a failed lookup cannot mean two things", () => { - const readers = steps().filter((step) => step.includes("/tmp/existing-comment-id")); + it("gives the render and the upsert step the same output to read", () => { + const readers = steps(job("report")).filter((step) => step.includes("EXISTING_COMMENT")); expect(readers.length).toBeGreaterThanOrEqual(2); - expect(readers.filter((step) => !step.includes("/tmp/comment-lookup-failed"))).toEqual([]); + expect( + readers.filter( + (step) => !step.includes("EXISTING_COMMENT: ${{ needs.changes.outputs.comment }}") + ) + ).toEqual([]); + }); + + // The lookup is what lets the report job be gated, so it has to happen before it, in the job that + // an unrelated pull request pays for anyway. + it("looks the comment up in the cheap job and not again in the report job", () => { + expect(job("changes")).toContain("issues/${PR_NUMBER}/comments"); + expect(job("changes")).toContain("pull-requests: read"); + expect(job("report")).not.toContain("--paginate"); }); it("takes one id from a lookup that paginates rather than passing every line on", () => { - const lookup = steps().find((step) => step.includes('startswith(""; +/** + * The commit a comment was rendered for. Data rather than something the renderers read for + * themselves: they stay pure, and a unit test or a local CLI run with no commit context renders the + * same comment without the line. + */ +export type CommitContext = { + /** The head sha, full. Shortened for the link text here rather than by the caller. */ + sha: string; + /** Compare URL for the pull request's range, base to head. */ + url: string; +}; + +const SHORT_SHA_LENGTH = 7; + +/** + * Directly under the heading and before anything the report says, because the comment is sticky: + * it is edited in place across pushes, so the first question about it is which push it reflects. + */ +function commitLines(commit: CommitContext | undefined): string[] { + if (!commit) return []; + return [`As of [\`${commit.sha.slice(0, SHORT_SHA_LENGTH)}\`](${commit.url}).`, ""]; +} + const MAX_CHANGED_ROWS = 15; /** @@ -272,12 +295,13 @@ export function hasDelta(head: MapReport, base: MapReport | null): boolean { * What replaces a comment whose findings a later push fixed. Going silent would leave the earlier * comment standing with findings that no longer exist, which is worse than a redundant comment. */ -export function renderResolvedComment(): string { +export function renderResolvedComment(commit?: CommitContext): string { return [ MARKER, "", "## Observability map", "", + ...commitLines(commit), "Nothing in this pull request moves the report any more. The findings an earlier push " + "reported are gone.", "", @@ -291,12 +315,13 @@ export function renderResolvedComment(): string { * a job that must never block a pull request, and the alternative to that was swallowing the * failure so the only signal was a comment that never appeared. */ -export function renderScanFailedComment(): string { +export function renderScanFailedComment(commit?: CommitContext): string { return [ MARKER, "", "## Observability map", "", + ...commitLines(commit), "The scan failed for this run, so there is no report. Anything above is from an earlier push " + "and is stale. The workflow log has the error.", "", @@ -309,8 +334,20 @@ export function renderScanFailedComment(): string { * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the * two by `fileName`, the same identifier `renderJson` carries. */ -export function renderPrComment(head: MapReport, base: MapReport | null): string { - const lines = [MARKER, "", "## Observability map", "", scoreLine(head, base), ""]; +export function renderPrComment( + head: MapReport, + base: MapReport | null, + commit?: CommitContext +): string { + const lines = [ + MARKER, + "", + "## Observability map", + "", + ...commitLines(commit), + scoreLine(head, base), + "", + ]; lines.push(...whatChangedSection(head, base), ""); lines.push(...fixFirstSection(head), ""); diff --git a/internal-packages/observability-map/src/report/prCommentCli.test.ts b/internal-packages/observability-map/src/report/prCommentCli.test.ts index 31ddfeeecb3..6160fa11b9b 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.test.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.test.ts @@ -87,6 +87,72 @@ describe("prCommentCli", () => { expect(r.out).toContain("The scan failed for this run"); }); + // The reconcile path. A pull request whose diff stops matching the paths the workflow watches + // scans nothing, so there are no reports to compare and no delta to compute, and the comment an + // earlier push left still shows findings that have gone. This is how the workflow says so. + it("prints the resolved comment for --resolved without reading any file", () => { + const r = run("--resolved"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + describe("the commit the comment is rendered for", () => { + const sha = "0123456789abcdef0123456789abcdef01234567"; + const url = "https://github.com/triggerdotdev/trigger.dev/compare/1111111...2222222"; + const stamp = `As of [\`0123456\`](${url}).`; + + it("stamps the report comment from the two commit flags", () => { + const r = run(headPath, basePath, `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the resolved comment the reconcile path posts", () => { + const r = run("--resolved", `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the resolved comment the delta path posts when the delta has gone", () => { + const r = run( + unchangedPath, + unchangedPath, + "--existing-comment", + `--commit-sha=${sha}`, + `--commit-url=${url}` + ); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the stale-report comment", () => { + const r = run("--scan-failed", `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + // Half a pair can only come from an edit to the workflow, and a comment quietly missing the + // line it was supposed to gain is the failure nobody would ever notice. + it("exits 1 rather than dropping the stamp when only one of the two flags is given", () => { + for (const half of [`--commit-sha=${sha}`, `--commit-url=${url}`]) { + const r = run(headPath, basePath, half); + expect(r.code).toBe(1); + expect(r.err).toContain("have to be given together"); + expect(r.out).toBe(""); + } + }); + + // What an unset workflow expression interpolates to. Both empty is the local run. + it("reads an empty flag value as no commit context rather than as half a pair", () => { + const r = run(headPath, basePath, "--commit-sha=", "--commit-url="); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("As of ["); + }); + }); + it("treats '-' as no base", () => { const r = run(headPath, "-"); expect(r.code).toBe(0); diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index a2aef9a3477..9e7905388b4 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -7,6 +7,7 @@ import { renderPrComment, renderResolvedComment, renderScanFailedComment, + type CommitContext, } from "./prComment.js"; /** Where output goes. Injectable so tests can read it without spawning a process. */ @@ -33,6 +34,27 @@ function readReport(path: string, label: string): MapReport { } } +/** An `--opt=value` argument, last one winning. An empty value reads as absent, since that is what + * an unset workflow expression interpolates to. */ +function flag(args: string[], name: string): string | undefined { + const prefix = `--${name}=`; + const values = args.filter((a) => a.startsWith(prefix)).map((a) => a.slice(prefix.length)); + return values.filter(Boolean).pop(); +} + +/** + * The commit the caller says this comment is for. Half a pair is rejected rather than dropped: it + * can only come from an edit to the workflow that passes one and not the other, and a comment + * silently missing the line it was supposed to gain is the failure nobody would notice. + */ +function commitFrom(args: string[]): CommitContext | undefined { + const sha = flag(args, "commit-sha"); + const url = flag(args, "commit-url"); + if (sha && url) return { sha, url }; + if (sha || url) throw new Error("--commit-sha and --commit-url have to be given together"); + return undefined; +} + /** * `-` or a missing second arg means no base: the CI job falls back to this when the base scan * itself failed, so the comment still renders rather than the job going red. @@ -43,18 +65,37 @@ function readReport(path: string, label: string): MapReport { * than left standing with findings that no longer exist. * * `--scan-failed` takes no report and prints the stale-report comment, for the case where the head - * scan produced nothing to read. + * scan produced nothing to read. `--resolved` takes no report either and prints the resolved state + * outright, for the case where the workflow knows there is nothing to compare: the paths the report + * watches did not move in this pull request at all, so nothing was scanned, and the comment an + * earlier push left has to stop showing findings that are no longer in the diff. + * + * `--commit-sha` and `--commit-url` are the commit every comment above is rendered as of. */ export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); const scanFailed = args.includes("--scan-failed"); + const resolved = args.includes("--resolved"); const existingComment = args.includes("--existing-comment"); const positional = args.filter((a) => !a.startsWith("--")); const headPath = positional[0]; const basePath = positional[1]; + let commit: CommitContext | undefined; + try { + commit = commitFrom(args); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } + if (scanFailed) { - io.out(`${renderScanFailedComment()}\n`); + io.out(`${renderScanFailedComment(commit)}\n`); + return 0; + } + + if (resolved) { + io.out(`${renderResolvedComment(commit)}\n`); return 0; } @@ -74,11 +115,11 @@ export function main(argv: string[], io: Io = processIo): number { } if (hasDelta(head, base)) { - io.out(`${renderPrComment(head, base)}\n`); + io.out(`${renderPrComment(head, base, commit)}\n`); return 0; } if (existingComment) { - io.out(`${renderResolvedComment()}\n`); + io.out(`${renderResolvedComment(commit)}\n`); } return 0; } From 420d2db499ea72546e8d09dc250e7c0550af7f64 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 14:52:35 +0100 Subject: [PATCH 108/117] docs(observability-map): move the inline design record into the README The package's comments carried most of the reasoning behind the tool: what was measured, which alternatives were rejected, and which residuals are still open. Relocates that into README sections rather than losing it, covering the evidence model, the dead-code defence, parse guards, the iteration-callback boundary, auth-scope's caller-id evidence, triviality, suppression parsing, the mutation harness, reporting, and how the suite is gated. --- internal-packages/observability-map/README.md | 529 ++++++++++++++++++ 1 file changed, 529 insertions(+) diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index a4dca165d7e..9b7f448706b 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -421,6 +421,535 @@ Read these before trusting a specific verdict. - **The score is a mean of means over a heuristic.** Read the fix list, the two headline figures and the CHECKS block. Watching the single number for small movements will mislead you. +## How the scanner reads a route + +`scan.ts` produces one `EntryPoint` per route module, carrying only body-scoped evidence. Three +rules decide what "the body" means, and every finding in this tool rests on them. + +**One hop, same file only.** A loader that delegates to a helper declared in the same file has that +helper's statements, try/catch and callees counted as its own. A helper's own helpers are not +followed, the visited set stops a cycle, and nothing imported from another module is ever opened. + +**Nested functions count as work.** A statement inside a callback written in the body is still a +statement the route runs. Leaving them out let `trace("x", async () => { whole body })` collapse a +route to one statement, which is inside the triviality limit, so every check reported +not-applicable for it. `wrap-body-in-trace` in the corpus is that shape. + +**Per export, not per file.** Six fields come in `loaderX`/`actionX` pairs, and the union is only +offered where the question itself is file-wide. This split is the fix for a whole family of false +passes, all the same shape: a file whose loader called `requireUser` and whose action called +nothing read as "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` +credited its hand-written action with the builder's authentication. `routeExports.ts` is the one +enumeration both per-export checks read, because `auth-scope` and `auth-boundary` each grew their +own `[loader, action]` literal and only one of them got each fix. + +`calleeNames` is the union and stays entry-point wide because the three questions that read it are: +`sensitivity.ts` asks what the file touches, `triviality.ts` asks how much the file does, +`audit-trail` asks whether the file records anything. There is deliberately no entry-point-wide +`checkedCallees`, so no check can reach for a union that would say a loader's reading of `getUser` +speaks for the action beside it. + +The split cannot drift from the union it came from: one push site in `scanFile` fills the whole-entry +list and each owning export's list, `scan.test.ts` pins the property on fixtures and +`integration.test.ts` pins it again over the real tree (`every callee name is attributed to an +export that exists`). + +Two fields exist because the bare callee name is not enough. `calleeName` keeps only the last +segment, so `prisma.organization.findFirst` arrives as `findFirst` with the receiver that says what +is being called gone; `calleeTexts` keeps the whole dotted path, which is how the per-export +triviality rule knows a three-statement body reaches the datastore. `auth-boundary` matches the bare +name on purpose, so a guard call cannot be hidden by its receiver. + +### Catch evidence, per clause + +`CatchEvidence` is one record per catch clause rather than a set of booleans per entry point, +because 39 routes have more than one catch and 17 mix a narrow parse guard with a broad handler. +Under the old aggregate booleans a single well-behaved catch spoke for the swallow next to it. + +- `rethrows`: throwing is the clause's only way out, i.e. a throw is reached on the clause's + guaranteed path AND the clause contains no live `return` anywhere. +- `throws`: a throw is reached on that path, whether or not it is the only way out. Kept separately + so a verdict can say what is true of a clause that both throws and returns. The detail line "takes + one way out regardless of what was thrown" is only true of a clause that never throws, and saying + it of a clause that does was a false accusation on 16 clauses in the tree. +- `branches`: the clause picks what to do from what it caught. An `if` or `switch` whose condition + references the caught binding and at least one of whose arms returns or throws, or a conditional + that is the whole value of a `return`/`throw`. `if (retries > 0)` does not count, + `if (e instanceof Error) { }` does not count, a bindingless `catch { }` cannot count at all, and an + `instanceof` used only to word a message does not count either, because every error still leaves by + the same path. +- `guardsParse`: the guarded region parses something. `JSON.parse`, `request.json()`, a zod + `parse`/`safeParse`, a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three + constructors are read as `ts.isNewExpression` because a `new` expression is not a call and the + call-callee scan never sees them. Any constructor at all would mean `new BranchesPresenter()` + excuses a catch guarding ordinary work, which was true of 77 try blocks in the tree. +- `guardCanRaise`: the region does anything that could reach the clause. False means `try { 0; }` and + little else, because any call counts, including one that cannot throw. +- `guardMayRaise`: the containment twin, false only when the region provably cannot raise. Everything + `canRaise`'s whitelist misses stays true here, so `guardCanRaise` implies `guardMayRaise`. +- `awaitsOnlyParse`: everything the region waits for is one of those parses, or a read of the body it + parses. +- `tryStatementCount`: statements in the guarded block, counted as `statementCount` counts them. + +`canRaise` is a whitelist and it misses real raising code, which is the safe direction but does +matter: a destructuring declaration (`const { a } = undefined` throws), a temporal-dead-zone read, a +coercion that raises, and a `delete` on a frozen object all read as unable to raise. That is why the +refused-swallow arm of `error-classification` reads the route's own deciding catches through +`guardMayRaise` and never through `guardCanRaise`. Ordering it off can-raise accused a route that +owns a real classifying catch of owning none, which was flatly untrue (`does not accuse a route that +owns a catch of owning none`). + +"Does this route catch anything" is `catches.length`, never `hasTryCatch`. A `try`/`finally` with no +catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error propagates +once the cleanup has run, and reading the old flag as a catch put +`admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. + +## The dead-code defence + +Both of the catch-clause answers are read off the clause's guaranteed path. The governing rule: the +walk may enter a construct exactly where the entered statements are guaranteed to execute whenever +the clause body runs, so no credit can ever come from code a semantics-preserving edit could have +added dead. + +Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` that has no catch +clause and whose finally contains no jump out of itself, the sole clause of a single-default +`switch`, the then-arm of an `if` whose condition is exactly the literal `true` keyword, and both +arms of an `if`/`else` with per-arm states merged by intersection. + +Not entered, deliberately: a bare `if` without an else, loops other than `do`, labelled statements, +function-like nodes, nested catch clauses, finally blocks, and the tryBlock of a `try` that has a +catch clause, where a throw is intercepted by the nested catch rather than escaping. + +That rule replaced a list of statically-false shapes an earlier round kept extending, and the list +was losing. `if (false)` and `while (false)` were recognised; `for (;false;)`, `if (true) {} else`, +`switch (1) { case 2: }`, `try {} catch`, `for (const x of [])`, `for (const k in {})`, `if ("")`, +`if (!true)` and `if (1 === 2)` were not, each worth 50 points a route. Asking for the throw to be +unconditional refuses all eleven without naming any of them. `dead-*` in the corpus is the +tree-scale proof, one entry per shape. + +`rethrows` asks for one thing more: no `return` anywhere in the clause. Without it a `throw error;` +written after a statement that already exited read as a rethrow, in seven spellings. +`dead-throw-after-*` in the corpus covers them. The cost is real and worth stating: +`catch (e) { if (transient) throw e; return null; }` no longer reads as a rethrow, so it fails rather +than sitting out. That is the direction to be wrong in, since the reverse hands out points. + +### Two folds, pointing opposite ways + +There are two literal folds in `scan.ts` and unifying them would be a bug. + +`containsLiveWhere` folds any literal guard `literalTruth` can decide, and it is strictly +subtractive against a plain containment read: wherever the truth cannot be decided, every hit +containment would have found is still found. That is what lets its two callers read it for opposite +purposes. In `catchClauseEvidence`'s `exited` flag a hit BLINDS the walk to whatever follows, and +containment blinded it on a provably dead statement, so prepending one to a deciding clause turned +its pass into a swallow verdict on 78 real routes. In `selectsADistinctPath` a hit GRANTS a branch, +and containment granted one for an arm whose only exit was dead: `dead-armed-instanceof-if`, measured +at 80 routes and the tree from 19 to 27. Subtracting dead hits only ever un-blinds in the first case +and only ever withholds in the second. + +The walk's own entry tickets fold nothing but the literal `true` keyword. `!!1`, `1` and `!false` are +deliberately not entry tickets, because entry GRANTS credit and a wrong grant pays, where +`literalTruth`'s wider folding only ever withholds blindness. Do not unify the two. + +`literalTruth` treats `&&`, `||`, an identifier, a call, a bigint and a template with substitutions +as undecidable on purpose, so a live guard can never be read as dead. The cost of that is +`dead-conjunction-instanceof-if`, a corpus expected failure: `e instanceof Error && false` both +references the caught binding and can never be true, and no fold in the file can see it. Widening the +fold is a different rule with its own measurement. + +The `exited` flag is raised at the END of each statement, after that statement's own branch check. A +deciding statement contains an exit by definition, so raising it first makes every such statement +refuse itself, measured at 78 routes losing their pass. This ordering leaves the real-tree report and +all 240 clauses' evidence byte-identical. + +### A finally that cancels the try + +A finally block that completes abruptly supersedes the try's and the catch's completion, so an exit +written in either never leaves the statement. Two places read that, in opposite directions. + +`catchClauseEvidence` refuses to enter a catchless try whose finally holds a jump out of itself, +because entry grants rethrow credit and the throw would never escape the clause. The refusal is a +containment read, and it is over-approximate on purpose: a jump that only may run still refuses +(`refuses the tryBlock when the finally only may break`). `dead-throw-in-cancelled-try` in the corpus +is the tree-scale shape, worth 80 routes and 8 global points when measured. + +`containsLiveWhere` then folds the same statement to its finally's own statements, so a refused +statement cannot blind the walk to the real classification below it (`keeps the classification after +a finally-break no-op`). A finally holding a `return` is covered by the explicit `containsLiveReturn` +read instead, because `try { throw e; } finally { return null; }` genuinely swallows. + +### The residual both branch tests share + +Two arms that produce the same outcome by different spellings still read as a real decision. +`if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides nothing, +and so does the `if` with no `else` whose arm returns what the statement after it returns. Telling +those apart needs the produced values compared for meaning rather than for text, which is a +different kind of analysis from anything else in the file. The textual comparison is the cheapest +thing that catches the copy-paste form, which is the one a mutation produces. + +## Parse guards, and the narrow-try count + +A catch clause counts as a parse guard, rather than as the route's error handling, when the try block +parses, waits for nothing except that parse, and is short. All three conditions are load bearing and +two of them are corrections. + +`awaitsOnlyParse` is what a statement count cannot express. +`try { const body = await request.json(); return await handleEverything(body); } catch { 500 }` is +two statements, one of them a parse, and the whole handler inside it: the count reads it as narrow +and it is the `otel.v1.logs.ts` swallow written compactly. Asking what the block waits for separates +them, and unlike the count it does not care how the statements are punctuated or how deeply the work +is nested. Awaiting is the signal rather than calling, because the calls that prepare a parse's input +are ordinary synchronous string work (`matchPattern.slice(4)` before a `new RegExp`), and requiring +every CALL to be a parse refuses four of the tree's clearest guards. + +Two residuals follow from awaiting being the signal, both in the round A fix 3 report. A block that +does its non-parse work synchronously still reads as a guard. And `guardedWork` looks for a +`ts.AwaitExpression`, which `for await (...)` and `await using` are not. Neither occurs in the tree +and neither is reachable by rewriting a real route, since both need work that is not there to begin +with. + +`NARROW_TRY_STATEMENTS` is 2, so the guarded operation can bind its result +(`const stripped = ...; new RegExp(stripped);`) and a third statement means the try has started to +cover the handler. The idiom it was hand-read against: 55 of 427 entry points, 11 of the failures at +the time, all eleven the deliberate `try { body = await request.json() } catch { 400 }` shape. + +It is an absolute count and not a ratio against the enclosing body, because a ratio is diluted by +anything else in the same body: padding the action with unrelated statements after the try relabelled +the same broad swallow as a narrow guard, moving the denominator without touching the clause. +`inert-statements-after-try` in the corpus is that shape. + +What the count is not is unpaddable, which an earlier docstring and a commit subject both claimed. +`countStatement` counts declarators and comma operands rather than semicolons, so +`const a = f(), b = g(), c = h();` is three and `a(), b(), c()` is three; that is what +`merge-declarations` and `merge-comma-expressions` check. A third way nobody has written down would +work, which is why the count is no longer the only condition and no longer the load-bearing one. + +Two rejected alternatives, both measured. Requiring the clause to answer with a 4xx credits, on its +own, 11 clauses guarding four to thirty statements, the widest swallows in the tree, including +`admin.api.v1.workers.ts`, whose 28-statement try answers every failure with a 400 carrying the +internal error message; added on top of the rest it costs three routes their pass, all three narrow +guards computing a fallback value rather than answering a request. And a narrow guard is not a way to +qualify as classification on its own: reading all eleven entry points that limb would clear found six +real swallows, including a silent run cancellation and two credential paths reporting a database +failure to the browser as a 400 with an internal message in it. + +## The iteration-callback boundary + +`items.map((item) => { try {...} })` is a fresh catch per element, so its clause is not the route's +own error handling. `trace(async () => {...})`, `mutateWithFallback({ pgMutation: ... })` and +`new ReadableStream({ start: ... })` all invoke their callback exactly once, so theirs is. The +structural signal is the method name, which is a list of eight, because nothing in a syntactic scan +can tell `users.map` from `Result.map`. + +Being wrong here is asymmetric, and the direction that used to pay no longer does. A refused catch is +kept WITH its evidence, built by the same machinery as an own catch, and judged on what it does +rather than on where it sits. A refused swallow fails the route whenever nothing the route owns +decides, and that arm is deliberately not conditioned on the route owning no catches, so an own inert +rethrow catch cannot lift a refused swallow out of the verdict. A route whose only catches are refused +and none of them swallows sits out at not-applicable and never passes, which is what keeps a prepended +dead deciding `.map` from minting a pass on the 261 catchless routes. `dead-deciding-map` holds that +at tree scale. + +That is what makes the name list survivable. `Result.map(...)` is a corpus entry that passes rather +than a hole: relocating a swallow behind the boundary still fails +(`still fails a swallow wrapped in a non-array receiver's .map(...)`), and relocating a decision earns +at most the route's exit from the denominator. A receiver that is an array literal of one element or +none is refused outright, since it cannot iterate. + +The other direction still costs precision. A per-item callback under a callee the list does not know, +`pMap(items, cb)` or `Array.prototype.map.call(items, cb)`, is attributed to the route, so a +per-element catch that decides can carry it to a pass. No mutation of a real route produces it: a +route has to already be iterating for the shape to exist. It is a wrong verdict waiting for a route +to be written that way rather than a laundering path, and it is why the list is worth extending when a +new iteration helper shows up in the tree. + +## What auth-scope reads as scoping + +Three conditions, and the first version had only the middle one, which made the check free to defeat. +Prepending `const __unused = { anything: user.id };` to every body raised `settings.sso` and +`settings.team`, the only two findings `auth-scope` has ever produced and both confirmed cross-org +exposures. `dead-caller-scope-object` and `dead-caller-scope-userid` are the two halves of that +shape. + +- The value has to be the caller's own id, anchored at both ends: the root is one of the auth + bindings a builder hands the handler and the last segment is an identity field, so `user.name` is + not a scope and neither is `run.userId`, which is a resource's owner. +- The property NAME has to be an identity field. Of the ten names that take a caller-id value in the + route tree, `sub`, `value` and `consumerId` are the three that are not, and `anything: user.id` is + what a mutation writes. +- The object has to be handed, through any depth of nesting, to a call that could narrow a read with + it. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. + +The third condition is a denylist of sinks rather than an allowlist of query callees, and that is a +measurement. 72 distinct callees are handed a caller id across the route tree, running from +`prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare +`regenerateApiKey`. No name pattern separates those from `sendToPlain`, so an allowlist would accuse +whichever route named its helper next, and a wrong accusation is the failure this check cannot +afford. The sinks refused are the log line and the response body, both of which take the very +`{ userId: user.id }` object a query filter takes: loggers account for 13 of the caller-id sites and +the two response serializers for 2 more. The shape is already in the tree rather than hypothetical, +in `engine.v1.dev.runs...attempts.start`, which logs `{ environmentId: ... }` beside the +`runStore.findRun` that earns its credit honestly. `log-caller-scope-userid` covers it at tree scale. + +A callee with no readable name of its own is credited, because refusing it would ACCUSE the route, +and under-crediting beats accusing a route that is fine. `String({ userId: user.id })` therefore +reads as scoping, the same way `try { String(0); }` reads as error handling and for the same reason. + +`authorization: undefined`, `null` and `false` are read as not declared, because +`apiBuilder.server.ts` gates every option behind `if (option)` and declaring one is what the check +credits. + +## Triviality, in detail + +Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder +wrapping it, and nothing in the calls or the hint text naming a datastore or a service. + +Both limits are 3 because both real shapes need three: parse the params, build a path, redirect, or +an environment guard and two returns. Allowing a fourth call admits +`_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls; allowing a fourth +statement admits the routes that authenticate and then hand off to a presenter, which have real work +behind them; allowing a fifth admits an admin route that calls a service and hand-rolls its own error +responses. + +The rule is deliberately reluctant, because a route wrongly called trivial is exempted and never +shows up in the report again. So `calleeNames` descends into the callee of every call at any depth +while `statementCount` stops at a nested function, which means the call count still catches bodies +the statement count reads as short. A builder means the config passed to it (`findResource`, +`authorization`) is work the scanner never walks, so the visible body is not the whole route. And a +try/catch is exactly what `error-classification` reads, so a body with one has an error path worth +reporting on however short it is. + +One rule, two views, so the entry-point-wide answer and a single export's answer cannot drift. The +per-export view exists because a per-export verdict judged against a file-wide triviality rule +accuses the wrong half of a file: `auth.github.ts` is +`export let loader = () => redirect("/login")` beside an action that calls +`authenticator.authenticate`, and the file-wide rule called it non-trivial because the ACTION is not, +so `auth-boundary` accused a one-line redirect stub of missing an auth guard. `checks/index.test.ts` +pins both directions (`reports not-applicable for a redirect-stub loader beside a guarded action` and +`fails an export whose own body does real work unguarded`). + +The two views differ in one term and the difference was measured both ways. The entry-point-wide view +matches the side-effect hints against the whole file, so an import of `prisma` disqualifies it even +when the query sits somewhere the scanner does not walk. A per-export view matches that export's own +callee PATHS instead. Reading the file's text into one export's verdict is the per-file-for-per-export +substitution the rule exists to damp, and it is defeatable: `log-caller-scope-userid` prepends a +`logger.error(...)` to every body, which with the term file-wide put the word `logger` in +`auth.github.ts` and turned its untouched one-line redirect loader from excused into accused. Emptying +the term instead is not the answer either: `calleeNames` keeps only a call's last segment, so +`prisma.orgMember.findMany` reads as `findMany` and a three-statement body that queries the datastore +matches no hint at all, which took five existing `auth-boundary` fixtures from `fail` to +`not-applicable`. The callee paths are body-scoped like the first option wants and name the receiver +like the second needs. + +## Reading the directive out of the source + +The suppression directive is read from a real parsed `ts.SourceFile`, and then filtered against the +spans the parser has already claimed as content. Both halves are needed. + +Parsing rather than scanning is what stops a template literal with a substitution being rescanned as +ordinary code after `${x}`, and what makes JSX text a node at all. Filtering by span is what stops +the two comment-range lexers reading the start of such a node as a comment anyway, which they do +because `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw lexers over source text from +an offset and consult no parse tree. A JSX text node that BEGINS with `//` or `/*` is the shape that +reached the real tree, in `resources.branches.create.tsx`'s `//`. + +The filter is on the range's start offset falling inside a claimed span, not on the gap between a +token's full start and its start. A gap filter was tried and rejected: it loses a same-line trailing +comment and a comment inside a JSX expression container, both of which are real. Both lexers are +called at every token boundary, because which one returns a given comment depends on whether it +shares a line with the token before it. + +Leaf tokens are walked through `.getChildren()` rather than `ts.forEachChild`, which skips bare +punctuation and keyword tokens. A comment can sit directly before one of those with nothing else +following it, the last line inside a block. + +The mutation corpus does not cover any of this and cannot: a suppression can only lower an entry's +score, because `scoreEntry` caps it at the pre-suppression ratio, so suppression bugs are invisible +to a harness that watches for the score rising. They need ordinary unit tests, which is what +`suppression.test.ts` is: `jsx text is content, not a comment` is the four cases that fail without +the JSX filter, and the positive control beside it, `still reads a directive from a comment in a JSX +expression container`, is what stops the filter being widened until it eats real comments. + +## The mutation harness + +Every mutation is a TEXT rewrite driven by AST positions, never a reprint. A reprint would change +formatting everywhere and make a failure impossible to read; splicing at node positions leaves the +rest of the file byte-identical, so a corpus failure can be diffed down to the one construct that +moved. Overlapping edits are dropped inner-first, which is what "the outer rewrite won" means. + +Neither kind of entry is ever executed. Semantics-preserving here means preserving the observable +behaviour of the route as written, which is what the scanner claims to measure. It is not a claim +that the mutated tree compiles against its real types. + +**Splices go at the HEAD of a catch clause, not the tail.** 234 of the tree's 260 clauses end in a +`return` or a `throw`, so an appended shape was dead by ordering before the rule under test ever +looked at it: eleven entries reported touching 172 files while exercising 26 clauses. At the head +every clause is reachable. The shapes spliced this way are dead wherever they sit, so moving them +does not make the rewrite any less preserving. + +**The harness's population is asserted against the scanner's.** `entryBodies` read two of the four +export forms `scan.ts` reads, missing `export const { action, loader } = builder(...)`, +`const { action } = builder(...); export { action };` and `export const action = route.action`, which +is 36 of the tree's entry points. No assertion could notice, because a mutation reaching fewer routes +lowers the score rather than raising it. `wraps a body in every non-delegating entry point the scanner +finds` pins it now, with `admin.tsx` the one named exclusion: its handler is a concise arrow with no +block for a block wrapper to wrap. + +The same failure mode is why the registry assertion and the additive-class assertion are ungated +while everything else in the file needs `OBS_MAP_MUTATION_CORPUS=1`. `auth-scope` was added a round +after `suppress-every-check` was written and never added to its directive list, so the suppression +invariant went untested at tree scale for 19 routes while the entry's description said "every +check". Omitting a check from a sweep leaves its failures in place, which lowers the score, so the +corpus cannot catch its own omission by failing. + +**The corpus deliberately disagrees with the scanner about where a handler sits.** `mutations.ts` +keeps its own copy of the builder handler shapes rather than importing them, because sharing the +scanner's notion would let a bug in that notion hide a laundering shape. Which exports exist is not +a judgement, though, and there the harness was simply behind, which is the distinction above. + +**The anti-vacuity threshold is on sites, not only files.** A file count says a rewrite touched a +file, not that it reached anything inside it. The guard the design asked for, verdict movement, +cannot be used, and not for the reason an earlier note gave: plenty of defended entries move verdicts +hard (`delete-every-catch` takes the tree from 19 to 8), but the IDEAL defended shape is one the +scanner is blind to, and `dead-if-false` and the ten entries beside it are defended precisely because +the tree comes out identical. Requiring movement would fail exactly the entries that work best. + +**A `lowers` exemption is a per-entry field with a reason, not a skip list.** An exempted entry must +still be falling, or the exemption is stale, and its falls must have exactly the measured residual +shape it was granted for: `error-classification` moving pass to not-applicable, every other check +unchanged, nothing moving to fail. Exactly two entries carry one, both non-array-receiver iteration +wrappers. + +**A `KNOWN_GAPS` entry runs as `it.fails`,** so closing the hole later turns the file red until the +entry is moved out deliberately. Two are open: `dead-classifying-try-with-call`, the shape +`dead-classifying-try` only looked like it closed, and `dead-conjunction-instanceof-if`, the sibling +the arm-liveness fix does not close. Both are described above. +`dead-branch-after-if-true` used to be listed on a measurement that was wrong; raising the exit flag +after each statement's branch check rather than before is byte-identical on the real tree and closes +the shape, so the `if (true)` family needed no condition folding after all. + +## Reporting + +The score's own arithmetic has three rules that the report is built to keep honest. + +Every denominator reads `rawChecks`, pre-suppression. `checks` is the display view. Suppressing the +one `request-context` or `audit-trail` finding on an entry must not shrink the gap denominators and +raise the printed percentage, on the same screen as a claim that suppression cannot do that. An +entry's score is capped by what it would have scored unsuppressed, which is how 33 became 50 became +100 before the cap existed. + +`score` is 100 for an entry no scored check applied to, and that is a placeholder rather than a +verdict. Rendering it as a figure turned a route refactored down to a trivial body into a 67-point +improvement, and a trivial route gaining real work into the PR's worst regression, so the PR +comment's cell says "not measured" instead. `globalWithout` recomputes from `rawChecks` minus the +suppression cap, because lowering both figures by the same rule would leave the difference between +them saying something about suppressions rather than about the check. + +`hasDelta` has to be true whenever `renderPrComment` would say something different, because anything +it misses is a change the pull request silently does not report. So it covers every figure the +comment renders: the global, the per-entry score, measured state and suppression set, an entry added +or removed, a check failing at head that did not at base, the parse failure count, the unknown +suppression warnings, the audit and context gaps, `delegating` and `checkContributions`. The +per-entry suppression set and the two gaps are the half that was missing, and it ran the dangerous +way: suppressing an already-failing check moves no score, no measured flag and no new failure, so a +pull request whose entire purpose was silencing findings posted nothing. What is defended is that the +union of the terms is complete rather than that each term is load bearing. Four are individually +reachable with a test each; the global, the removed-entry check and the per-entry score are shadowed +by another term today and are kept because which term shadows which depends on the shape of the +change. `MapReport.suppressions` is the one term deliberately left out, because its totals are summed +from the very per-entry arrays the loop compares. + +Two sections of the PR comment grow with the tree and both are capped, because GitHub's comment limit +is 65,536 characters and a 422 loses the whole comment to the section warning about a typo. A +mistyped directive applied tree wide rendered 87,938 characters. The delegated list is capped at +fifteen rather than ten because a file name is one comma-separated item rather than a line naming +every known check, and the longest route file name in the tree is 130 characters. + +The `AUDIT` line has one shape for every count and no branch on the count, because the branch carried +the bug: a zero used to print "No audit helper exists in the webapp", which is false. The count was +already correct, so the sentence was the only wrong thing. + +A suppression whose id names no check is carried through to both renderers rather than dropped, +because dropping it silently is what made a typo look like an acknowledgement. + +## Tests, timeouts and CI + +`docstringReferences.test.ts` enforces that every test name a docstring in `src/` claims to be +covered by exists. The rule was asked for six times in prose and broken six times, most recently by a +docstring naming a test that was never written, so prose does not enforce itself. What is checked: +every backticked kebab-case token, every backticked glob against the corpus ids by prefix, and every +backticked prose phrase of five words or more with no code punctuation. What is not: a reference +written without backticks, a title of fewer than five words, a comment with no node after it +(leading ranges only, so a comment on the last line of a block is never scanned), and a `.test.ts` +file or `mutations.ts`, both exempted by name. The kebab half is the half that has actually failed. +Three negative controls run the same predicates over an invented docstring, so the guarantee does not +rest on `src/` currently happening to contain a bad reference. + +The real-tree tests are gated by `TREE_SCAN_TIMEOUT`, which is a hang detector and nothing else. +Neither test asserts anything about how long a scan takes, so a number tight enough to be a +performance budget would only be a way to fail on a busy runner, and a performance budget that flakes +gets the whole suite marked unreliable. The old 30s was chosen on an idle machine and does flake. +Measured on an 8-core box, this file alone at load average 0.9: 6.3s to 6.4s for the scan, 10.8s to +11.2s for the sweep. Twenty-four runs as two batches of twelve concurrent copies on those same 8 +cores: 24.2s to 34.0s for the scan and 27.6s to 39.7s for the sweep, with one of the first twelve +dying on a 30s timeout. That contention is not hypothetical: `unit-tests-internal.yml` runs twelve +concurrent shard processes on one runner. The local reproduction is harsher than CI on purpose, +twelve processes over 8 cores against the 32-vCPU runner's 0.375 per core, so 120s is 3x the worst +contended run measured. 60s was the other candidate and is not enough on those numbers. + +Parse failures come from a `ts.Program`, not from the diagnostics array the parser hangs on the +source file, which is internal and which the compiler is free to rename. An undetected parse failure +shrinks the denominator and inflates the score, so it must not be the kind of thing a compiler +upgrade can switch off silently. The host hands the program the source file we already have, so +nothing is parsed twice; the cost is the program machinery, and a full scan of the real tree went +from about 850ms to about 1450ms over five runs of each. + +The suite's turbo task is uncacheable. Its real inputs are mostly not its own files, they are +`apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and the workflow files, so +turbo replayed a pass recorded before a route changed: a route file with a syntax error in it fails +under vitest and came back FULL TURBO in 301ms with the failure cached as a success. `inputs` naming +`../../apps/webapp/...` does bust the cache but replaces turbo 1.x's default file set instead of +adding to it, which drops the package's own files from the hash. The reasoning lives in `turbo.json` +beside the config it explains. + +Three roads reach this suite and all three are asserted. `pr_checks.yml` calls +`unit-tests-observability-map.yml` behind an `obsmap` paths filter and lists it in the `all-checks` +aggregate, without which a test job gates nothing: the first attempt put the job inside +`observability-map.yml`, which reads well and gates nothing, because `all-checks` needs an explicit +list of jobs and cannot see another workflow. The filter watches all of `apps/webapp/app` plus the +report workflow, because the suite reads more than the routes folder and a rename outside it matched +only `webapp`, ran no job, and broke the build for whoever pushed next. It deliberately does NOT name +this package or the two non-webapp roots: `internal` already matches `internal-packages/**` and +`packages/**` and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, so naming +them here ran the suite twice on every PR touching the package. Widening `internal` to the route +paths instead was tried and rejected, since it runs all eighteen internal packages with postgres, +clickhouse, redis and electric to protect one test. + +The report workflow's own text is asserted from `integration.test.ts`, because it is the one thing the +docstring checker cannot reach and the C1 defect was exactly that: two steps disagreeing about what a +missing comment id meant, under a comment claiming they agreed. The render step read it as "a comment +exists" and emitted the resolved state, the upsert step read it as "no id" and POSTed, so a transient +lookup failure either added a second marker comment beside the stale one or announced findings were +gone on a pull request that never had any. The sentinel pair those two shared is gone: the lookup +moved into the cheap `changes` job so the report job's gate could read it, the report job does not +start unless the lookup finished cleanly, and the id both steps use is one job output. Those are text +checks over the workflow rather than a parse of its semantics, so they catch the wiring coming apart +and nothing about whether GitHub agrees. + +Both scan steps write their own file through `--out` rather than capturing stdout with a shell +redirect. `pnpm --filter` takes its recursive path and some versions announce +`Scope: N of M workspace projects` on it; a single line of that in head.json fails the renderer's +`JSON.parse` and the workflow degrades to the stale-report comment on every run, quietly and +permanently. It does not reproduce on the pinned 10.33.2, which was checked, and what is asserted is +the shape that cannot have the bug rather than the version that happens not to. The render step has +no `--out` to reach for, and a banner there puts a stray line in a markdown comment instead of +breaking a parse, so it is left alone. + +The corpus runs on the package's own paths and on a schedule rather than on every route pull request. +It measures the tool's resistance to laundering, which only an edit to the tool can weaken, and it +costs four and a half minutes. The nightly is the other half of that trade: dropping the schedule +would leave tree drift uncovered rather than covered late. + ## Layout `scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only From 272c3df0e2f6368133617170f839af86d8e05e5c Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 14:53:13 +0100 Subject: [PATCH 109/117] refactor(observability-map): apply the comment rule to the source files Cuts every comment that restated the code, and shortens the rest to the proportional statement of the non-obvious why: a hidden constraint, a call into something undocumented, or a named residual a reader of that exact function has to see. The measurements and rejected alternatives they carried now live in the README, and the safety properties keep their test names inline so docstringReferences.test.ts still checks them. No behaviour change: the real-tree report is byte identical. --- .../observability-map/src/adapters/remix.ts | 8 +- .../src/checks/auditTrail.ts | 48 +- .../src/checks/authBoundary.ts | 111 +-- .../observability-map/src/checks/authScope.ts | 73 +- .../src/checks/errorClassification.ts | 180 ++--- .../src/checks/requestContext.ts | 51 +- .../observability-map/src/cli.ts | 21 +- .../observability-map/src/mutations.ts | 235 ++---- .../observability-map/src/report/prComment.ts | 128 +--- .../src/report/prCommentCli.ts | 33 +- .../observability-map/src/report/terminal.ts | 72 +- .../observability-map/src/routeExports.ts | 22 +- .../observability-map/src/scan.ts | 694 +++++------------- .../observability-map/src/score.ts | 120 +-- .../observability-map/src/sensitivity.ts | 77 +- .../observability-map/src/suppression.ts | 86 +-- .../observability-map/src/triviality.ts | 84 +-- .../observability-map/src/types.ts | 188 ++--- 18 files changed, 623 insertions(+), 1608 deletions(-) diff --git a/internal-packages/observability-map/src/adapters/remix.ts b/internal-packages/observability-map/src/adapters/remix.ts index 7815fa3e6a4..00dcc76e4ab 100644 --- a/internal-packages/observability-map/src/adapters/remix.ts +++ b/internal-packages/observability-map/src/adapters/remix.ts @@ -9,12 +9,8 @@ export type Family = | "other"; /** - * The name that carries routing meaning for a given `fileName`. - * - * Flat routes (`api.v1.runs.$runId.ts`) are already that name. Directory routes - * (`_app.orgs.$slug/route.tsx`) hold their module in a fixed `route.ts`/`route.tsx` file, so the - * directory segment before the slash is the meaningful name and `route.tsx` itself is not a path - * segment. + * The name that carries routing meaning for a given `fileName`. A directory route holds its module in + * a fixed `route.ts`/`route.tsx`, so the segment before the slash is the meaningful name. */ function routeName(fileName: string): string { const slashIndex = fileName.indexOf("/"); diff --git a/internal-packages/observability-map/src/checks/auditTrail.ts b/internal-packages/observability-map/src/checks/auditTrail.ts index 001e5f6e8dc..d1a0c06ecf1 100644 --- a/internal-packages/observability-map/src/checks/auditTrail.ts +++ b/internal-packages/observability-map/src/checks/auditTrail.ts @@ -5,29 +5,15 @@ import { isTrivial } from "../triviality.js"; const ID = "audit-trail"; /** - * Calls that write a record of who did something. + * Calls that write a record of who did something. All three reach + * `prisma.impersonationAuditLog.create` in `models/admin.server.ts`, which nothing here matches + * directly, because `calleeNames` records `create` for a member call and no route writes the row + * itself. Matched against `importedNames` too, so an import counts. * - * This list named nothing at all until it was checked. `auditLog`, `recordAudit` and - * `writeAuditEvent` are exported nowhere in apps/webapp, packages/core or internal-packages, - * so the pass branch below could never fire, every applicable route failed, and both renderers - * printed "No audit helper exists in the webapp" while `models/admin.server.ts` was writing - * `prisma.impersonationAuditLog.create({ action, adminId, targetId, ipAddress })` on two paths. - * The rot went unnoticed because `webappSymbols.test.ts` covered every other name list in the - * package and not this one. It covers this one now. - * - * All three names below reach that write: `redirectWithImpersonation` writes the START row, - * `clearImpersonation` writes the STOP row, and `startImpersonation` returns one or the other. - * - * Two of them are also in `SENSITIVE_SYMBOLS`, which is worth saying out loud because it looks like - * the circularity the sensitivity list was cleaned up to remove. It is not quite the same shape: - * `requireAdminApiRequest` was a pure mitigation counted as a hazard, whereas impersonation - * genuinely is the hazard AND genuinely writes the record. The consequence is real all the same, so - * here it is: a route made sensitive only by one of these calls cannot fail this check, because the - * call that put it in the cohort is the call that satisfies it. - * - * Matched against `importedNames` and `calleeNames`, so an import of one counts. Nothing matches - * the underlying `prisma.impersonationAuditLog.create` path directly: `calleeNames` records - * `create` for a member call, and no route in the tree writes the row itself. + * Two of these are also in `SENSITIVE_SYMBOLS`, so a route made sensitive only by one of them cannot + * fail this check: the call that put it in the cohort is the call that satisfies it. That is not the + * circularity the sensitivity list was cleaned up to remove, since impersonation genuinely is the + * hazard AND genuinely writes the record, but the consequence is real either way. */ export const AUDIT_SYMBOLS = [ "redirectWithImpersonation", @@ -36,19 +22,13 @@ export const AUDIT_SYMBOLS = [ ]; /** - * Whether a sensitive mutation leaves a record of who did it. - * - * Applicability follows the same rule as every other check, which it did not before: would this - * evidence necessarily be visible in the body if it existed? It gated on sensitivity and - * `hasAction` alone, so on `resources.impersonation.ts`, a four-statement body, `auth-boundary` - * declined to judge because any guard would be behind the import while this check accused the route - * over an audit write behind that same import. Two checks, opposite verdicts, one fact. + * Whether a sensitive mutation leaves a record of who did it. Applicability follows the same rule as + * every other check, which it did not before: gating on sensitivity and `hasAction` alone had this + * check accusing `resources.impersonation.ts` over an audit write behind the very import + * `auth-boundary` declined to judge it for. * - * So a trivial body is not-applicable here too. A delegating one is handled centrally by - * `scoreEntry`, which answers for every check before any of them runs, so there is no test for it - * here. The order matters and mirrors - * `auth-boundary`: a known audit call is read BEFORE the triviality exemption, because presence is - * evidence even where absence is not. + * The order below matters and mirrors `auth-boundary`: a known audit call is read BEFORE the + * triviality exemption, because presence is evidence even where absence is not. */ export const auditTrail = { id: ID, diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 4abbeb7dc73..546f19e6860 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -7,31 +7,14 @@ import { BUILDERS } from "./errorClassification.js"; const ID = "auth-boundary"; /** - * The guard helpers this webapp actually has, matched against the calling export's own - * `loaderCalleeNames`/`actionCalleeNames`, each scoped to that export's handlers and following one - * hop into a same-file helper. A guard the route only imports and never calls does not count, and - * neither does one the OTHER export calls. + * The guard helpers this webapp actually has, matched against the calling export's own callee names. + * A guard the route only imports and never calls does not count, and neither does one the OTHER + * export calls. * - * A name list rather than the three patterns it replaces, because all three over-matched and this - * is the one check where a false pass hides a security gap: - * - * - `/^(require|authenticate)/` passed any callee at all beginning `require`. Live in the tree: - * `requireSsoEntitlement`, a plan check, cleared `_app.orgs.$organizationSlug.settings.sso`. A - * local `requireValidParams(request)` would do the same for any route someone writes next. - * - `/Authenticated/` passed `resolveAuthenticatedEnv`, used by ten routes, which is - * `findFirst({ where: { id: environmentId } })` in - * `internal-packages/run-engine/src/engine/controlPlaneResolver.ts`: it hydrates an environment - * record, it authenticates nothing. The docstring that put it here asserted the opposite. It also - * passed `commitAuthenticatedSession`, a cookie write, on six routes. - * - `/^verify.*(Hash|Hmac|Signature|Webhook|Callback|Token)/` was sound on the tree, and is kept as - * three names for the same reason as the rest. - * - * Every name resolves to a declaration in the webapp or in the packages it authenticates through; - * `webappSymbols.test.ts` fails if one stops doing so. What that test cannot check is that a - * declaration with the right name is the guard we meant: `authenticateAdmin` and - * `authenticatePlainRequest` are local helpers inside a single route file, so a second route - * declaring its own no-op `authenticateAdmin` would be credited. That is a narrower hole than a - * five-character prefix and it is the reason the list is names rather than patterns. + * Names rather than the three patterns it replaces, all of which over-matched: README, "Sensitivity, + * and the names the tool matches on". `webappSymbols.test.ts` fails if a name stops resolving, but + * cannot check that a declaration with the right name is the guard we meant, which is the residual + * the two local helpers below carry. */ export const GUARDS = new Set([ // Session and PAT identity, `apps/webapp/app/services/session.server.ts` and friends. @@ -59,20 +42,16 @@ export const GUARDS = new Set([ "authenticateUserActor", "authenticateAuthorizeSession", "authenticateAuthorizeBearer", - // remix-auth, reached as `authenticator.authenticate(...)` / `authenticator.isAuthenticated(...)`. - // The login surface is sensitive under `sensitivity.ts` and cannot require an already - // authenticated caller, so establishing identity from the credential presented is what a guard - // means there. + // remix-auth, reached as `authenticator.authenticate(...)`. The login surface is sensitive and + // cannot require an already authenticated caller, so establishing identity from the credential + // presented is what a guard means there. "authenticate", "isAuthenticated", // Local helpers, each declared inside the one route that uses it. "authenticateAdmin", "authenticatePlainRequest", - // Proof of possession: a callback URL carrying an HMAC is authenticated by checking that HMAC, - // and a login-surface second factor is authenticated by checking the code presented. - // `login.mfa`'s action is the second half of a login, so like `authenticate` above it establishes - // identity from the credential rather than requiring an already authenticated caller. It reached - // this list when per-export attribution stopped its loader's `isAuthenticated` speaking for it. + // Proof of possession: an HMAC on a callback URL, or a login-surface second factor. Same + // reasoning as `authenticate` above for the two `login.mfa` names. "verifyHttpCallbackHash", "verifyWebhook", "verifyUserActorToken", @@ -81,38 +60,22 @@ export const GUARDS = new Set([ ]); /** - * Guards that answer with null instead of throwing. Calling one is not evidence of a boundary, - * because the route is free to ignore the answer, so these are only credited when THAT EXPORT's - * handlers demonstrably read what they returned (`EntryPoint.loaderCheckedCallees`). - * - * The distinction is the whole reason this set is separate from `GUARDS`. `requireUserId` redirects - * on its own, so calling it IS the boundary; `getUserId` hands back `string | null` and a route - * that drops it has no boundary at all. Both routes in the sensitive cohort that use one read the - * answer, `invite-accept.tsx` refusing an invite addressed to another email and `login._index` - * sending an already-authenticated caller away, and this rule is what makes that a measured fact - * rather than something a hand-read established once. - * - * What it still cannot see is whether the test that reads the answer guards anything. See - * `EntryPoint.loaderCheckedCallees` for the exact shape of that residual. + * Guards that answer with null instead of throwing, so calling one is not itself a boundary: these + * are credited only when THAT EXPORT's handlers read what they returned + * (`EntryPoint.loaderCheckedCallees`, which also carries the residual, that the test reading the + * answer need not guard anything). */ export const SOFT_GUARDS = new Set(["getUser", "getUserId"]); type GuardedExport = { name: ExportName; guarded: boolean; how: string; export: RouteExport }; /** - * The exports this file declares, each with its own verdict. + * The exports this file declares, each with its own verdict. Per export because the exposure is per + * export, and every input here used to be entry-point-wide, which is a false PASS on the one check + * where that hides a security gap. * - * Per export, because the exposure is per export, and this is the same defect `auth-scope` was - * fixed for one round earlier. Every input here was entry-point-wide: `calleeNames` is the union of - * both bodies, `checkedCallees` was too, and `usesBuilder` was an OR over the two initializer - * callees. So a file whose loader called `requireUser` and whose action called nothing read as - * "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` credited its - * hand-written action with the builder's authentication. Three inputs, one bug, and it is a false - * PASS on the one check where that hides a security gap. - * - * `routeExports` lists only the exports the file actually declares, so an export that calls nothing - * at all is judged rather than skipped: an empty body is exactly the unguarded case. It is shared - * with `auth-scope`, which grew its own copy of the same `[loader, action]` literal. + * `routeExports` lists only the exports the file declares, so an export that calls nothing at all is + * judged rather than skipped: an empty body is exactly the unguarded case. */ function guardedExports(ep: EntryPoint): GuardedExport[] { return routeExports(ep).map((e) => { @@ -133,29 +96,10 @@ function guardedExports(ep: EntryPoint): GuardedExport[] { /** * Whether a route that handles credentials, tokens or money checks who is asking. * - * A fail here is an accusation, and it is only supportable when the body is the place a guard - * would have to be. That holds when the route does its privileged work in the open: reads the - * request, queries the datastore, mints the token. It does not hold for a trivial body, so those - * are reported not-applicable rather than failed. - * - * The reasoning is the triviality rule's own definition rather than a convenience. A trivial body - * has three statements or fewer, three calls or fewer, no try/catch, no builder, and no mention of - * prisma, redis, fetch or the engine anywhere in its source. It therefore cannot contain a visible - * privileged operation. Either it does nothing privileged at all, like the `/orgs/:slug/billing` - * redirect stub, or the privileged work sits behind an import, like `clearImpersonation`, which - * authenticates and writes an audit row in `app/models/admin.server.ts`. In the second case the - * guard is in the same unopened file as the work. Absence of evidence, and reporting it as a - * finding puts a wrong answer at the top of the fix list. - * - * This is not the rule `request-context` uses, deliberately. There the thing being looked for, a - * field on a log call inside a catch, would be in the body if it existed at all, because the catch - * is in the body. Absence of a log is evidence. Here the thing being looked for guards work that - * is not in the body either, so its absence proves nothing. The test that separates them: would - * this evidence necessarily be visible in the body if it existed? - * - * The design also matched `importedNames`. Across the 67 sensitive entry points that widening - * changes nothing, every route with a `require*` import calls it from the body too, so the - * file-wide half only ever stood to hand out a pass for a dead import. It is gone. + * A fail here is an accusation, so it is only made when the body is the place a guard would have to + * be. A trivial body cannot contain a visible privileged operation, so any guard is behind the same + * import as the work and its absence proves nothing. That is the applicability rule in README, "When + * a check declines to judge", and it is deliberately not the rule `request-context` uses. */ export const authBoundary = { id: ID, @@ -167,9 +111,8 @@ export const authBoundary = { // Never empty: `scanFile` returns null unless the file declares a loader or an action. const exports = guardedExports(ep); const guarded = exports.filter((e) => e.guarded); - // Triviality excuses per export, matching the attribution: the reasoning below is about one - // body being the place a guard would have to be, and reading it entry-point-wide let a busy - // action make a redirect-stub loader answerable for a guard it has nothing to guard. + // Triviality excuses per export, matching the attribution: read entry-point-wide, a busy action + // made a redirect-stub loader answerable for a guard it has nothing to guard. const accused = exports.filter((e) => !e.guarded && !isTrivialExport(e.export)); if (accused.length > 0) { return { diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts index 832928fa620..ecb6d889fc4 100644 --- a/internal-packages/observability-map/src/checks/authScope.ts +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -8,14 +8,9 @@ const ID = "auth-scope"; type BuilderExport = { name: string; callee: string; scoped: boolean; why: string }; /** - * The builder-wrapped exports of an entry point, each with its own verdict. - * - * Per export, because the exposure is per export. `authorization` is declared on the builder call - * one export made, and a caller filter is written in the handler one export runs, so neither says - * anything about the other half of the file. - * - * The enumeration itself is `routeExports`, shared with `auth-boundary`, which had to be given the - * same per-export treatment a round later and wrote a second copy of this literal to get it. + * The builder-wrapped exports of an entry point, each with its own verdict. Per export because the + * exposure is per export: `authorization` is declared on the builder call one export made, and a + * caller filter is written in the handler one export runs. */ function builderExports(ep: EntryPoint): BuilderExport[] { return routeExports(ep) @@ -32,57 +27,21 @@ function builderExports(ep: EntryPoint): BuilderExport[] { } /** - * Whether a route the builder authenticated is also narrowed to the caller. - * - * `auth-boundary` passes every builder-wrapped route, and that is correct as far as it goes: the - * nine builders in `BUILDERS` all authenticate. What they do not all do is authorize. - * `authorization` is an optional option and `apiBuilder.server.ts` runs the RBAC gate inside - * `if (authorization)`, so a PAT route can be authenticated and completely unscoped. A PAT names - * its target org or project by id or slug, and with no plugin installed the OSS fallback ability is - * permissive, so nothing on that path stops a member of one org naming another org's project. - * `apps/webapp/CLAUDE.md` states the rule this measures: "A PAT route must resolve its target - * org/project scoped to the caller's membership. Skipping it opens cross-org access." - * - * Two ways for an export to be scoped, and EVERY builder-wrapped export has to be one of them: - * - * - its builder options declare `authorization:` with a real value, which is the RBAC gate, or - * - its own handler filters by the caller's own id, the - * `members: { some: { userId: authentication.userId } }` shape in `api.v1.projects.ts` and the - * `presenter.call({ userId: user.id })` shape the dashboard routes use. - * - * `ability.can(...)` in the handler is deliberately NOT a third way, and it was one for part of - * round C. `apps/webapp/CLAUDE.md` is explicit that it cannot be: the OSS fallback ability is - * permissive (`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and - * `buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an - * ability check enforces the ROLE and the membership-scoped query is the tenant floor. Crediting it - * made this check agree with a route that resolves its target org from a URL slug and puts nothing - * else in front of it. - * - * Applicable only where it is answerable: sensitive and builder-wrapped. Outside - * that it would be a second near-universal fail, which is the shape the `request-context` figure - * already has and which the report has to collapse rather than list. There is no triviality test - * here because there is nothing left for one to refuse: `isTrivial` answers false for any route - * with an initializer callee, so a builder-wrapped route is never trivial. Nor is there a - * delegating test: `scoreEntry` answers not-applicable for a delegating entry before any check - * runs, so one here would be unreachable, and one WAS here saying otherwise. - * - * Three residuals, running in both directions. - * - * Accusing: `scopesByCallerIn` reads property assignments in that export's own handlers only. A - * handler that pulls the id into a local first, `const userId = user.id; ... { userId }`, or that - * builds its filter in a same-file helper, scopes itself and is not seen. + * Whether a route the builder authenticated is also narrowed to the caller. The IDOR class it + * measures, the two ways to be scoped, and why `ability.can(...)` is deliberately not a third: + * README, "Authenticated is not the same as scoped". * - * Crediting: a caller id passed as an ACTOR argument rather than as a filter still counts. - * `ssoController.generatePortalLink({ organizationId: orgId, userId: user.id })` records who asked; - * it does not constrain which org is read. Telling the two apart means knowing what the callee does - * with the argument, which for the dashboard means following it into a presenter. No route in the - * tree is credited by this alone today. + * There is no triviality test here because `isTrivial` answers false for any route with an + * initializer callee, and no delegating test because `scoreEntry` answers for every check before any + * of them runs. One WAS here saying otherwise. * - * Crediting: a helper that runs the membership query for you is credited through the caller id - * handed to it. `ApiKeysPresenter`, `TeamPresenter`, `regenerateApiKey` and - * `DeleteOrganizationService` all do `members: { some: { userId } }` internally and throw when it - * misses, which makes those four correct, and it is the same syntax as the actor-argument case - * above. All were hand-read in round C. + * Three residuals, running both ways. Accusing: `scopesByCallerIn` reads property assignments in that + * export's own handlers only, so a handler pulling the id into a local first + * (`const userId = user.id; ... { userId }`) is not seen. Crediting: a caller id passed as an ACTOR + * argument rather than as a filter still counts, e.g. `generatePortalLink({ organizationId, userId })` + * records who asked without constraining which org is read. Crediting: a helper running the membership + * query for you is credited through the caller id handed to it, which is the same syntax; the four + * helpers this applies to were hand-read and are correct. */ export const authScope = { id: ID, diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 34cfce33d34..22044700b57 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -5,12 +5,11 @@ const ID = "error-classification"; /** * The route builders that authenticate the request, which is what `auth-boundary` reads them for. - * They also catch and classify, passing a thrown `Response` through untouched and reporting - * anything else through `logBoundaryError`, but `error-classification` no longer credits that: a - * route with no catch of its own is judged on nothing, wrapper or not. + * They also catch and classify, but `error-classification` does not credit that: a route with no + * catch of its own is judged on nothing, wrapper or not. * * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not - * authenticate, so counting it here would hand two routes a free pass on `auth-boundary`. + * authenticate, so listing it would hand two routes a free `auth-boundary` pass. * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. */ export const BUILDERS = new Set([ @@ -27,22 +26,12 @@ export const BUILDERS = new Set([ /** * How much a try block may guard and still count as narrow. Two, so the guarded operation can bind - * its result (`const stripped = ...; new RegExp(stripped);`), but a third statement means the try - * has started to cover the handler rather than one operation. The idiom this was chosen for and - * hand-read against originally: 55 of 427 entry points, 11 of the failures at the time, all eleven - * the deliberate `try { body = await request.json() } catch { 400 }` shape. + * its result (`const stripped = ...; new RegExp(stripped);`). * - * An absolute count, not a ratio against the enclosing body. A ratio is diluted by anything else in - * the same body: padding the action with unrelated statements after the try relabelled the same - * broad swallow as a narrow guard, moving the denominator without touching the clause at all. - * `inert-statements-after-try` in the mutation corpus is that shape, and it holds. - * - * What the count is NOT is unpaddable, which an earlier docstring and commit subject both claimed. - * `countStatement` now counts declarators and comma operands rather than semicolons, so the two - * known ways to pack a try into fewer statements move the number the same as writing it out; that - * is what `merge-declarations` and `merge-comma-expressions` in the corpus check. A third - * way nobody has written down would work, which is why the count is no longer the only condition - * and no longer the load-bearing one. + * An absolute count and not a ratio against the enclosing body, or padding the body relabels the + * same broad swallow as a narrow guard (`inert-statements-after-try`). The count is NOT unpaddable, + * which an earlier docstring claimed: it is one condition of three and no longer the load-bearing + * one. See README, "Parse guards, and the narrow-try count". */ const NARROW_TRY_STATEMENTS = 2; @@ -50,29 +39,10 @@ const NARROW_TRY_STATEMENTS = 2; * Whether a catch clause is a guard rather than the route's error handling: the try block parses, * waits for nothing except that parse, and is short. * - * `awaitsOnlyParse` is the condition the previous wave was missing, and it is the one a statement - * count cannot express. `try { const body = await request.json(); return await handleEverything(body); } - * catch { return 500; }` is two statements, one of them a parse, and the whole handler inside it: - * the count reads it as narrow and it is the `otel.v1.logs.ts` swallow written compactly. Asking - * what the block waits for separates them, and unlike the count it does not care how the statements - * are punctuated or how deeply the work is nested inside one of them. - * - * The design's own suggestion, requiring the clause to answer with a 4xx, was measured first and is - * not used. On its own it credits 11 clauses guarding four to thirty statements, the widest swallows - * in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try answers every failure - * with a 400 carrying the internal error message. Added on top it costs three routes their pass, - * all three narrow parse guards that compute a fallback value rather than answering a request - * (`try { return new URL(referer).origin; } catch { return undefined; }`), and it buys only the case - * of a narrow parse guard answering 500. Requiring every CALL to be a parse, rather than every - * await, was measured too and is worse still: it refuses the four `matchPattern.slice(4); new - * RegExp(...)` guards, because preparing a parse's input is ordinary synchronous string work. - * - * Two residuals, since awaiting is the signal. A try block that does its non-parse work - * synchronously still reads as a guard. And `guardedWork` looks for a `ts.AwaitExpression`, which - * `for await (const chunk of work(await request.json()))` and `await using` are not, so a block - * whose only non-parse work is one of those reads as a guard too. Neither occurs in the tree and - * neither is reachable by rewriting a real route, since both need work that is not there to begin - * with. Both are in the round A fix 3 report. + * Two residuals, since awaiting is the signal rather than calling. A block that does its non-parse + * work synchronously still reads as a guard, and `guardedWork` looks for a `ts.AwaitExpression`, + * which `for await (...)` and `await using` are not. Neither occurs in the tree and neither is + * reachable by rewriting a real route. Both are in the round A fix 3 report. */ function isParseGuard(clause: CatchEvidence): boolean { return ( @@ -83,26 +53,14 @@ function isParseGuard(clause: CatchEvidence): boolean { } /** - * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, on - * an `if`, a `switch` or an `instanceof`, or it guards a parse it can answer for. - * - * Rethrowing is not a third way, which is the correction from the last wave. A clause whose only - * effect is `throw e` leaves the error propagating exactly as it would with no catch at all, so - * treating that as a pass while no catch is not-applicable paid 50 points a route for wrapping a - * body in `try { ... } catch (e) { throw e }`, and 27 across the tree. The two are observationally - * identical and are now scored identically. - * - * The cost is real and worth stating: `catch (e) { logger.error(...); throw e }` also reads as - * inert, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. That - * withholds credit from a route that reports before propagating, which is the safe direction to be - * wrong in, since crediting it would reopen the hole a bare `logger.error` line wide. - * `request-context` still reads that log and asks whether it names a tenant, so the reporting is - * unrewarded here rather than unmeasured. - * - * A narrow guard is not a way to qualify either. A one-statement try around `await - * service.call(run)` is narrow and is still a swallow: reading all eleven entry points that limb - * would clear said six were real, including a silent run cancellation and two credential paths - * that report a database failure to the browser as a 400 with an internal message in it. + * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, or + * it guards a parse it can answer for. Rethrowing is not a third way, and neither is being narrow + * without parsing. + * + * The cost is real and worth stating here: `catch (e) { logger.error(...); throw e }` reads as + * inert too, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. + * That is the safe direction, since crediting it reopens the hole a bare `logger.error` line wide, + * and `request-context` still reads that log and asks whether it names a tenant. */ function decides(clause: CatchEvidence): boolean { return clause.branches || isParseGuard(clause); @@ -119,62 +77,18 @@ function swallows(clause: CatchEvidence): boolean { } /** - * Who decides what a failure means, and on what evidence. - * - * Judged per catch clause, so an entry point is only as good as its worst one. That is the point of - * the per-clause evidence: 39 routes have more than one catch and 17 mix a narrow guard with a - * broad handler, and under the old aggregate booleans a single well-behaved catch spoke for the - * swallow next to it. - * - * A route with no catch is not-applicable, not a pass. It makes no classification decision, so - * there is nothing here to judge and nothing to credit. Crediting it was worse than merely - * generous: with `request-context` also passing the same routes, emptying every catch clause in the - * tree scored it 100, so the metric paid you for deleting error handling. Out of the denominator - * is the honest place for it, and it takes the builder credit with it: a builder-wrapped route with - * no catch of its own now sits out too, rather than collecting a point for the wrapper. - * - * "Does this route catch anything" is `catches.length`, never `hasTryCatch`. A try/finally with no - * catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error - * propagates once the cleanup has run, and reading the old flag as a catch put - * `admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. - * - * `callbackCatches` is the third case, and it is what stops "no catch is not-applicable" from being - * a payout. A refused catch is judged on its evidence, never on its placement: the same - * `catchClauseEvidence` an own catch gets, with two arms reading it. A refused swallow fails the - * route whenever nothing the route owns decides, and that arm is deliberately not conditioned on - * the route owning no catches, so an own inert rethrow catch cannot lift a refused swallow out of - * the verdict (`fails a per-item swallow even when the route owns an inert rethrow catch`). A - * route whose only catches are refused and none of them swallows sits out, and never passes: the - * not-applicable ceiling is what keeps a prepended dead deciding `.map` from minting a pass on the - * 261 catchless routes, which `dead-deciding-map` in the mutation corpus holds at tree scale and - * `sits out a catchless route with a prepended dead deciding map` pins on a fixture. What the old - * blanket placement rule blocked, relocating a swallow behind the boundary, still fails - * (`still fails a swallow wrapped in a non-array receiver's .map(...)`); what it wrongly accused, - * a route whose only error handling genuinely is per item, now sits out instead of failing - * (`sits out a route whose only catch is a deciding per-item boundary`). - * - * A clause whose try block holds nothing that could raise is read as no clause at all, - * `guardCanRaise` on the evidence. Prepending `try { 0; } catch (e) { if (e instanceof Error) { - * return json(x, { status: 400 }); } throw e; }` to every body takes the tree from 19 to 44 and - * raised 224 routes, because the 261 routes that catch nothing were sitting at not-applicable and a - * dead clause moved each of them to pass. - * - * What that refuses is `try { 0; }`, and it is defeated by one inert call: `try { String(0); }` - * reads as classification and pays the same 224 routes, because `canRaise` accepts any call at all. - * The rule closes the shape that was found, not the family, and telling an inert call from a - * throwing one needs types the scanner does not have. `dead-classifying-try-with-call` in the - * mutation corpus is the open shape, running as an expected failure. - * - * The refused-swallow arm reads the route's own deciding catches through `guardMayRaise`, never - * through `guardCanRaise`. `canRaise` is a whitelist and misses real raising code (a destructuring - * declaration is not on its list, and `const { a } = undefined` throws), so ordering the arm off - * `reachable` accused a route that owns a real classifying catch of owning none, which was simply - * untrue; `does not accuse a route that owns a catch of owning none` pins the verdict. The - * containment read `guardMayRaise` is false only for the provably-inert `try { 0; }`, so the one - * clause that must not block the accusation, the prepended dead classifier `dead-classifying-try` - * refuses, still does not block it (`still fails a per-item swallow beside a deciding catch over a - * dead guard`). The already-open residual is unchanged: `try { String(0); }` reads as may-raise - * AND can-raise, which is `dead-classifying-try-with-call`, the corpus's expected failure. + * Who decides what a failure means, and on what evidence. Judged per catch clause, so an entry point + * is only as good as its worst one. + * + * A route with no catch is not-applicable rather than a pass, and that takes the builder credit with + * it. "Does this route catch anything" is `catches.length`, never `hasTryCatch`, because a + * try/finally leaves the flag true and the list empty. + * + * The open hole a reader of this function has to know about: `guardCanRaise` refuses `try { 0; }` + * and is defeated by one inert call, so `try { String(0); }` reads as classification and takes the + * tree from 19 to 44. `dead-classifying-try-with-call` in the mutation corpus is that shape, + * running as an expected failure. Read the rule as "refuses `try { 0; }`", never as "an unreachable + * catch cannot be credited". Everything else here: README, "The dead-code defence". */ export const errorClassification = { id: ID, @@ -187,11 +101,8 @@ export const errorClassification = { if (swallowed.length > 0) { const which = reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; - // "One way out" is only true of a clause that never throws. A clause holding a `throw` that - // is not its only exit is a swallow by this check's definition (it decides nothing about the - // error) and it is NOT one way out, so saying so was a false accusation. 16 clauses in the - // tree changed `rethrows` from true to false this round and every one of them would have - // been eligible for it. + // "One way out" is only true of a clause that never throws, so this reads `throws` and not + // `rethrows`. 16 clauses in the tree would otherwise carry the wrong detail line. const everyWayOut = swallowed.every((c) => !c.throws); return { id: ID, @@ -201,15 +112,11 @@ export const errorClassification = { : `catches its errors and chooses what to do without looking at what was thrown${which}`, }; } - // A refused (iteration-callback) catch is judged on its evidence, never on its placement. - // The fail arm first: a refused swallow fails whenever nothing the route owns decides. // Deliberately NOT conditioned on `ep.catches.length === 0`: an own inert catch, which - // `wrap-body-in-rethrow` adds to every route, must not lift a refused swallow out of the - // verdict, or wrapping a per-item-swallow route in try/rethrow reads "every catch rethrows". - // `fails a per-item swallow even when the route owns an inert rethrow catch` pins that. - // "Nothing the route owns decides" is read off `ep.catches` under `guardMayRaise`, not off - // `reachable`: a deciding catch `canRaise` cannot see still decides, and only the - // provably-inert `try { 0; }` guard is excluded. See the `guardMayRaise` paragraph above. + // `wrap-body-in-rethrow` adds to every route, must not lift a refused swallow out of the verdict + // (`fails a per-item swallow even when the route owns an inert rethrow catch`). And read off + // `ep.catches` under `guardMayRaise`, never `reachable`, since a deciding catch `canRaise` + // cannot see still decides (`does not accuse a route that owns a catch of owning none`). const reachableCb = ep.callbackCatches.filter((c) => c.guardCanRaise); const ownDecides = ep.catches.some((c) => decides(c) && c.guardMayRaise); if (!ownDecides && reachableCb.some(swallows)) { @@ -220,14 +127,9 @@ export const errorClassification = { "a catch inside an iteration callback swallows what it caught, and nothing the route owns decides", }; } - // The ceiling: refused catches never reach the pass arm, so a route whose only catches are - // refused and none of them swallows sits out of the denominator rather than collecting - // anything. Read off `ep.catches`, not `reachable`: a route that owns a catch owns one, - // whether or not `canRaise` could see what it guarded; ordering this off `reachable` turned - // every `canRaise` miss on a route that also has a per-item catch into an accusation that was - // flatly false. The detail asserts nothing about ownership or per-item-ness the scanner - // cannot know: a once-invoked Result-style wrapper with a deciding inner catch reads the same - // as its inline equivalent would. + // The ceiling that keeps `dead-deciding-map` from minting a pass on the 261 catchless routes: + // refused catches never reach the pass arm. Read off `ep.catches` and not `reachable`, since a + // route that owns a catch owns one whether or not `canRaise` could see what it guarded. if (ep.catches.length === 0 && ep.callbackCatches.length > 0) { return { id: ID, diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index f54dc8d5f9e..da3f1aa2014 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -4,30 +4,19 @@ import { isTrivial } from "../triviality.js"; const ID = "request-context"; /** - * A field name that plausibly names a TENANT: environment, organization, project or user, the four - * things every entry point ultimately belongs to. Anchored on the root word, not just the suffix, - * in the full and abbreviated camelCase the webapp actually writes for each: `environmentId`/ - * `envId`, `organizationId`/`organizationSlug`/`orgId`, `projectId`/`projectParam`, `userId`. A bare - * `id`, and a resource id that happens to share the same `Id`/`Param` suffix, `batchId`, - * `notificationId`, `chatId`, `spanParam`, `runFriendlyId`, `taskIdentifier`, does not qualify: - * those name a resource the failure touched, not who it happened to. - * - * The abbreviated roots, `env` and `org`, require a suffix; the full words do not. A bare `env` is - * ambiguous with a deployment environment name (`{ env: process.env.NODE_ENV }`), which is not a - * tenant, and nothing in the tree relies on it being bare, so the field alone cannot qualify. + * A field name that plausibly names a TENANT: environment, organization, project or user. Anchored on + * the root word and not just the suffix, so `batchId` and `spanParam` do not qualify: those name a + * resource the failure touched rather than who it happened to. The abbreviated roots require a + * suffix, because a bare `env` is ambiguous with `{ env: process.env.NODE_ENV }`. */ const TENANT_FIELD = /^(environment|organization|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$|^(env|org)(Id|Ids|Slug|Ref|Param|Identifier)$/; /** - * Levels against the real logger (`packages/core/src/logger.ts`): `log`, `error`, `warn`, `info`, - * `debug`, `verbose`, in that order, no `fatal` and no `trace` (the `trace` in - * `apps/webapp/app/services/logger.server.ts` is the AsyncLocalStorage field helper, unrelated to - * log level). `log` is level 0, the level `TRIGGER_LOG_LEVEL` never filters out, so it qualifies - * alongside `error` and `warn`. `info`, `debug` and `verbose` do not: `info` is not reserved for - * failure reporting, so a route can log an info line inside a catch that says nothing about the - * catch actually handling anything, and `debug`/`verbose` are routinely dropped or sampled out - * before anyone reads an incident. + * Read against the real logger (`packages/core/src/logger.ts`), whose levels are `log`, `error`, + * `warn`, `info`, `debug`, `verbose`, with no `fatal` and no `trace`. `log` is level 0, which + * `TRIGGER_LOG_LEVEL` never filters out. `info` is not reserved for failure reporting and + * `debug`/`verbose` are routinely sampled out before anyone reads an incident, so neither qualifies. */ const QUALIFYING_LEVELS = new Set(["log", "error", "warn"]); @@ -39,25 +28,11 @@ function logLevel(callee: string): string { /** * Whether a failure here can be traced to whoever it happened to. * - * Everything the platform attaches centrally is accounted for, which is what makes this worth - * asking. `logger` pushes the http context, `{ requestId, path, host, method }`, onto every line - * through AsyncLocalStorage, and `Logger.onError` forwards the error to Sentry. Neither carries a - * tenant: no route calls `trace({ environmentId }, ...)`, and the builders' own boundary log is - * `logBoundaryError(message, error, url)`, a url and an error. So an incident tells you which route - * and which request failed, and never whose environment it was, unless the route passed the field - * itself. 11 of 427 entry points do, naming an environment, organization, project or user; the - * other 10 that used to be counted here only named a resource the failure touched, not a tenant. - * - * Every non-trivial entry point is judged, and a route that never catches fails like any other. - * That is the whole point rather than an oversight: its failures go to the global handler, which - * names no tenant, so it genuinely cannot say whose request broke. Passing those routes, as this - * check used to, meant deleting every catch clause in the tree scored it 100. Excusing them as - * not-applicable would be the same mistake in quieter clothes, since it would once again reward - * having no failure handling to inspect. - * - * The consequence is a check that fails 90% of what it looks at, which is an honest reading of a - * codebase where the fix is one platform change, tenant fields through `trace(...)` in the auth - * path, rather than 300 route edits. Weight it accordingly, but do not read the count as noise. + * Every non-trivial entry point is judged, and a route that never catches fails like any other. That + * is the whole point rather than an oversight: its failures go to the global handler, which names no + * tenant. Passing those routes, as this check used to, meant deleting every catch clause in the tree + * scored it 100, and excusing them as not-applicable is the same mistake in quieter clothes. What the + * platform attaches centrally and why none of it is a tenant: README, "What 19 means". */ export const requestContext = { id: ID, diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts index 4ec6be1ba8e..a54995e1954 100644 --- a/internal-packages/observability-map/src/cli.ts +++ b/internal-packages/observability-map/src/cli.ts @@ -15,10 +15,8 @@ import { renderJson } from "./report/json.js"; const DEFAULT_ROUTES = "apps/webapp/app/routes"; /** - * Walks up from this file looking for `pnpm-workspace.yaml`, so the routes directory resolves - * correctly whether `map` is run from the repo root or from the package directory (where - * `pnpm --filter` puts you). Resolving `DEFAULT_ROUTES` against `process.cwd()` instead would only - * work from the repo root. + * Walks up looking for `pnpm-workspace.yaml`, so the routes directory resolves whether `map` runs from + * the repo root or from the package directory, where `pnpm --filter` puts you. */ function findRepoRoot(startDir: string): string { let dir = startDir; @@ -40,10 +38,8 @@ const processIo: Io = { }; /** - * Entry points matching what the user typed, by file name or by route path, exact first. - * - * Route paths matter because they are what the report prints: `map /api/v1/token` used to exit 1 - * because only the file name was matched, so the identifier on screen was not one you could paste + * Entry points matching what the user typed, by file name or by route path, exact first. Route paths + * matter because they are what the report prints, so the identifier on screen is one you can paste * back in. */ function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { @@ -130,15 +126,14 @@ export function main(argv: string[], io: Io = processIo): number { } const report = buildReport(entryPoints, parseFailures); - // JSON only. `renderTerminal` puts these lines in the report body, so warning here as well - // printed each one twice in a terminal run. Stderr is what the JSON path has instead, since a - // warning on stdout would be inside the document a caller parses. + // JSON only: `renderTerminal` already puts these lines in the report body, and a warning on stdout + // would be inside the document a caller parses. if (asJson) for (const line of unknownSuppressionLines(report)) io.err(`${line}\n`); io.out(asJson ? renderJson(report) : renderTerminal(report)); io.out("\n"); if (!noWrite) { - // `--out` exists so a test can point the write somewhere disposable. Without it the only way - // to exercise the write path was to let the tests create and delete a file in the repo root. + // `--out` exists so a test can point the write somewhere disposable rather than creating and + // deleting a file in the repo root. const outFlag = flagValue(args, "--out"); const outPath = outFlag === null diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index 4273223f881..7c5091409b0 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -1,41 +1,16 @@ import ts from "typescript"; /** - * Source-to-source mutations for the tree-scale corpus in `mutationCorpus.test.ts`. - * - * Every mutation here is a *text* rewrite driven by AST positions, never a reprint. A reprint would - * change formatting everywhere and make a failure impossible to read; splicing at node positions - * leaves the rest of the file byte-identical, so a corpus failure can be diffed down to the one - * construct that moved. - * - * Two kinds of entry live in the corpus and they are labelled `preserving` and `deleting`: - * - * - `preserving`: the rewrite does not change what the route does. Dead code that can never run, - * a wrapper that runs the same statements once, a comment, a merge of adjacent `const`s. The - * property under test is the one the tool claims: no such edit may raise the score. - * - `deleting`: the rewrite removes error handling or logging. The route is worse afterwards, so - * the score must not rise either, for a different and simpler reason. - * - * Within `preserving` there are two directions, and for a long time the corpus only had one of - * them. A subtractive rewrite takes real signal away or moves it about: delete the catches, wrap - * the body, merge the statements. An ADDITIVE rewrite puts fake signal in: a classifying catch over - * a try that cannot throw, a test whose two arms are the same, a rethrow that can never run. The - * additive direction is the one someone reaches for when a CI comment nags them, and it is where - * the two largest holes ever found here lived. `ADDITIVE_IDS` lists the entries that cover it, and - * `mutationCorpus.test.ts` asserts the class is not empty so it cannot quietly go away again. - * - * Neither kind is ever executed. "Semantics-preserving" here means preserving the observable - * behaviour of the route as written, which is what the scanner claims to measure; it is not a - * claim that the mutated tree compiles against its real types. + * Source-to-source mutations for the tree-scale corpus in `mutationCorpus.test.ts`. Why they are text + * rewrites rather than reprints, what `preserving` and `deleting` mean, and why the additive direction + * is tracked separately in `ADDITIVE_IDS`: README, "The mutation harness". */ export type MutationKind = "preserving" | "deleting"; /** - * The result of rewriting one file: the new source, and how many places in it the rewrite actually - * landed. `sites` is what the corpus's anti-vacuity guard reads. A file count says nothing about - * whether the rewrite reached anything, and a mutation that quietly matched two constructs in a - * file would otherwise look identical to one that matched forty. + * The new source, and how many places in it the rewrite landed. `sites` is what the anti-vacuity guard + * reads, because a file count says nothing about whether the rewrite reached anything inside the file. */ export type MutationResult = { source: string; sites: number }; @@ -45,13 +20,9 @@ export type Mutation = { /** What the rewrite does, in one line, for the corpus table in the report. */ what: string; /** - * Set only on a `preserving` entry that is EXPECTED to lower some routes' scores, with the - * one-line reason on the entry itself. The mirror assertion in `mutationCorpus.test.ts` requires - * falls to be empty for every preserving entry without this field, and for an entry with it, - * requires falls to be nonzero and every fall to be exactly `error-classification` moving pass - * to not-applicable with nothing moving to fail. Deliberately a per-entry field rather than a - * set or a skip list: an exemption is a decision with a reason, enforced in both directions, and - * it must not be a place entries get filed so the suite stays green. + * Set only on a `preserving` entry EXPECTED to lower some routes' scores, with the reason on the + * entry. A per-entry field rather than a skip list, so an exemption is a decision with a reason + * enforced in both directions and not a place entries get filed to keep the suite green. */ lowers?: string; /** The mutated file, or null when this file has nothing for the mutation to touch. */ @@ -71,11 +42,8 @@ function parse(fileName: string, source: string): ts.SourceFile { } /** - * Splice edits into `source`, right to left so earlier offsets stay valid. - * - * An edit that falls inside an earlier edit's range is dropped rather than applied: a mutation that - * deletes a catch clause and one that rewrites a statement inside that clause would otherwise - * produce overlapping splices. Dropping the inner one is what "the outer rewrite won" means. + * Splice edits into `source`, right to left so earlier offsets stay valid. An edit inside an earlier + * edit's range is dropped, which is what "the outer rewrite won" means. */ function applyEdits(source: string, edits: Edit[]): MutationResult | null { if (edits.length === 0) return null; @@ -141,12 +109,9 @@ function propertyNameOf(property: ts.ObjectLiteralElementLike): string | null { } /** - * Handler functions on a builder's object argument, in the two shapes the route builders use: - * `handler` at the top level and `methods.POST.handler`. - * - * Deliberately a copy of the same shapes `src/scan.ts` recognises rather than an import of them. - * The harness has to be able to disagree with the scanner about where a route body is; sharing the - * scanner's own notion would let a bug in that notion hide a laundering shape from the corpus. + * Handler functions on a builder's object argument, in the two shapes the route builders use. + * Deliberately a copy of the shapes `src/scan.ts` recognises rather than an import: the harness has to + * be able to disagree with the scanner about where a route body is. */ function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { for (const property of object.properties) { @@ -180,12 +145,9 @@ function rootCall(call: ts.CallExpression): ts.CallExpression { } /** - * The handler functions an export's initializer resolves to. - * - * `locals` is consulted for the two indirect spellings, both of which `scan.ts` resolves and - * neither of which this reached: `export const action = route.action` beside - * `const route = createActionApiRoute(...)`, which is 7 of the tree's API routes, and - * `export const action = handleThing` naming a local. `seen` stops `const a = b; const b = a`. + * The handler functions an export's initializer resolves to. `locals` is consulted for the two indirect + * spellings, `export const action = route.action` and `export const action = handleThing`; `seen` stops + * `const a = b; const b = a`. */ function fromInitializer( expr: ts.Expression, @@ -260,22 +222,12 @@ function localDeclarations(sf: ts.SourceFile): LocalDeclarations { } /** - * Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. - * - * Reads the same four export forms `scan.ts` reads: an exported function declaration, an exported - * `const`, an exported object binding pattern, and a named export clause resolved back through a - * local. It read only the first two, which is the shape of every API route in the tree - * (`const { action, loader } = createActionApiRoute(...); export { action, loader };` and the - * direct `export const { action } = ...`), so `wrapEveryBody` and the other whole-body entries - * silently skipped 36 of the 427 entry points while reporting a file count that suggested - * otherwise. `mutationCorpus.test.ts` pins the population now ("wraps a body in every - * non-delegating entry point the scanner finds"), so the harness cannot lag the scanner here again - * without going red. + * Block bodies of the exported `loader`/`action` handlers, the region a whole-body wrapper wraps. Reads + * the same four export forms `scan.ts` reads, and `wraps a body in every non-delegating entry point the + * scanner finds` pins that so the harness cannot lag the scanner again. * - * This is NOT a retreat from the deliberate independence `collectNamedHandlers` documents. That - * independence is about disagreeing over where a HANDLER sits inside a builder's argument, which is - * a judgement the corpus has to be able to make for itself. Which exports exist is not a judgement, - * and the harness was simply behind. + * Not a retreat from the independence `collectNamedHandlers` keeps: that is about where a HANDLER sits + * inside a builder's argument, which is a judgement. Which exports exist is not. */ function entryBodies(sf: ts.SourceFile): ts.Block[] { const functions: EntryFunction[] = []; @@ -352,15 +304,10 @@ function bindingNameOf(clause: ts.CatchClause): string | null { } /** - * Splice a statement in at the HEAD of every catch clause that names its binding. `snippet` - * receives the binding name. - * - * The head, not the tail, and that is the whole point of the helper. Appending put the shape after - * whatever the clause already did, and 234 of the tree's 260 clauses end in a `return` or a - * `throw`, so in those the spliced shape was dead by ordering before the rule under test ever - * looked at it: eleven corpus entries reported touching 172 files while exercising 26 clauses. At - * the head every clause is reachable, so every clause exercises the rule. The shapes spliced this - * way are dead wherever they sit, so moving them does not make the rewrite any less preserving. + * Splice a statement in at the HEAD of every catch clause that names its binding, which is the whole + * point of the helper: 234 of the tree's 260 clauses end in a `return` or a `throw`, so an appended + * shape was dead by ordering before the rule under test looked at it. See README, "The mutation + * harness". */ function prependToEveryCatch( id: string, @@ -385,9 +332,8 @@ function prependToEveryCatch( }; } -/** Whether a statement list ends in a way that makes anything spliced in after it dead. Used only - * to keep the dead-throw mutations honest: appending `throw e;` after statements that might fall - * through would change what the route does, and this corpus is not allowed to do that. */ +/** Whether a statement list ends in a way that makes anything spliced in after it dead. Appending + * `throw e;` after statements that might fall through would change what the route does. */ function endsInAnExit(statements: readonly ts.Statement[]): boolean { const last = statements[statements.length - 1]; return last !== undefined && (ts.isReturnStatement(last) || ts.isThrowStatement(last)); @@ -418,19 +364,14 @@ function containsLooseJump(node: ts.Node): boolean { } /** - * Wrap every catch clause's body in a construct that definitely exits, then write `throw e;` after - * it. The throw can never run, and before `definitelyExits` learned to see through the wrapper each - * of these read as the clause rethrowing, which is `not-applicable` instead of `fail` and worth 50 - * points a route. - * - * Only applied to a clause whose statements already end in a `return` or a `throw`, so the appended - * throw really is unreachable, and never to one holding a loose `break` or `continue`, which a `do` - * or a `switch` would capture. + * Wrap every catch clause's body in a construct that definitely exits, then write `throw e;` after it. + * Applied only to a clause already ending in a `return` or a `throw`, so the appended throw really is + * unreachable, and never to one holding a loose `break` or `continue`, which a `do` or a `switch` would + * capture. * - * `dead-throw-after-switch-break` guards the opposite direction of the same rule. A `break` in a - * switch clause is no longer read as leaving the statement list the switch sits in, and the cheap - * way to write that is "a clause holding a break does not exit", which would take this whole family - * back: the clause here returns AND breaks, and the return is what has to win. + * `dead-throw-after-switch-break` guards the opposite direction: the cheap way to stop reading a + * switch clause's `break` as an exit is "a clause holding a break does not exit", which would take this + * whole family back, since the clause here returns AND breaks and the return has to win. */ function deadThrowAfter(id: string, what: string, wrap: (body: string) => string): Mutation { return { @@ -525,10 +466,8 @@ function logStatementEdits(sf: ts.SourceFile): Edit[] { } /** - * Remove the catch clause from every `try`. With a `finally` present the clause alone goes and the - * `try`/`finally` stands; without one the whole `try` collapses to the bare block it guarded, which - * is still a legal statement. Point edits either way, so a nested rewrite inside the clause is - * simply dropped by `applyEdits` rather than colliding. + * Remove the catch clause from every `try`. With a `finally` the clause alone goes; without one the + * whole `try` collapses to the bare block it guarded, which is still a legal statement. */ function catchDeletionEdits(sf: ts.SourceFile): Edit[] { const edits: Edit[] = []; @@ -547,17 +486,15 @@ function catchDeletionEdits(sf: ts.SourceFile): Edit[] { // -- the corpus ------------------------------------------------------------------------------- /** - * Every laundering shape found by a reviewer on this branch, plus the five extra dead-code shapes - * and two extra iteration receivers found while writing this file. Each entry is a whole-tree - * rewrite; `mutationCorpus.test.ts` asserts the global score does not rise for any of them. + * Every laundering shape found on this branch. Each entry is a whole-tree rewrite; + * `mutationCorpus.test.ts` asserts the global score does not rise for any of them. */ export const MUTATIONS: Mutation[] = [ prependToEveryFile( "suppress-every-check", "prepend an obs-map-disable directive for every check to every file", - // Every check, which this said it was and was not: `auth-scope` was added a round after the - // entry was written and never added here, so the "a suppression cannot raise a score" - // invariant went untested at tree scale for the 19 routes it applies to. + // Every registered check, which `suppresses every registered check in the exhaustive sweep` + // holds: `auth-scope` was once missing here and nothing noticed. [ "// obs-map-disable error-classification -- mutation corpus", "// obs-map-disable request-context -- mutation corpus", @@ -636,11 +573,9 @@ export const MUTATIONS: Mutation[] = [ "try {", "} catch (obsMapMutationError) { throw obsMapMutationError; }" ), - // The A/B partner of the entry above: the only difference is the ternary. `error-classification` - // asks whether ANY reachable catch decides, so one deciding clause bought at zero cost would take - // every route in the tree to a pass. That is what the same-arms rule in `selectsAnErrorPath` - // refuses, and this is the tree-scale proof of it on the throw path, which was untested while - // the throw path could not credit a ternary at all. + // The A/B partner of the entry above, differing only in the ternary. One deciding clause bought at + // zero cost would take every route in the tree to a pass, which is what the same-arms rule in + // `selectsAnErrorPath` refuses; this is the tree-scale proof of it on the throw path. wrapEveryBody( "wrap-body-in-same-arms-throw-ternary", "wrap every route body in try { ... } catch (e) { throw e instanceof Error ? e : e }", @@ -674,10 +609,9 @@ export const MUTATIONS: Mutation[] = [ "moves every catch behind the iteration boundary; refused deciding catches cap at " + "not-applicable, so a pass legitimately becomes n/a (mechanism C ruling)" ), - // Round D item 3. `auth-scope` fired on any property at all whose value was a caller id, wherever - // it sat, so one dead statement at the head of a body cleared it. These are the two halves: the - // wrong property name, and the right property name in an object nothing is handed. Both raised - // `settings.sso` and `settings.team`, the only two findings the check has ever produced. + // `auth-scope` once fired on any property at all whose value was a caller id, so one dead statement + // cleared it. These are the two halves: the wrong property name, and the right name in an object + // nothing is handed. wrapEveryBody( "dead-caller-scope-object", "prepend a dead object holding the caller id under an arbitrary key to every route body", @@ -690,20 +624,18 @@ export const MUTATIONS: Mutation[] = [ "const obsMapDeadUserId = { userId: user.id };", "" ), - // Round E item 3. The two entries above both prepend a DEAD object, which the handed-to-a-call - // condition refuses on its own, so neither of them would notice a future edit widening that - // condition. This one is live: the object really is handed to a real call, and the only thing - // refusing it is the callee constraint. It is also the cheaper shape to write, since a log line - // survives review in a way `const obsMapDeadUserId = ...` does not. + // The two entries above prepend a DEAD object, which the handed-to-a-call condition refuses on its + // own, so neither would notice a future edit widening that condition. This one is live and only the + // callee constraint refuses it. It is also the shape that survives review. wrapEveryBody( "log-caller-scope-userid", "prepend a logger call handed the caller id under userId to every route body", 'logger.error("obs-map", { userId: user.id });', "" ), - // C1a. `auth-boundary` matched `/^(require|authenticate)/`, so any callee at all beginning - // `require` cleared a sensitive route. These two prepend the shapes that paid: an invented guard - // and a real helper whose name merely contains "Authenticated" while it does a lookup by id. + // `auth-boundary` once matched `/^(require|authenticate)/`, so any callee beginning `require` + // cleared a sensitive route. These two prepend the shapes that paid: an invented guard, and a real + // helper whose name merely contains "Authenticated" while it does a lookup by id. wrapEveryBody( "fake-require-guard", "prepend an invented requireObsMapValidRequest() call to every route body", @@ -809,11 +741,9 @@ export const MUTATIONS: Mutation[] = [ "splice if (1 === 2) { throw e; } into every catch", (e) => `if (1 === 2) { throw ${e}; }` ), - // The returns half of the dead-prepend family. The eleven entries above put a dead THROW in; - // this one puts a dead RETURN in, which used to veto `rethrows` through the containment read - // and turn a rethrow-only clause into a swallow verdict on 11 real routes. Same blinding class - // as `dead-if-false`, so like that entry it is not additive: it fakes no signal, it used to - // destroy real signal. + // The returns half of the dead-prepend family: a dead RETURN, which used to veto `rethrows` through + // the containment read and turn a rethrow-only clause into a swallow verdict on 11 real routes. Not + // additive, since it destroys real signal rather than faking any. prependToEveryCatch( "dead-if-false-return", "preserving", @@ -832,10 +762,8 @@ export const MUTATIONS: Mutation[] = [ "splice if (e instanceof Error) { } into every catch", (e) => `if (${e} instanceof Error) { }` ), - // The if/else arm walk merges per-arm evidence by INTERSECTION, so a real classifier sitting in - // one arm only earns nothing: one arm running is a condition, not a guarantee. This is the entry - // that goes red the day someone "simplifies" the intersection to a union, at which point every - // clause in the tree earns a branch from a dead arm. Additive: it plants fake signal. + // Goes red the day someone simplifies the arm walk's INTERSECTION to a union, at which point every + // clause in the tree earns a branch from a dead arm. Additive. prependToEveryCatch( "dead-classifier-one-arm", "preserving", @@ -843,41 +771,27 @@ export const MUTATIONS: Mutation[] = [ (e) => `if (false) { if (${e} instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }` ), - // The sibling of `empty-instanceof-if`. That entry's arm is empty; this one's arm holds an exit - // that can never run, which is the same no-op written so that a containment read cannot tell the - // difference. `selectsADistinctPath` asked a plain containment question, true of - // `if (false) { return null; }`, so the test read as a real classification decision and turned a - // swallowing catch into a passing one: 80 routes and the tree from 19 to 27 when measured. - // Additive: it plants fake signal. `catchClauseEvidence`'s own `exited` flag had already been - // moved onto `containsLiveExit` for exactly this reason and the branch predicate beside it was - // left behind, which is why the shape is spelled with the corpus's own `if (false)` and not - // something exotic. + // The sibling of `empty-instanceof-if`, with an arm holding an exit that can never run: the same + // no-op written so a containment read cannot tell the difference. Worth 80 routes and the tree from + // 19 to 27 when measured. Additive. prependToEveryCatch( "dead-armed-instanceof-if", "preserving", "splice if (e instanceof Error) { if (false) { return null; } } into every catch", (e) => `if (${e} instanceof Error) { if (false) { return null; } }` ), - // The sibling the entry above does NOT close, found while closing it and filed here rather than - // fixed. Moving the ARM's exit read onto `containsLiveExit` folds a dead arm; it does not fold a - // dead CONDITION, and a condition that both references the caught binding and is provably false - // reaches the same grant. `literalTruth` cannot see it: `&&` and `||` are documented there as - // always null, deliberately, so `e instanceof Error && false` is an undecidable guard to every - // fold in the file. Closing it means widening that fold, which is a different rule with its own - // measurement, so this runs as a `KNOWN_GAPS` expected failure instead of sitting unrecorded. + // The sibling the entry above does NOT close: folding a dead ARM does not fold a dead CONDITION, and + // `literalTruth` treats `&&` as always null on purpose. Runs as a `KNOWN_GAPS` expected failure. prependToEveryCatch( "dead-conjunction-instanceof-if", "preserving", "splice if (e instanceof Error && false) { return null; } into every catch", (e) => `if (${e} instanceof Error && false) { return null; }` ), - // A finally that leaves itself by `break` cancels the try's completion, so nothing hosted in - // that tryBlock ever escapes the clause: the whole statement is a no-op. The walk's - // catchless-try entry credited it anyway, minting a branch from the hosted classifier on 80 - // routes and 8 global points when measured. Additive: it plants fake signal. The classifier is - // guarded (`if (e instanceof Error)`) rather than a bare `throw e` so `definitelyExits` cannot - // read the statement as an unconditional exit; the bare spelling trips a separate, pre-existing - // over-cut in `definitelyExits`'s try/finally case that this entry is not about. + // A finally leaving itself by `break` cancels the try's completion, so the whole statement is a + // no-op the walk's catchless-try entry once credited: 80 routes and 8 global points. Additive. The + // classifier is guarded rather than a bare `throw e`, so `definitelyExits` cannot read the statement + // as an unconditional exit and trip a separate over-cut this entry is not about. prependToEveryCatch( "dead-throw-in-cancelled-try", "preserving", @@ -886,8 +800,7 @@ export const MUTATIONS: Mutation[] = [ `do { try { if (${e} instanceof Error) { throw ${e}; } } finally { break; } } while (false);` ), - // The additive class. Everything above either takes signal away or moves it about; these put in - // signal that is not real, which is the direction the corpus was blind to. + // The additive class: signal that is not real, which is the direction the corpus was blind to. { id: "dead-classifying-try", kind: "preserving", @@ -1014,10 +927,8 @@ export const MUTATIONS: Mutation[] = [ }, }, - // The no-pass ceiling on refused (iteration-callback) catches, at tree scale. A two-element - // array literal iterates, so the boundary rule refuses this catch; it decides and cannot run - // its deciding arm (JSON.parse("0") never throws). Under any future rule that CREDITS refused - // catches, the ~261 catchless routes rise from not-applicable to pass and this entry goes red. + // The no-pass ceiling on refused catches, at tree scale: under any future rule that CREDITS them, + // the 261 catchless routes rise from not-applicable to pass and this entry goes red. wrapEveryBody( "dead-deciding-map", "prepend a dead deciding per-item catch inside a two-element .map to every route body", @@ -1064,8 +975,8 @@ export const MUTATIONS: Mutation[] = [ const previous = statements[i - 1]!; const current = statements[i]!; if (!ts.isExpressionStatement(previous) || !ts.isExpressionStatement(current)) continue; - // A directive prologue is an ExpressionStatement, and `"use client", foo();` is no longer - // a directive. That is a behaviour change, which this entry claims not to make. + // `"use client", foo();` is no longer a directive, which is a behaviour change this entry + // claims not to make. if (ts.isStringLiteral(previous.expression)) continue; if (source[previous.end - 1] !== ";") continue; edits.push({ start: previous.end - 1, end: current.getStart(), text: ", " }); @@ -1098,8 +1009,8 @@ export const MUTATIONS: Mutation[] = [ /** * The entries that add fake signal rather than removing or restructuring real signal. Named so - * `mutationCorpus.test.ts` can assert the class exists: the corpus went three rounds with this half - * of the property untested, and an empty list here is exactly that state coming back. + * `mutationCorpus.test.ts` can assert the class exists: an empty list here is the corpus going back to + * the three rounds it spent with this half of the property untested. */ export const ADDITIVE_IDS = [ "dead-classifying-try", diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 511d73178f9..214a11f2fd2 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -13,9 +13,8 @@ import { export const MARKER = ""; /** - * The commit a comment was rendered for. Data rather than something the renderers read for - * themselves: they stay pure, and a unit test or a local CLI run with no commit context renders the - * same comment without the line. + * The commit a comment was rendered for. Data rather than something the renderers read for themselves, + * so they stay pure and a run with no commit context renders the same comment without the line. */ export type CommitContext = { /** The head sha, full. Shortened for the link text here rather than by the caller. */ @@ -26,10 +25,8 @@ export type CommitContext = { const SHORT_SHA_LENGTH = 7; -/** - * Directly under the heading and before anything the report says, because the comment is sticky: - * it is edited in place across pushes, so the first question about it is which push it reflects. - */ +/** Directly under the heading, because the comment is edited in place across pushes and the first + * question about it is which push it reflects. */ function commitLines(commit: CommitContext | undefined): string[] { if (!commit) return []; return [`As of [\`${commit.sha.slice(0, SHORT_SHA_LENGTH)}\`](${commit.url}).`, ""]; @@ -37,31 +34,19 @@ function commitLines(commit: CommitContext | undefined): string[] { const MAX_CHANGED_ROWS = 15; -/** - * A mistyped directive applied tree wide renders one line per file: 87,938 characters against - * GitHub's 65,536 limit, a 422, and the workflow's error tolerance swallowing it. The cap is what - * stops the whole comment being lost to the section warning about a typo. - */ +/** A mistyped directive applied tree wide rendered 87,938 characters against GitHub's 65,536 limit, + * so the whole comment was lost to the section warning about a typo. */ const MAX_UNKNOWN_SUPPRESSION_LINES = 10; /** - * The same failure in the other section that grows with the size of the tree. `delegating` holds - * one file name per route whose body lives elsewhere, joined into a single line, and a codemod that - * moves route bodies into `.server.ts` modules is both the refactor this feature exists to notice - * and the one that makes the list tree-sized. The cap was claimed here before it was written: the - * note above used to open "every other section of this comment is bounded by construction", which - * was not true of this one. - * - * Fifteen matches the changed-entries table rather than the ten above, because a delegating file - * name is one comma-separated item rather than a line naming every known check. The bound that - * matters is the section's worst case: the longest route file name in the tree is 130 characters, - * so fifteen of those plus separators is under 2kB against GitHub's 65,536. + * The same failure in the other section that grows with the tree, since a codemod moving route bodies + * into `.server.ts` modules is both the refactor this feature exists to notice and the one that makes + * the list tree-sized. Fifteen rather than the ten above because a file name is one comma-separated + * item: the longest in the tree is 130 characters, so fifteen of those is under 2kB. */ const MAX_DELEGATED_ROUTES = 15; -// Scored checks only, same exclusion terminal.ts's scoredFailures makes: audit-trail fails almost -// every sensitive mutation today, so listing it per route would nag with something unfixable -// instead of surfacing the route-specific gaps this column exists for. +// Scored checks only, the same exclusion `scoredFailures` makes. const failingIds = (e: ScoredEntry) => scoredFailures(e).map((c) => c.id); function scoreLine(head: MapReport, base: MapReport | null): string { @@ -87,10 +72,9 @@ function scoreLine(head: MapReport, base: MapReport | null): string { const NOT_MEASURED = "not measured"; /** - * `score` is 100 for an entry no scored check applied to, a placeholder the score itself excludes - * from every mean. Rendering that 100 as a figure turned a route refactored down to a trivial body - * into a 67-point improvement, and a trivial route gaining real work into the PR's worst - * regression. So the cell says what the terminal gauge says for a null mean instead. + * `score` is a placeholder 100 for an entry no scored check applied to. Rendering that as a figure + * turned a route refactored down to a trivial body into a 67-point improvement, and a trivial route + * gaining real work into the PR's worst regression. */ const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); @@ -105,22 +89,13 @@ type ChangedRow = { baseScore: number | string; headScore: number | string; nowFailing: string[]; - /** - * Ids this pull request newly suppressed on the entry. Rendered on the route cell, because a - * suppression added to a check that was passing drops the score by round A's cap and produces a - * row with an empty "now failing" column, which is indistinguishable from a real regression: - * `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 that way. The score movement is honest, - * the row without this note was not. - */ + /** Ids this pull request newly suppressed. Rendered on the route cell, because a suppression added + * to a passing check drops the score by the cap and produces a row with an empty "now failing" + * column, indistinguishable from a real regression. */ suppressed: string[]; - /** - * How much the entry got worse, used to sort the table. A new entry has no base score to - * subtract from, so it is scored against a perfect 100: a new entry landing at 60 sorts the - * same as an existing one that dropped 40 points, which is the ordering "what needs fixing - * first" implies. Zero whenever either side is unmeasured, because there is no arithmetic to do - * between a figure and an absence; such a row is in the table to disclose the transition, not to - * claim a size for it. - */ + /** How much the entry got worse, used to sort the table. A new entry is scored against a perfect + * 100, so one landing at 60 sorts with an existing one that dropped 40. Zero whenever either side + * is unmeasured, since there is no arithmetic to do between a figure and an absence. */ drop: number; }; @@ -132,9 +107,8 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re for (const h of head.entries) { const b = baseByFile.get(h.fileName); if (!b) { - // A new entry that passes every check it was measured against has nothing to fix, which is - // what `drop: 0` means everywhere else in this table. A new entry nothing applied to is a - // different statement and still gets a row, since its 100 is a placeholder rather than a pass. + // A new entry passing every check it was measured against has nothing to fix. One nothing + // applied to still gets a row, since its 100 is a placeholder rather than a pass. if (h.measured && h.score === 100 && h.suppressed.length === 0) continue; rows.push({ routePath: h.routePath, @@ -147,10 +121,9 @@ function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; re }); continue; } - // Measured state and the suppression set are both part of what changed. A measured-to- - // unmeasured transition can leave the score at its placeholder value, and suppressing a check - // that was already failing moves no score at all, so skipping on the number alone hid both. A - // pull request whose whole purpose is to silence findings has to produce a row. + // Measured state and the suppression set are both part of what changed: a measured-to-unmeasured + // transition can leave the score at its placeholder, and suppressing an already-failing check + // moves no score at all, so skipping on the number alone hid both. const suppressed = newlySuppressed(h, b); const suppressionChanged = suppressed.length > 0 || h.suppressed.length !== b.suppressed.length; if (b.measured === h.measured && b.score === h.score && !suppressionChanged) continue; @@ -236,35 +209,11 @@ const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) /** * Whether this pull request moves the report at all, so the job can stay quiet when it does not. * - * The rule this has to satisfy is that it must be true whenever `renderPrComment` would say - * something different, because anything it misses is a change the pull request silently does not - * report. So it covers every figure the comment renders, not only the score: the global, the - * per-entry score, measured state and suppression set, an entry added or removed, a check failing - * at head that did not at base, the parse failure count, the unknown suppression warnings, and the - * audit and context gaps. - * - * `delegating` and `checkContributions` are compared outright rather than through the per-entry - * loop. Both are rendered, and both can move while every entry keeps its score: a check going from - * applicable-and-passing to not-applicable leaves an entry at 100 and changes what the CHECKS block - * says about it. - * - * The per-entry suppression set and the two gaps are the half that was missing, and it ran the - * dangerous way. Suppressing an already-failing check moves no score, no measured flag and no new - * failure, so a pull request whose entire purpose was to silence findings posted nothing, while a - * mistyped directive did post because the unknown warnings were compared. `audit-trail` going from - * fail to pass, the first audit record in the webapp, was in the same hole, and so was the CONTEXT - * figure moving behind a suppression, since that figure reads pre-suppression data. - * - * The terms overlap on purpose, and what is defended is that their union is complete rather than - * that each one is load bearing. Four are individually reachable, each with a test that fails when - * only that term is removed: the parse failure count, the unknown suppression warnings, the audit - * gap and the context gap. The global, the removed-entry check and the per-entry score are each - * shadowed by another term today, and are kept because which term shadows which depends on the - * shape of the change rather than on anything stable. - * - * `MapReport.suppressions` is the one term deliberately left out. Its two totals are summed from - * the very per-entry `suppressed` arrays the loop below compares one by one, so it cannot move - * without the loop moving. That is arithmetic rather than a happy overlap. + * This has to be true whenever `renderPrComment` would say something different, because anything it + * misses is a change the pull request silently does not report. The terms overlap on purpose, and + * what is defended is that their union is complete rather than that each one is load bearing. + * `MapReport.suppressions` is the one term deliberately left out, because its totals are summed from + * the very per-entry arrays the loop below compares. See README, "Reporting". */ export function hasDelta(head: MapReport, base: MapReport | null): boolean { if (!base) return true; @@ -291,10 +240,8 @@ export function hasDelta(head: MapReport, base: MapReport | null): boolean { return false; } -/** - * What replaces a comment whose findings a later push fixed. Going silent would leave the earlier - * comment standing with findings that no longer exist, which is worse than a redundant comment. - */ +/** What replaces a comment whose findings a later push fixed. Going silent leaves the earlier comment + * standing with findings that no longer exist. */ export function renderResolvedComment(commit?: CommitContext): string { return [ MARKER, @@ -311,9 +258,9 @@ export function renderResolvedComment(commit?: CommitContext): string { } /** - * What the job posts when the head scan did not produce a report. The alternative was a red x on - * a job that must never block a pull request, and the alternative to that was swallowing the - * failure so the only signal was a comment that never appeared. + * What the job posts when the head scan did not produce a report. The alternatives were a red x on a + * job that must never block a pull request, or swallowing the failure so the only signal was a comment + * that never appeared. */ export function renderScanFailedComment(commit?: CommitContext): string { return [ @@ -330,10 +277,7 @@ export function renderScanFailedComment(commit?: CommitContext): string { ].join("\n"); } -/** - * Pure function, no I/O: `head` and `base` are already-built reports. Matches entries across the - * two by `fileName`, the same identifier `renderJson` carries. - */ +/** Pure function, no I/O: `head` and `base` are already-built reports, matched by `fileName`. */ export function renderPrComment( head: MapReport, base: MapReport | null, diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index 9e7905388b4..c4f8158f31c 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -18,8 +18,8 @@ const processIo: Io = { err: (s) => process.stderr.write(s), }; -/** Reads and parses one report file, raising a message naming the file rather than letting an - * unreadable path or malformed JSON surface as a stack trace. */ +/** Raises a message naming the file rather than letting an unreadable path or malformed JSON surface + * as a stack trace. */ function readReport(path: string, label: string): MapReport { let raw: string; try { @@ -34,8 +34,8 @@ function readReport(path: string, label: string): MapReport { } } -/** An `--opt=value` argument, last one winning. An empty value reads as absent, since that is what - * an unset workflow expression interpolates to. */ +/** An `--opt=value` argument, last one winning. An empty value reads as absent, since that is what an + * unset workflow expression interpolates to. */ function flag(args: string[], name: string): string | undefined { const prefix = `--${name}=`; const values = args.filter((a) => a.startsWith(prefix)).map((a) => a.slice(prefix.length)); @@ -43,9 +43,8 @@ function flag(args: string[], name: string): string | undefined { } /** - * The commit the caller says this comment is for. Half a pair is rejected rather than dropped: it - * can only come from an edit to the workflow that passes one and not the other, and a comment - * silently missing the line it was supposed to gain is the failure nobody would notice. + * The commit the caller says this comment is for. Half a pair is rejected rather than dropped, since a + * comment silently missing the line it was supposed to gain is the failure nobody would notice. */ function commitFrom(args: string[]): CommitContext | undefined { const sha = flag(args, "commit-sha"); @@ -56,21 +55,13 @@ function commitFrom(args: string[]): CommitContext | undefined { } /** - * `-` or a missing second arg means no base: the CI job falls back to this when the base scan - * itself failed, so the comment still renders rather than the job going red. + * `-` or a missing second arg means no base, which is what the CI job falls back to when the base scan + * failed, so the comment still renders rather than the job going red. * - * Empty output means "post nothing". The job only comments when the pull request moves the report, - * and `--existing-comment` is how the workflow says a comment from an earlier push is already on - * the pull request: with the delta gone, that comment is replaced with a resolved state rather - * than left standing with findings that no longer exist. - * - * `--scan-failed` takes no report and prints the stale-report comment, for the case where the head - * scan produced nothing to read. `--resolved` takes no report either and prints the resolved state - * outright, for the case where the workflow knows there is nothing to compare: the paths the report - * watches did not move in this pull request at all, so nothing was scanned, and the comment an - * earlier push left has to stop showing findings that are no longer in the diff. - * - * `--commit-sha` and `--commit-url` are the commit every comment above is rendered as of. + * Empty output means "post nothing". `--existing-comment` is how the workflow says a comment from an + * earlier push is already there, so with the delta gone that comment is replaced with a resolved state + * rather than left standing. `--scan-failed` and `--resolved` take no report at all, for a head scan + * that produced nothing to read and for a run where the watched paths did not move. */ export function main(argv: string[], io: Io = processIo): number { const args = argv.slice(2); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts index 0bb904bae50..5b0df7849ac 100644 --- a/internal-packages/observability-map/src/report/terminal.ts +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -10,31 +10,15 @@ const gauge = (score: number | null) => { return `${"▰".repeat(filled)}${"▱".repeat(10 - filled)} ${String(score).padStart(3)}`; }; -/** - * Failing checks that actually feed `score`. `audit-trail` is deliberately excluded here: it is - * excluded from the score for the same reason (see `score.ts`), and every sensitive mutation fails - * it today, so folding it in would flood this list with the same finding repeated 46 times instead - * of the fixable, route-specific gaps the list exists to surface. That gap is reported once, as - * `AUDIT`, below. - */ +/** Failing checks that feed `score`, so never `audit-trail`: every sensitive mutation fails that + * today, and it is reported once as the `AUDIT` line instead. */ export const scoredFailures = (e: ScoredEntry) => e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); /** - * An entry whose only finding is `request-context`. 401 of 412 entry points fail that check, so - * listing each one turns the fix list into a single house-style finding repeated, which is the - * reason `audit-trail` is kept out of the list too. Collapsed into the `CONTEXT` figure instead. - * An entry that fails something else as well stays in the list with all of its findings, so a - * route like `/account/tokens` still shows the request-context gap alongside the rest. - */ -/** - * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, - * then by name. Exported because `prComment.ts` renders the same list with different bullets and - * had a byte-identical copy of this filter and sort, in a file that already imports - * `scoredFailures` and `contextOnly` from here. - * - * `contextOnly` routes are left out because `request-context` fails almost everything, so a list - * headed by three of them tells a reader nothing they cannot read off the gap figure. + * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, then + * by name. Exported because `prComment.ts` renders the same list with different bullets and had a + * byte-identical copy of this filter and sort. */ export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => entries @@ -46,15 +30,17 @@ export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => a.fileName.localeCompare(b.fileName) ); +/** An entry whose only finding is `request-context`, which fails almost everything, so it is + * collapsed into the `CONTEXT` figure rather than listed. An entry that fails something else as well + * keeps all of its findings and stays in the list. */ export const contextOnly = (e: ScoredEntry) => { const failures = scoredFailures(e); return failures.length === 1 && failures[0]!.id === "request-context"; }; /** - * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. Empty when every - * directive named a real check. A typo suppresses nothing, so without this the author reads the - * finding as acknowledged and the tool goes on reporting it with no hint why. + * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. A typo suppresses nothing, + * so without this the author reads the finding as acknowledged and the tool goes on reporting it. */ export function unknownSuppressionLine(fileName: string, ids: string[]): string { return ( @@ -70,20 +56,10 @@ export function unknownSuppressionLines(report: MapReport): string[] { } /** - * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when - * there is nothing to report, i.e. no sensitive mutation exists. - * - * One shape for every count, and no branch on the count, because the branch is what carried the - * bug. A zero used to print "No audit helper exists in the webapp", which is false: the helper - * exists and `AUDIT_SYMBOLS` names it, `apps/webapp/app/models/admin.server.ts` writes - * `prisma.impersonationAuditLog.create(...)`, and `webappSymbols.test.ts` proves those symbols - * resolve. The count was already correct, so the sentence was the only wrong thing and it is gone - * rather than reworded. A zero here means nothing reached the helper, which is what - * "0 of N record an actor" already says. - * - * The full-tree scan reads 3 of 49 today, so the zero branch is not taken and nobody sees it. It is - * one `--routes=` away from being taken, and one reshaped impersonation route away on the full - * tree, which is why removing it beats leaving it unreachable. + * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when no + * sensitive mutation exists. One shape for every count and no branch on the count, because the branch + * is what carried the bug: a zero used to print "No audit helper exists in the webapp", which is + * false. "0 of N record an actor" already says what a zero means. */ export function auditLine(report: MapReport): string | null { const { sensitiveMutations, withAudit } = report.auditGap; @@ -110,13 +86,9 @@ export function contextLine(report: MapReport): string | null { } /** - * The DELEGATED lines, shared with `prComment.ts`. Empty when every route's body is in its own - * file. Worded as a shortfall rather than a note: these routes left the denominator and no check - * looked at any of them. - * - * `limit` caps how many file names are named, for the caller that has a size limit to respect. The - * count in front of the list is always the full one, so a capped line still reports the real - * shortfall and only shortens the evidence. The terminal passes no limit and prints them all. + * The DELEGATED lines, shared with `prComment.ts`. Worded as a shortfall rather than a note: these + * routes left the denominator and no check looked at any of them. `limit` shortens the evidence for a + * caller with a size limit; the count in front of the list is always the full one. */ export function delegatedLines(report: MapReport, limit = Infinity): string[] { if (report.delegating.length === 0) return []; @@ -131,10 +103,8 @@ export function delegatedLines(report: MapReport, limit = Infinity): string[] { ]; } -/** - * The CHECKS block: what the composite is made of. `sole` is the figure that says most, since an - * entry only one scored check applies to scores 0 or 100 on that one boolean. - */ +/** The CHECKS block: what the composite is made of. `sole` is the figure that says most, since an + * entry only one scored check applies to scores 0 or 100 on that one boolean. */ export function checkContributionLines(report: MapReport): string[] { if (report.checkContributions.every((c) => c.applicable === 0)) return []; const width = Math.max(...report.checkContributions.map((c) => c.id.length)); @@ -235,8 +205,8 @@ export function renderTerminal(report: MapReport): string { } lines.push(""); - // Not one flattering number: an entry with nothing applicable is not the same as an entry that - // passed, and lumping them together counted routes as solid for doing nothing. + // Two figures rather than one, because lumping "nothing applicable" in with "passed" counted routes + // as solid for doing nothing. const clean = report.entries.filter((e) => e.measured && scoredFailures(e).length === 0).length; lines.push( `no findings: ${clean} passed every applicable check, ${report.unmeasured} had none to apply` diff --git a/internal-packages/observability-map/src/routeExports.ts b/internal-packages/observability-map/src/routeExports.ts index a53b2ccf979..588c8fc8956 100644 --- a/internal-packages/observability-map/src/routeExports.ts +++ b/internal-packages/observability-map/src/routeExports.ts @@ -4,10 +4,8 @@ export type ExportName = "loader" | "action"; /** * One export of a route file, carrying that export's own evidence and nothing from the other one. - * - * Every field here has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the - * mistake this type exists to make hard. `calleeNames` is the union of both bodies, `hasTryCatch` - * is true if either has one, and `statementCount` counts both. + * Every field has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the mistake + * this type exists to make hard. */ export type RouteExport = { name: ExportName; @@ -28,19 +26,11 @@ export type RouteExport = { }; /** - * The exports this route file declares, in `loader`, `action` order. - * - * One enumeration for the whole package, because two checks asking the same per-export question - * each grew their own. `auth-scope`'s `builderExports` and `auth-boundary`'s `guardedExports` were - * hand-maintained `[loader, action]` literals in adjacent files, reading the same six - * `loaderX`/`actionX` field pairs, with different tests for whether an export was there at all: - * one used `hasLoader`/`hasAction` and the other inferred it from a non-null initializer callee. - * Adding a seventh per-export fact meant editing both, and this whole branch is a record of what - * happens when a rule lives in two places and only one gets the fix. + * The exports this route file declares, in `loader`, `action` order. One enumeration for the whole + * package, because the two per-export checks each grew their own hand-maintained `[loader, action]` + * literal, disagreeing about how to tell whether an export was there at all. * - * Absent exports are not returned, so a caller never has to remember to filter them: the shape of - * the bug in `auth-boundary` was crediting an export for something the other one did, and a list - * that only contains real exports is one fewer way to write it. + * Absent exports are not returned, so a caller never has to remember to filter them. */ export function routeExports(ep: EntryPoint): RouteExport[] { const all: RouteExport[] = [ diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index 78b48d54e08..e58813953e9 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -73,55 +73,32 @@ function propertyPath(expr: ts.Expression): string | null { } /** - * Expressions that are the caller's own id, in the spellings the route tree uses: - * `userId: authentication.userId` under the API builders, `userId: user.id` under the dashboard - * builders, and the `authenticationResult` and `sessionAuth` variants. Read by `auth-scope` as - * evidence that the handler narrowed its query to whoever is asking. - * - * Anchored at both ends. The root has to be one of the auth bindings a builder hands the handler, - * and the last segment has to be an identity field, so `user.name` is not a scope and neither is - * `run.userId`, which is a resource's owner rather than the caller. + * Expressions that are the caller's own id, in the spellings the route tree uses. Anchored at both + * ends: the root is one of the auth bindings a builder hands the handler and the last segment is an + * identity field, so `user.name` is not a scope and neither is `run.userId`, which is a resource's + * owner rather than the caller. */ const CALLER_ID_PATH = /^(authentication|authenticationResult|auth|sessionAuth|user)(\.[A-Za-z0-9_$]+)*\.(userId|id|actor)$/; /** - * Property names that mean the value is being used to say WHOSE, rather than merely carrying the - * caller's id around. Read off the tree: of the ten names that take a caller-id value in - * `apps/webapp/app/routes`, these are the tenant and identity fields, and `sub`, `value` and - * `consumerId` are the three that are not. `anything: user.id` is what a mutation writes, and it - * does not match. + * Property names that mean the value says WHOSE rather than merely carrying the caller's id around. + * Read off the tree: of the ten names taking a caller-id value, `sub`, `value` and `consumerId` are + * the three left out. */ const CALLER_ID_FIELD = /^(id|userId|user|memberId|orgMemberId|createdBy|createdByUserId|environmentId|runtimeEnvironmentId|organizationId|orgId|projectId)$/; /** - * Callees that are handed the caller's id and cannot narrow a read with it: the log line and the - * response body. Both take the very `{ userId: user.id }` object a query filter takes, so crediting - * them let one log statement clear `auth-scope` for a whole export. That is cheaper than the - * actor-argument residual `checks/authScope.ts` discloses, and it lands on the one check whose - * purpose is catching cross-org exposure. - * - * The shape is already in the tree rather than hypothetical: `engine.v1.dev.runs...attempts.start` - * writes `logger.error("...", { environmentId: authentication.environment.id })` beside the - * `runStore.findRun` that earns that export its credit honestly. Loggers account for 13 of the - * caller-id sites under `apps/webapp/app/routes` and the two response serializers for 2 more. - * - * A denylist of sinks rather than an allowlist of query callees, and that is a measurement rather - * than a preference. 72 distinct callees are handed a caller id across the route tree, running from - * `prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare - * `regenerateApiKey` and `resolveOrganizationForApiUser`. No name pattern separates those from - * `sendToPlain`, so an allowlist would accuse whichever route named its helper next, and a wrong - * accusation is the failure this check cannot afford. Refusing the sinks that are known not to - * scope shrinks the residual without pretending to close it: `someHelper({ userId: user.id })` that - * ignores its argument still credits, which needs types the scanner does not have. + * Callees handed the caller's id that cannot narrow a read with it: the log line and the response + * body. A denylist of sinks rather than an allowlist of query callees, which is a measurement and not + * a preference. See README, "What auth-scope reads as scoping". */ const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; /** * Whether a callee could plausibly narrow a read with the object it is handed. A callee with no - * readable name of its own is credited: refusing it would ACCUSE the route, and under-crediting the - * constraint beats accusing a route that is fine. + * readable name of its own is credited, because refusing it would ACCUSE the route. */ function couldScopeAQuery(callee: ts.Expression): boolean { const text = calleeText(callee); @@ -131,19 +108,12 @@ function couldScopeAQuery(callee: ts.Expression): boolean { /** * Whether the object literal holding this property is handed to a call that could scope a query, - * through any depth of nesting: `findMany({ where: { members: { some: { userId } } } })` is, and - * `const unused = { userId };` is not. Arrays count, so `{ OR: [{ userId }] }` still reaches its - * call. - * - * Two things are refused. A filter built and dropped, which is the dead-object shape - * `dead-caller-scope-object` and `dead-caller-scope-userid` cover. And a filter handed to a callee - * that provably cannot read with it, which is `NON_SCOPING_CALLEE` and which the corpus entry - * `log-caller-scope-userid` covers at tree scale. + * through any depth of nesting. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. * - * What this does NOT refuse is a filter handed to a named call that ignores it: - * `String({ userId: user.id });` reads as scoping, the same way `try { String(0); }` reads as error - * handling, and for the same reason. Knowing whether an arbitrary callee uses the argument needs - * types the scanner does not have. + * Refuses a filter built and dropped (`dead-caller-scope-object`, `dead-caller-scope-userid`) and + * one handed to a `NON_SCOPING_CALLEE` (`log-caller-scope-userid`). Does NOT refuse a filter handed + * to a named call that ignores it: `String({ userId: user.id })` reads as scoping, the same way + * `try { String(0); }` reads as error handling. */ function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { let node: ts.Node = property; @@ -164,32 +134,13 @@ function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { } /** - * Whether any handler in `fns` assigns the caller's own id to an object-literal property, the - * `where: { members: { some: { userId: authentication.userId } } }` and - * `presenter.call({ userId: user.id })` shapes. + * Whether any handler in `fns` assigns the caller's own id to an object-literal property. Three + * conditions, all load bearing: README, "What auth-scope reads as scoping". * - * Per export rather than per entry point, and that is the whole point of computing it here instead - * of in the main body walk. A file whose loader narrows itself to the caller and whose action does - * not is not a scoped route, and the entry-point-wide version said it was: it passed - * `_app.orgs.$organizationSlug.settings.team/route.tsx`, whose loader calls - * `TeamPresenter.call({ userId: user.id })` while its action resolves the target org from the URL - * slug and gates only on `ability.can`. - * - * Nested functions are walked, since a filter built inside a callback still filters. Same-file - * helpers are NOT followed, unlike the main walk: a route that computes its filter in a helper is - * reported as unscoped. Nothing in the tree does. - * - * Three conditions, and the first version of this had only the middle one, which made the whole - * check free to defeat. Prepending `const __unused = { anything: user.id };` to every body raised - * `settings.sso` and `settings.team`, the only two findings `auth-scope` has ever produced and both - * confirmed cross-org exposures, because any property at all taking a caller id counted wherever it - * sat. `dead-caller-scope-object` and `dead-caller-scope-userid` in the mutation corpus are the two - * halves of that shape. - * - * So the property NAME has to be an identity field, and the object it sits in has to be handed to a - * call that could scope a query. The third condition is what stops `logger.error("create failed", - * { userId: user.id })`, written anywhere in a builder-wrapped handler, clearing the check for that - * export. See `isHandedToAScopingCall` for what that does and does not refuse. + * Per export rather than per entry point, which is why it is computed here rather than in the main + * body walk. Nested functions are walked, since a filter built inside a callback still filters. + * Same-file helpers are NOT followed, unlike the main walk, so a route computing its filter in a + * helper is reported as unscoped. Nothing in the tree does. */ function scopesByCallerIn(fns: Iterable): boolean { let found = false; @@ -270,8 +221,7 @@ const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$| /** * Constructors that parse untrusted input and throw when it is malformed. Deliberately short: any - * constructor at all would mean `new BranchesPresenter()` or `new Set(...)` excuses a catch that - * guards ordinary work, which was true of 77 try blocks in the route tree. + * constructor at all excuses a catch guarding ordinary work, which was true of 77 try blocks. */ const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); @@ -285,13 +235,9 @@ function isParseCall(node: ts.Node): boolean { } /** - * Body reads: the thing a parse guard waits for before it parses. `request.json()` is in - * `PARSE_CALLEE` already because it reads and parses in one call, and these are the same operation - * with the parse written separately, `const raw = await request.text(); new RegExp(raw);`. - * - * Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which is what bounds it: a body - * read on its own still does not make a try block a parse guard, so the widest this list can do is - * let a block that already parses also read the thing it parses. + * Body reads: what a parse guard waits for before it parses, when the parse is written separately + * from the read. Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which bounds it: a + * body read on its own does not make a try block a parse guard. */ const BODY_READ_METHODS = new Set(["text", "formData", "arrayBuffer", "blob", "bytes"]); @@ -302,14 +248,9 @@ function isBodyRead(node: ts.Node): boolean { } /** - * Syntax that can raise. Everything a try block might do that produces something for a catch clause - * to catch: a call, a construction, a tagged template, an `await` or `yield` (the awaited promise - * rejects), a member access (the base may be null or undefined), a `throw`, an iteration (the - * iterator protocol raises on a non-iterable), and `instanceof`/`in` (a TypeError on a non-object - * right side). - * - * See `guardedWork` for what is NOT on this list and why that is a disclosed residual rather than - * an oversight. + * Syntax that can raise, i.e. anything a try block might do that produces something for a catch + * clause to catch. A whitelist, so it also misses real raising code; `guardedWork` has both + * directions of that residual. */ function canRaise(node: ts.Node): boolean { return ( @@ -330,45 +271,17 @@ function canRaise(node: ts.Node): boolean { } /** - * What the guarded region does, in the three terms `error-classification` needs. - * - * `guardsParse` is whether anything in it parses at all. A `new URL(x)` counts and has to be read - * as a `ts.isNewExpression` here, because the call-callee scan that builds `calleeNames` never sees - * it. - * - * `awaitsOnlyParse` is whether everything the block waits for is a parse or a read of the body it - * parses. Awaiting is the signal, not calling: the calls that prepare a parse's input are ordinary - * synchronous string work (`matchPattern.startsWith("(?i)")`, `.slice(4)` before a `new RegExp`), - * and refusing those refuses four of the tree's clearest guards, while the swallow this has to - * catch reaches a service: `try { const body = await request.json(); return await - * handleEverything(body); }`. - * - * `canRaise` is whether the block does anything at all that could reach the clause. A clause whose - * try block cannot raise is not error handling, and reading one as classification paid 50 points a - * route to anyone willing to prepend `try { 0; } catch (e) { if (e instanceof Error) { return - * json(x, { status: 400 }); } throw e; }` to a body: on the real tree that took the global from 15 - * to 42 and raised 224 routes when it was measured, before round C moved the baseline to 19. More - * than every other shape found on this branch put together, and still true at today's figures, which - * `dead-classifying-try-with-call` shows live at 19 to 44. `dead-classifying-try` in the mutation - * corpus is the refused version. + * What the guarded region does, in the three terms `error-classification` needs. Each term's rule + * and measurement: README, "Catch evidence, per clause". * - * What this refuses is `try { 0; }` and nothing cleverer. `canRaise` accepts ANY call, member - * access or `in`, and none of those has to be able to throw, so one inert call defeats the rule: - * `try { String(0); } catch (e) { if (e instanceof Error) { return json(x, { status: 400 }); } throw - * e; }` reads as classification and takes the tree from 19 to 44, exactly as `try { 0; }` did. - * `dead-classifying-try-with-call` in the mutation corpus is that shape, running as an expected - * failure. Telling a call that can throw from one that cannot needs types the scanner does not have, - * so the rule closes the shape found rather than the family it belongs to. Read the docstrings that - * point here as "refuses `try { 0; }`", never as "an unreachable catch cannot be credited". - * - * The list also misses things that CAN raise, which is the safe direction, and the misses matter - * because a real clause can be dropped by one: a destructuring declaration (`const { a } = undefined` - * throws), a temporal-dead-zone read (`try { const x = later; }`), a coercion that raises - * (`try { const x = 1 + someSymbol; }`) and a `delete` on a frozen object all read as unable to - * raise. + * Two residuals in opposite directions, both live. `guardCanRaise` refuses `try { 0; }` and nothing + * cleverer, because `canRaise` accepts any call at all, so `try { String(0); }` reads as + * classification and takes the tree from 19 to 44: `dead-classifying-try-with-call`, the corpus's + * expected failure. And `canRaise` misses code that CAN raise, a destructuring declaration and a + * temporal-dead-zone read among them, which is why `guardMayRaise` exists beside it. * * Nested function bodies are skipped throughout: a callback written inside the try is not work the - * try is guarding on this pass through. A `throw` inside one is not either, which is deliberate. + * try is guarding on this pass through, and a `throw` inside one is not either. */ function guardedWork(tryBlock: ts.Block): { guardsParse: boolean; @@ -406,11 +319,8 @@ function containsInstanceOf(node: ts.Node): boolean { } /** - * Whether a binding name pattern declares `target`, recursively: a plain `error`, a destructured - * `{ error }` (shorthand) or `{ code: error }` (renamed), an array pattern `[error]`, and any of - * those nested inside another. A destructured parameter or declaration re-declares the name just as - * completely as a plain one does, so a shadow check that only recognised `ts.isIdentifier` missed - * every destructured shape, function parameters and variable declarations alike. + * Whether a binding name pattern declares `target`, recursively, including every destructured shape: + * a shadow check that only recognised `ts.isIdentifier` missed all of them. */ function bindingDeclares(name: ts.BindingName, target: string): boolean { if (ts.isIdentifier(name)) return name.text === target; @@ -438,14 +348,10 @@ function declaresInScope(statements: readonly ts.Statement[], name: string): boo } /** - * Whether `node` contains a genuine read of the given catch binding, e.g. `e` in `e instanceof X` - * or `error.code`. An identifier only counts when it is a real reference. Two shapes share the - * binding's text without reading it: the property side of a member expression (`fallback.error`) - * and an object literal key (`{ error: true }`), both excluded by checking which side of the - * parent node the identifier sits on. A name re-declared in a nested scope, as a function or catch - * parameter (including a destructured one) or as a var/let/const/function/class in a block, refers - * to that declaration instead, so the walk stops at the boundary that re-declares it rather than - * crediting the outer binding. + * Whether `node` contains a genuine read of the given catch binding. Two shapes share the binding's + * text without reading it, the property side of a member expression and an object literal key, and a + * name re-declared in a nested scope refers to that declaration instead, so the walk stops at the + * boundary that re-declares it. */ function referencesBinding(node: ts.Node, bindingName: string): boolean { if (ts.isIdentifier(node) && node.text === bindingName) { @@ -484,24 +390,16 @@ function normalizedText(node: ts.Node): string { } /** - * Whether a conditional expression tests the error to pick what the clause does, rather than to - * word what it says. The caller only offers it the whole value of a `return`/`throw`, so - * `return e instanceof Response ? e : json({}, { status: 500 })` reaches here and - * `return json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 })` does not. - * The second is message formatting: every error leaves by the same path. - * - * Goes through `referencesBinding`, the same predicate the `if`/`switch` check uses, rather than - * accepting any `instanceof` in the condition: an `instanceof` that never reads the caught binding - * is not a decision made on the error, and a bindingless catch has nothing here to reference. + * Whether a conditional expression tests the error to pick what the clause does, rather than to word + * what it says. The caller only offers it the whole value of a `return`/`throw`, so + * `return json({ error: e instanceof Error ? e.message : String(e) })` does not reach here: that is + * message formatting and every error leaves by the same path. * - * The two arms also have to differ, which is the same requirement `selectsADistinctPath` makes of - * an `if`. `return e instanceof Error ? (X) : (X)` is a test whose outcome is the same either way, - * and it was worth 50 points a route; `same-arms-ternary` in the mutation corpus is the tree-scale - * version, and `scan.test.ts` has the unit case. The throw path is held to the same rule by - * `wrap-body-in-same-arms-throw-ternary`, which would take every route in the tree to a pass if it - * were not. Parentheses and whitespace are stripped - * before the comparison, so the shape has to differ in something a reader would call a difference. - * The residual both branch tests share is stated once, on `selectsADistinctPath`. + * The two arms have to differ, the same requirement `selectsADistinctPath` makes of an `if`, with + * parentheses and whitespace stripped. `same-arms-ternary` and + * `wrap-body-in-same-arms-throw-ternary` are the tree-scale versions, the second of which would take + * every route in the tree to a pass. The residual both branch tests share is on + * `selectsADistinctPath`. */ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { if (bindingName === null) return false; @@ -512,8 +410,8 @@ function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string /** * Which bare (unlabelled) jumps, at this point in the recursion, leave the statement list the - * question is being asked about. A bare jump targets the nearest enclosing construct of its kind, - * so descending past one of those targets changes the answer for the jumps it captures. + * question is being asked about. A bare jump targets the nearest enclosing construct of its kind, so + * descending past one of those changes the answer for the jumps it captures. */ type BareJumps = { break: boolean; continue: boolean }; @@ -521,41 +419,25 @@ type BareJumps = { break: boolean; continue: boolean }; * encloses the list. */ const ESCAPES: BareJumps = { break: true, continue: true }; -/** A `do` body, asked about from the list the `do` sits in. `break` ends the loop and `continue` - * goes to the condition, and both of those reach the statement written after the `do`. */ +/** A `do` body, asked about from the list the `do` sits in. Both jumps reach the statement written + * after the `do`, so neither leaves that list. */ const IN_DO_BODY: BareJumps = { break: false, continue: false }; /** * A statement that leaves the statement list it sits in on every path through itself, so anything - * after it in the same list never runs. - * - * Recognises a nested construct, not only a bare `return`/`throw`/`break`/`continue`. Recognising - * only the bare form is what let a dead `throw error;` count as a rethrow when the statement before - * it was a block, a `do` body or an `if`/`else` that returned; `dead-throw-after-*` in the mutation - * corpus is that family, and `scan.test.ts` has one case per construct. + * after it in the same list never runs. Recognises a nested construct and not only the bare jump + * forms, which is what stopped a dead `throw error;` counting as a rethrow (`dead-throw-after-*`). * * A bare `break` or `continue` only counts where it actually leaves the list, which is what `jumps` - * carries. A `break` inside a switch clause targets the switch, so a switch whose clauses all break - * falls through to the statement after it and does NOT exit; reading that break as an exit accused - * `catch (e) { switch (e.code) { ... break; } throw e; }` of swallowing a rethrown error, which is - * both a false verdict and a detail line that says the opposite of what the route does. A `continue` - * inside a switch clause targets an enclosing loop instead, which the switch cannot be, so it is - * inherited rather than dropped: dropping it would stop - * `do { switch (x) { default: continue; } throw e; } while (c)` cutting a throw that really is dead. - * `break and continue inside the construct they target` in `scan.test.ts` holds both halves. - * - * A labelled `break`/`continue` always counts. Its target has to enclose the statement list, since - * nothing between the list and the jump can carry the label: `definitelyExits` answers false for a - * labelled statement, so the recursion never descends through one. + * carries: a `break` in a switch clause targets the switch, a `continue` targets an enclosing loop + * that the switch cannot be. `break and continue inside the construct they target` holds both + * halves. A labelled jump always counts, since nothing between the list and the jump can carry the + * label. * - * A sound under-approximation. `if` without an `else` (unless its guard is the literal `true` - * keyword, the one condition this function folds), a labelled statement (a `break` to the label - * escapes it) and every other loop form answer false, because none of them is guaranteed to run its - * body. That extends to a `do` that never falls through, `do { continue; } while (true)`, which is - * false for the same reason `while (true) { }` always was: separating it from - * `do { continue; } while (c)` means folding a LOOP condition, which this function still does not - * do. Saying false when the truth is true only leaves a later statement in the list, which - * is the direction that withholds evidence rather than inventing it. + * A sound under-approximation: everything not listed answers false, including a `do` that never + * falls through, since separating that from `do { continue; } while (c)` means folding a loop + * condition. Saying false when the truth is true only leaves a later statement in the list, which + * withholds evidence rather than inventing it. */ function definitelyExits(statement: ts.Statement, jumps: BareJumps = ESCAPES): boolean { if (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) return true; @@ -567,12 +449,9 @@ function definitelyExits(statement: ts.Statement, jumps: BareJumps = ESCAPES): b // A `do` body runs before its condition is ever read. if (ts.isDoStatement(statement)) return definitelyExits(statement.statement, IN_DO_BODY); if (ts.isIfStatement(statement)) { - // A guard that is exactly the `true` keyword always takes its then-arm, so the statement - // definitely exits iff that arm does, with or without an else. Keyword-exact, same spelling - // rule as the walk's `if (true)` entry and for the same reason: this GRANTS a reachability - // cut, and a wrong grant pays. `cuts a dead trailing statement after an if true that exits` - // is the pin; `dead-throw-after-if-true` and `dead-branch-after-if-true` in the mutation - // corpus are the tree-scale versions. + // Keyword-exact, the same spelling rule as the walk's `if (true)` entry and for the same + // reason: this GRANTS a reachability cut and a wrong grant pays. + // `cuts a dead trailing statement after an if true that exits` is the pin. if (unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword) { return definitelyExits(statement.thenStatement, jumps); } @@ -614,19 +493,14 @@ function reachableStatements(statements: readonly ts.Statement[]): readonly ts.S } /** - * Whether the tree rooted at `node` contains a `break` or `continue` that would leave it, i.e. a - * jump no construct INSIDE `node` captures. What the catch walk asks of a finally block before - * entering the tryBlock beside it: a finally that completes abruptly cancels the try's completion, - * so a throw in that tryBlock never leaves the clause and crediting it minted evidence - * (`reads a throw a finally break discards as no rethrow` and its continue and switch-hosted - * siblings pin the refusal; `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale - * shape, 80 routes when measured). + * Whether the tree rooted at `node` contains a `break` or `continue` that would leave it. What the + * catch walk asks of a finally block before entering the tryBlock beside it, since a finally that + * completes abruptly cancels the try's completion (`dead-throw-in-cancelled-try`). * - * A containment read, not a liveness one, on purpose: the caller is deciding whether to GRANT - * credit and a wrong grant pays, so a jump that only may run still refuses - * (`refuses the tryBlock when the finally only may break`). Two over-approximations in the same - * direction: a labelled jump always counts, even when its label sits inside `node`, and a `return` - * is not looked for here because the returns veto already reads it off the whole statement. + * A containment read and not a liveness one, on purpose: the caller is deciding whether to GRANT + * credit, so a jump that only may run still refuses + * (`refuses the tryBlock when the finally only may break`). A `return` is not looked for here + * because the returns veto already reads it off the whole statement. */ function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolean { if (ts.isFunctionLike(node)) return false; @@ -651,17 +525,14 @@ function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolea } /** - * Literal truthiness of a guard expression: true, false, or null when not decidable from the - * token alone. Only literal tokens fold; an identifier, call, bigint, `&&`, `||` or a template - * literal with substitutions is always null, so a live guard can never be read as dead. The - * always-true side is pinned by `still refuses an error test after an always-true spelling that - * throws` and the fall-through slice by `reads a switch fall-through onto a live return as live`. + * Literal truthiness of a guard expression: true, false, or null when not decidable from the token + * alone. An identifier, call, bigint, `&&`, `||` or a template with substitutions is always null, so + * a live guard can never be read as dead. That is deliberate and it is what leaves + * `dead-conjunction-instanceof-if` open. Pinned by `still refuses an error test after an always-true + * spelling that throws`. */ function literalTruth(expr: ts.Expression): boolean | null { const target = unwrap(expr); - // The five bare-literal kinds are `literalValue`'s list, not a second copy of it: this was the - // same five node-kind tests written out three lines above the function that already had them, - // differing only in returning the truthiness rather than the value. const literal = literalValue(target); if (literal !== undefined) return Boolean(literal); if (ts.isPrefixUnaryExpression(target) && target.operator === ts.SyntaxKind.ExclamationToken) { @@ -704,25 +575,13 @@ function tryBlockMayThrow(block: ts.Block): boolean { } /** - * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch - * does not already rule out. A plain containment walk, minus the hits it can prove never run: - * `if (false) { throw e; }` contains a throw and can never run one. + * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch does + * not already rule out. Strictly subtractive against a plain containment read, which is what lets its + * two callers read it for opposite purposes: see README, "Two folds, pointing opposite ways". * - * Folds literal guards only, so wherever `literalTruth` cannot decide, every hit the plain walk - * would have found is still found. That makes this strictly subtractive against containment, which - * is what lets both of its callers read it for opposite purposes: - * - * - `catchClauseEvidence`'s `exited` flag, where a hit BLINDS the walk to whatever follows. - * Containment blinded it on a dead statement, so prepending one to a deciding clause turned its - * pass into a swallow verdict on 78 real routes. Subtracting dead hits only ever un-blinds. - * - `selectsADistinctPath`, where a hit GRANTS a branch. Containment granted one for an arm whose - * only exit was dead, which is `dead-armed-instanceof-if` in the mutation corpus, measured at 80 - * routes and the tree from 19 to 27. Subtracting dead hits only ever withholds. - * - * The `exited` half is pinned by the mirror twins under `dead and deferred code prepended to a - * deciding catch does not blind it` (recovered) and the `BRANCH_EXITED` family (refusing). The - * `selectsADistinctPath` half is pinned by `an arm whose only exit is dead decides nothing` and its - * siblings, plus the corpus entry. + * The `exited` half is pinned by `dead and deferred code prepended to a deciding catch does not blind + * it` and the `BRANCH_EXITED` family; the `selectsADistinctPath` half by `an arm whose only exit is + * dead decides nothing` and `dead-armed-instanceof-if`. */ function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean { const walk = (node: ts.Node): boolean => { @@ -772,13 +631,11 @@ function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean return clauses.slice(matched).some((c) => c.statements.some(walk)); } if (ts.isTryStatement(node)) { - // A finally that always completes abruptly (a return, a throw, or a jump out of the block) - // supersedes the try's and the catch's completion: an exit written in either never leaves - // the statement, so only the finally's own statements stay live. Folded only when - // `definitelyExits` can prove it; a conditional jump keeps the containment answer, the - // direction that refuses credit rather than inventing it. Without this fold the - // `dead-throw-in-cancelled-try` prepend blinded the walk to every real classification below - // it (`keeps the classification after a finally-break no-op`). + // A finally that always completes abruptly supersedes the try's and the catch's completion, so + // only its own statements stay live. Folded only where `definitelyExits` can prove it; a + // conditional jump keeps the containment answer, which refuses credit rather than inventing + // it. Without this the `dead-throw-in-cancelled-try` prepend blinded the walk to every real + // classification below it (`keeps the classification after a finally-break no-op`). if (node.finallyBlock !== undefined && definitelyExits(node.finallyBlock)) { return walk(node.finallyBlock); } @@ -804,31 +661,18 @@ function containsLiveReturn(node: ts.Node): boolean { /** * Whether an `if`/`switch` sends at least one arm somewhere the others do not go, by returning or - * throwing from inside it. `if (e instanceof Error) { }` and `if (e instanceof Error) { log(e); }` - * both fail this: every error still leaves the clause by the same path afterwards, so the test - * changed the wording and not the outcome. The empty-body form was the cheapest no-op in the tool, - * worth 50 points a route; `empty-instanceof-if` in the mutation corpus is the tree-scale version. + * throwing from inside it. `if (e instanceof Error) { }` fails this, since every error still leaves + * by the same path afterwards (`empty-instanceof-if`). Two textually identical arms do not count, + * the same comparison `selectsAnErrorPath` makes of a ternary. * - * An `if`/`else` whose two arms are textually identical does not count, the same comparison - * `selectsAnErrorPath` makes of a ternary's arms. - * - * The exit an arm is credited for has to be a LIVE one, `containsLiveExit` and never a plain - * containment read. `if (e instanceof Error) { if (false) { return null; } }` contains an exit that - * can never run, so under containment it read as a real decision and took a swallowing catch to a - * pass for the price of a mechanical edit: 80 routes and the tree from 19 to 27 when measured. The - * same liveness rule had already been put on `catchClauseEvidence`'s `exited` flag, for the same - * eleven dead spellings, and this predicate beside it kept the containment read. `an arm whose only - * exit is dead decides nothing` and its siblings are the unit pins; `dead-armed-instanceof-if` in - * the mutation corpus is the tree-scale version. Being subtractive against containment, the fold - * can only ever withhold a branch, never invent one, so a live arm reads exactly as it did. + * The exit an arm is credited for has to be a LIVE one, never a plain containment read + * (`an arm whose only exit is dead decides nothing`, `dead-armed-instanceof-if`). * * The residual both branch tests share, stated here once for both: two arms that produce the same - * outcome by different spellings still read as a real decision. - * `if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides - * nothing, and so does the `if` with no `else` whose arm returns what the statement after it - * returns. Telling those apart needs the produced values compared for meaning rather than for text, - * which is a different kind of analysis from anything else in this file. The textual comparison is - * the cheapest thing that catches the copy-paste form, which is the one a mutation produces. + * outcome by different spellings still read as a real decision, e.g. + * `if (e instanceof Error) { return json(x); } return Response.json(x);`. Telling those apart needs + * the produced values compared for meaning rather than for text, which is a different kind of + * analysis from anything else in this file. */ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { if (ts.isIfStatement(statement)) { @@ -839,95 +683,39 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b } return containsLiveExit(statement.thenStatement); } - // Per clause statement rather than over the whole switch, so a live exit in any clause counts - // whatever the discriminant is. Reading the switch as one node would hand `containsLiveWhere`'s - // discriminant fold a `switch (e.code)` it cannot decide, which changes nothing, and a - // `switch (1)` it can, which is not this predicate's business: an unreachable CLAUSE is caught - // by the same fold one level down, and the statement is only reached at all when its condition - // references the caught binding. + // Per clause statement rather than over the whole switch: reading the switch as one node would + // hand `containsLiveWhere`'s discriminant fold a `switch (1)` it can decide, which is not this + // predicate's business, since an unreachable CLAUSE is caught by the same fold one level down. return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsLiveExit)); } /** * What a catch clause does with the error, beyond the fact that it caught one. * - * Both answers are read off the clause's own guaranteed path. The governing rule: the walk may - * enter a construct exactly where the entered statements are guaranteed to execute whenever the - * clause body runs, so no credit can ever come from code a semantics-preserving edit could have - * added dead. Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` - * that has NO catch clause and whose finally (if any) contains no jump out of itself (a finally - * that completes abruptly cancels the try's completion, so nothing in that tryBlock ever escapes - * the clause; `reads a throw a finally break discards as no rethrow` and - * `dead-throw-in-cancelled-try` in the mutation corpus hold it), the sole clause of a - * single-DefaultClause `switch`, the then-arm of an - * `if` whose condition is exactly the literal `true` keyword, and both arms of an `if`/`else` with - * per-arm states merged by intersection (evidence in both arms is unconditional; evidence in one - * is not). Each entry is pinned by `reads a clause wrapped in a single-default switch as the bare - * clause` and its sibling identity pairs. - * - * NOT entered, deliberately: a bare `if` without an else (except the literal-true case), loops - * other than `do` (a body that may run zero times), labelled statements, function-like nodes (the - * iteration-callback boundary is `walkBody`'s attribution rule and this walk never crosses any - * function boundary), nested catch clauses, finally blocks, and the tryBlock of a `try` WITH a - * catch clause, where a throw is intercepted by the nested catch rather than escaping the clause. - * A `throw` or a test in any of those positions does not count. - * - * That is the whole dead-code defence, and it replaces the list of statically-false shapes an - * earlier round kept extending. The list was losing: `if (false)` and `while (false)` were - * recognised, and `for (;false;)`, `if (true) {} else`, `switch (1) { case 2: }`, `try {} catch`, - * `for (const x of [])`, `for (const k in {})`, `if ("")`, `if (!true)` and `if (1 === 2)` were not, - * each worth 50 points a route. Asking for the throw to be unconditional refuses all eleven without - * naming any of them. `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. - * - * `rethrows` asks for one thing more: that the clause contains no `return` at all. The claim it - * feeds is that the clause passes the error through unchanged, which is only true when throwing is - * the ONLY way out. Without it a `throw error;` written after a statement that already exited read - * as a rethrow, in seven spellings: after a bare block, a `do` body, an `if (true)`, an `if`/`else` - * where both arms return, a `switch` with a returning default, and a `try`/`finally` that returns. - * `definitelyExits` handles every one of those, including the `if (true)` spelling since it folds - * the literal `true` keyword, and the no-return rule holds the rest of the line. `dead-throw-after-*` - * in the mutation corpus covers them. + * Both answers are read off the clause's own guaranteed path: the walk enters a construct exactly + * where the entered statements are guaranteed to execute whenever the clause body runs, so no credit + * can ever come from code a semantics-preserving edit could have added dead. Which constructs are + * entered and which are refused, and the eleven dead spellings this replaced: README, "The dead-code + * defence". `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. * * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer - * reads as a rethrow, so it reads as a swallow and fails rather than sitting out, and neither does - * `catch (e) { if (e instanceof Response) return e; throw e; }`, which passes on its branch instead. - * That is the direction to be wrong in, since the reverse hands out points. + * reads as a rethrow, so it fails rather than sitting out. That is the direction to be wrong in. */ function catchClauseEvidence(clause: ts.CatchClause): { rethrows: boolean; throws: boolean; branches: boolean; } { - // `rethrows`, `branches` and `exited` travel in a state record so a walk can be run against an - // isolated copy (the if/else arm walks) as well as the shared root. `returns` stays a single - // shared flag: it is a clause-wide veto, never per-arm evidence. + // The state record travels so an if/else arm can be walked against an isolated copy. `returns` + // stays a single shared flag, because it is a clause-wide veto and never per-arm evidence. // - // On `exited`: set once a statement the walk has already passed could have left the clause. An - // error test - // after one of those is dead code, so it decides nothing. Raised at the END of each statement, - // after that statement's own branch check: a deciding statement contains an exit by definition, - // so raising it first makes every such statement refuse itself, which was measured at 78 routes - // losing their pass and the tree dropping from 15 to 6, measured before round C moved the baseline - // to 19. This ordering leaves the real-tree report - // and all 240 clauses' evidence byte-identical. The tests are the cases in `dead throw written - // after something that already exited`. + // `exited` is raised at the END of each statement, after that statement's own branch check: a + // deciding statement contains an exit by definition, so raising it first makes every such + // statement refuse itself, measured at 78 routes losing their pass. // - // Raised off `containsLiveExit`, never a plain containment read. Containment is true of - // `if (false) { throw e; }` itself, so a provably dead statement raised the flag and blinded the - // walk to the real classification below it: prepending one to a deciding clause turned its pass - // into a swallow verdict on 78 real routes, the same false accusation for all eleven dead - // spellings. The liveness fold only ever withholds this blindness; where `literalTruth` cannot - // decide, the containment answer stands and refusal is intact. The recovered half is `dead and - // deferred code prepended to a deciding catch does not blind it`; the refusing half is the - // `BRANCH_EXITED` list plus `still refuses an error test after an always-true spelling that - // throws`. - // - // `vetoReturns` is whether this walk's statements may feed the `returns` veto. True everywhere - // except the if/else arm walks: `returns` is read at the PARENT level, as a live containment - // read over the whole statement, so an arm walk re-reading its own statements adds nothing for - // a live guard and adds a false veto for a folded-dead arm (the walk enters both arms; the fold - // has already excluded the dead one from the parent read). `dead-classifier-one-arm` in the - // mutation corpus is the tree-scale shape this protects. + // `vetoReturns` is false only in the arm walks. `returns` is read at the PARENT level over the + // whole statement, so an arm walk re-reading its own statements adds a false veto for a + // folded-dead arm (`dead-classifier-one-arm`). type ClauseState = { rethrows: boolean; branches: boolean; @@ -939,20 +727,16 @@ function catchClauseEvidence(clause: ts.CatchClause): { const bindingName = catchBindingName(clause); const walk = (statements: readonly ts.Statement[], state: ClauseState) => { - // A block that re-declares the binding name means an `if` below it referencing that name is - // referencing the shadowing declaration, not this clause's error. Nothing in such a block can - // speak for the clause, so the whole list is skipped for branch purposes. + // A block re-declaring the binding name means an `if` below it references the shadowing + // declaration, not this clause's error, so the whole list is skipped for branch purposes. const shadowed = bindingName !== null && declaresInScope(statements, bindingName); for (const statement of reachableStatements(statements)) { if (ts.isThrowStatement(statement)) { state.rethrows = true; // Read the branch check here, before the path is cut. A thrown ternary picks WHICH error - // leaves, which is a classification, and reading it only at the shared check below meant - // the throw arm of that condition was unreachable: this arm always continued first. So - // `throw e instanceof Response ? e : new ServerError(e)` read as inert while the same - // clause written with `return` passed. `selectsAnErrorPath` is the same predicate either - // way, so the same-arms rule applies and `throw e instanceof Error ? e : e;` is refused. + // leaves, which is a classification, and the shared check below is unreachable from this arm + // because it always continues first. if ( bindingName !== null && !shadowed && @@ -973,37 +757,26 @@ function catchClauseEvidence(clause: ts.CatchClause): { continue; } // A `do` body runs before its condition is ever read, so it is on the straight-line path - // whatever the condition says. The only loop form that is; `definitelyExits` agrees. + // whatever the condition says. The only loop form that is. if (ts.isDoStatement(statement)) { const body = statement.statement; walk(ts.isBlock(body) ? body.statements : [body], state); if (containsLiveExit(statement)) state.exited = true; continue; } - // The three handlers below share one template: walk the inner list with the SAME shared - // state, then read `returns` and `exited` off the whole statement and continue. The explicit - // `containsLiveReturn` read is load-bearing: the `continue` skips the shared read below, and - // `try { throw e; } finally { return null; }` genuinely swallows (the finally return eats - // the throw), so the veto must still see the finally block the walk does not enter. `reads a - // try whose finally returns as swallowing, not rethrowing` is the pin. + // The three handlers below share one template: walk the inner list with the SAME shared state, + // then read `returns` and `exited` off the whole statement and continue. The explicit + // `containsLiveReturn` read is load-bearing, because the `continue` skips the shared read below + // and `try { throw e; } finally { return null; }` genuinely swallows (`reads a try whose + // finally returns as swallowing, not rethrowing`). // - // A `try` WITHOUT a catch clause: its tryBlock always runs when the clause body does, and a - // throw there escapes the clause, so rethrow credit is genuine — unless the finally can - // complete abruptly. A `finally` holding a `return` is covered by the explicit - // `containsLiveReturn` read below; a `finally` holding a `break` or `continue` that leaves - // it cancels the try's completion the same way, so the throw never escapes and the tryBlock - // must not be entered (`reads a throw a finally break discards as no rethrow`, its continue - // and switch-hosted siblings, and `refuses the tryBlock when the finally only may break`; - // `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale shape). The refusal - // is a containment read and entry requires its absence, because entry GRANTS credit; the - // matching liveness fold in `containsLiveWhere` then keeps the refused statement from - // blinding what follows it (`keeps the classification after a finally-break no-op`). The - // finallyBlock itself is NOT walked (classification living only in a finally block is - // under-credited; the tree has no such clause). A `try` WITH a catch clause is not entered - // at all: a throw in that tryBlock is intercepted by the nested catch, so crediting it would - // launder a returnless swallow into not-applicable. `does not read the tryBlock of a caught - // try as this clause's rethrow` is the pin, and the nested clause is judged separately as - // its own `ep.catches` entry. + // A catchless `try` is entered only when its finally cannot complete abruptly, since a finally + // that does cancels the try's completion and the throw never escapes the clause + // (`reads a throw a finally break discards as no rethrow`, `dead-throw-in-cancelled-try`). The + // finallyBlock itself is NOT walked, so classification living only there is under-credited. A + // `try` WITH a catch clause is not entered at all, since a throw in its tryBlock is + // intercepted by the nested catch, which is judged separately as its own `ep.catches` entry + // (`does not read the tryBlock of a caught try as this clause's rethrow`). if ( ts.isTryStatement(statement) && statement.catchClause === undefined && @@ -1014,13 +787,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { if (containsLiveExit(statement)) state.exited = true; continue; } - // A `switch` whose caseBlock is exactly one DefaultClause: that clause's statements always - // run, as a bare list. A bare `break` in it neither rethrows, branches nor raises `exited` - // (`containsLiveExit` does not count breaks), and `reachableStatements` cuts anything after - // a top-level `break`, which is correct: after a break, nothing in the clause list runs. - // Any other switch shape is not entered and falls through to the branch gate below exactly - // as before, so a real `switch (e.code) { case ...: }` keeps its top-level credit. `reads a - // clause wrapped in a single-default switch as the bare clause` is the pin. + // A `switch` whose caseBlock is exactly one DefaultClause: those statements always run, as a + // bare list. Any other switch shape falls through to the branch gate below, so a real + // `switch (e.code)` keeps its top-level credit. `reads a clause wrapped in a single-default + // switch as the bare clause` is the pin. if (ts.isSwitchStatement(statement)) { const clauses = statement.caseBlock.clauses; const only = clauses.length === 1 ? clauses[0] : undefined; @@ -1031,12 +801,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { continue; } } - // An `if` whose condition (after `unwrap()`) is exactly the `true` keyword: the then-arm - // always runs; the else-arm never does and is NEVER walked (`reads a dead else arm under if - // true as contributing nothing` is the pin). Keyword-exact on purpose: `!!1`, `1` and - // `!false` are deliberately not entry tickets, because entry GRANTS credit and a wrong grant - // pays, where `literalTruth`'s wider folding only withholds blindness. This asymmetry is - // deliberate; do not unify the two folds. Takes precedence over the if/else arm walk below. + // An `if` whose condition is exactly the `true` keyword: the then-arm always runs, the else-arm + // is NEVER walked (`reads a dead else arm under if true as contributing nothing`). + // Keyword-exact on purpose, because entry GRANTS credit where `literalTruth`'s wider folding + // only withholds blindness. Do not unify the two folds. Takes precedence over the arm walk. if ( ts.isIfStatement(statement) && unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword @@ -1048,12 +816,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { continue; } // Any other reachable statement that could return means throwing is not the only way out. - // Read here rather than over the whole clause so a `return` the walk has already cut as dead - // does not count, which is what a `do { throw e; } while (false); return null;` produces. - // The LIVE read, not the containment one: `if (false) { return null; }` holds a return that - // can never run, and vetoing the rethrow on it regressed a rethrow-only clause from - // not-applicable to fail on 11 real routes. `still sets rethrows past a dead return in an - // if (false) arm` is the pin. + // Read per statement rather than over the whole clause, so a `return` the walk has already cut + // as dead does not count, and read LIVE rather than by containment, since vetoing on + // `if (false) { return null; }` regressed a rethrow-only clause from not-applicable to fail on + // 11 real routes (`still sets rethrows past a dead return in an if (false) arm`). if (state.vetoReturns && containsLiveReturn(statement)) returns = true; if (bindingName !== null && !shadowed && !state.exited) { @@ -1071,15 +837,10 @@ function catchClauseEvidence(clause: ts.CatchClause): { } } - // An `if` WITH an else (its condition not the literal `true` keyword, which the handler - // above already took): one arm always runs, so evidence present in BOTH arms is - // unconditional and evidence in one arm only is conditional and earns nothing. Each arm is - // walked against an isolated state and the results merge into the parent by INTERSECTION. - // Union is the laundering direction: `if (false) { } else { 0; }` must earn - // nothing, which `dead-classifier-one-arm` in the mutation corpus and `does not credit a - // classifier that sits in one arm only` pin. `returns` is never intersected and never - // per-arm: the shared read above already vetoed off the whole statement, over-approximate - // across the live arms, because narrowing a veto per-arm is the unsafe direction. + // An `if` WITH an else: one arm always runs, so evidence in BOTH arms is unconditional and + // evidence in one arm only earns nothing. The arms merge by INTERSECTION, because union is the + // laundering direction (`dead-classifier-one-arm`, `does not credit a classifier that sits in + // one arm only`). `returns` is never intersected, since narrowing a veto per arm is unsafe. if (ts.isIfStatement(statement) && statement.elseStatement !== undefined) { const armWalk = (arm: ts.Statement): ClauseState => { const armState: ClauseState = { @@ -1111,14 +872,9 @@ function catchClauseEvidence(clause: ts.CatchClause): { /** * Method names that invoke their callback once per element, never once as a whole. The structural - * signal that separates a per-item boundary (`items.map((item) => { try {...} })`, a fresh catch - * for every element) from a route's own body expressed through one more layer of function nesting - * (`trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })`, - * `new ReadableStream({ start: async (c) => {...} })`), all of which invoke their callback exactly - * once. - * - * A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. The - * consequence is written down where it matters, on `isIterationCallback`. + * signal that separates a per-item boundary from a route's own body expressed through one more layer + * of function nesting (`trace(async () => {...})` and friends, which run their callback exactly + * once). A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. */ const ITERATION_METHODS = new Set([ "map", @@ -1139,33 +895,13 @@ function isAtMostSingletonArray(expr: ts.Expression): boolean { } /** - * Whether the function-like `node` is the callback argument of a per-item iteration, e.g. the arrow - * function in `items.map((item) => ...)`. + * Whether the function-like `node` is the callback argument of a per-item iteration. Both directions + * of being wrong, and what makes the name list survivable: README, "The iteration-callback boundary". * - * Being wrong here is asymmetric. Calling a per-item callback the route's own continuation - * mis-attributes a per-element catch to the route, which was the bug the boundary was added for. - * Calling the route's own continuation a per-item callback hides the route's catch, and - * `error-classification` used to read a route with no catch as not-applicable, which is 50 points - * more than the swallow it was hiding. So the second direction paid, and `[0].map(async () => { - * whole body })` collected it. - * - * Two things changed. A receiver that is an array literal of one element or none is refused here, - * because it cannot iterate. And the direction that used to pay no longer pays: `walkBody` keeps - * the catches it refuses, evidence and all, and `error-classification` fails a route with a - * refused swallow when nothing the route owns decides, while a refused catch that decides caps at - * not-applicable and never a pass. That is what makes the name list survivable, and it is why - * `Result.map(...)`, which no name list can tell from `users.map(...)`, is a corpus entry that - * passes rather than a hole: relocating a swallow behind the boundary still fails, and relocating - * a decision earns at most the route's exit from the denominator. - * - * The other direction still costs points and the earlier version of this comment said otherwise. - * A per-item callback under a callee the name list does not know, `pMap(items, cb)` or - * `Array.prototype.map.call(items, cb)`, is attributed to the route, so a per-element catch that - * decides can carry the route to `pass`. No mutation of a real route produces it: the reviewer - * tried `Array.prototype.map.call` over the tree and it moved nothing, because a route has to - * already be iterating for the shape to exist. It is a wrong verdict waiting for a route to be - * written that way, not a laundering path, and it is why this list is worth extending when a new - * iteration helper shows up in the tree. + * The residual a reader here needs: a per-item callback under a callee this list does not know, + * `pMap(items, cb)`, is attributed to the route, so a per-element catch that decides can carry it to + * a pass. That is a wrong verdict waiting for a route to be written that way rather than a laundering + * path, and it is why the list is worth extending when a new iteration helper shows up in the tree. */ function isIterationCallback(node: ts.Node): boolean { const parent = node.parent; @@ -1203,11 +939,9 @@ function collectMethodHandlers(methods: ts.ObjectLiteralExpression, out: EntryFu } /** - * The handler on an object argument, in the two shapes the route builders use: `handler` at the - * top level of the config (`createSSELoader({ handler })`) and `methods.POST.handler`. Matching by - * name at any depth would pick up an unrelated config callback that happens to be called - * `handler`, as well as the sibling lambdas (`findResource`, `authorization.resource`) that are - * not the entry-point body. + * The handler on an object argument, in the two shapes the route builders use: `handler` at the top + * level of the config and `methods.POST.handler`. Matching by name at any depth would pick up the + * sibling lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. */ function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { for (const property of object.properties) { @@ -1250,8 +984,7 @@ function collectHandlerFunctions(call: ts.CallExpression, out: EntryFunction[]): } /** Literals a builder option can be given that mean it was not given: `apiBuilder.server.ts` gates - * every one of these behind `if (option)`. Written out because `authorization: undefined` reads as - * a declared gate to anything counting keys, and declaring one is what `auth-scope` credits. */ + * every option behind `if (option)`, and declaring one is what `auth-scope` credits. */ function isDeclaredValue(property: ts.ObjectLiteralElementLike): boolean { if (!ts.isPropertyAssignment(property)) return true; const value = unwrap(property.initializer); @@ -1260,10 +993,9 @@ function isDeclaredValue(property: ts.ObjectLiteralElementLike): boolean { } /** - * Top-level property names of every object-literal argument to the root call, e.g. `params`, - * `authorization`, `method`. Only the root call and only the top level: `authorization` on - * `createMultiMethodApiRoute` is declared once beside `methods` rather than per method - * (`apiBuilder.server.ts`), so nothing here needs to descend. + * Top-level property names of every object-literal argument to the root call. Only the top level, + * because `authorization` on `createMultiMethodApiRoute` is declared once beside `methods` rather + * than per method (`apiBuilder.server.ts`). */ function collectOptionKeys(call: ts.CallExpression): string[] { const keys: string[] = []; @@ -1526,23 +1258,15 @@ function collectLocalFunctions(sf: ts.SourceFile): Map { return functions; } -/** - * Compiler options for the throwaway program below. `noLib` and `noResolve` keep it from going to - * disk: nothing here needs a type, only the syntax the parser already produced. - */ +/** `noLib` and `noResolve` keep the throwaway program below off the disk: nothing here needs a type, + * only the syntax the parser already produced. */ const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, allowJs: true }; /** - * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off - * the diagnostics array the parser hangs on the source file, which is internal and which the - * compiler is free to rename. The whole parse-failure discipline rests on this, and an undetected - * parse failure shrinks the denominator and inflates the score, so it must not be the kind of - * thing a compiler upgrade can switch off silently. - * - * The host hands the program the `sf` we already have, so this does not parse the source a second - * time. The cost is the program machinery around it, and it is not free: a full scan of the real - * route tree went from about 850ms to about 1450ms, measured over five runs of each. A slower - * scan of a tool that runs once a pull request is the cheaper of the two prices. + * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off the + * internal diagnostics array the parser hangs on the source file, which a compiler upgrade could + * rename out from under us. The host hands the program the `sf` we already have, so nothing is parsed + * twice. Costs and reasoning: README, "Tests, timeouts and CI". */ function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { const host: ts.CompilerHost = { @@ -1640,12 +1364,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { continue; } // `export const { action, loader } = createActionApiRoute(...)`. Skipping a non-identifier - // binding name here produced no entry point at all for this shape: not a parse failure and - // not unmeasured, simply absent from the denominator. The two-step spelling - // (`const { action } = builder(...); export { action };`) already resolved, because - // `collectLocalDeclarations` reads the binding pattern and the export clause looks the name - // up there, so only the direct form was missing. The exported name is the ELEMENT name, so - // `{ loader: action }` exports an action and `{ action: internal }` exports neither. + // binding name here left this shape absent from the denominator entirely, neither a parse + // failure nor unmeasured. The exported name is the ELEMENT name, so `{ loader: action }` + // exports an action and `{ action: internal }` exports neither. if (!ts.isObjectBindingPattern(decl.name)) continue; for (const element of decl.name.elements) { if (!ts.isIdentifier(element.name)) continue; @@ -1689,20 +1410,16 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { }; const localFunctions = collectLocalFunctions(sf); - // A body that delegates to a same-file helper does the work in that helper, so the helper's - // statements, try/catch and callees belong to the entry point. One hop only: a helper's own - // helpers are not followed, and the visited set stops a cycle and any double counting. - // - // The helper's callees belong to whichever EXPORTS reach it, too, which is what `helperOwners` - // carries. A helper called from both halves of the file is owned by both; the union is taken on - // the second discovery rather than dropped, because `visited` has already queued it by then. + // One hop into a same-file helper, whose statements, try/catch and callees belong to the entry + // point. `visited` stops a cycle and any double counting; `helperOwners` carries which EXPORTS + // reach the helper, taking the union on a second discovery because `visited` has already queued it. const visited = new Set(target.functions); const helpers: EntryFunction[] = []; const helperOwners = new Map>(); const walkBody = (fn: EntryFunction, followHelpers: boolean, owners: ReadonlySet) => { - // One push site feeds the entry-point-wide list and each owning export's list, so - // `calleeNames` and the per-export lists cannot drift apart. See `EntryPoint.loaderCalleeNames`. + // One push site feeds the entry-point-wide list and each owning export's list, so they cannot + // drift apart. See `EntryPoint.loaderCalleeNames`. const sinks: BodyFacts[] = [wholeEntry]; for (const owner of owners) sinks.push(byExport[owner]); const collectTested = (node: ts.Node) => { @@ -1716,26 +1433,11 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { addStatements(countFunctionStatements(fn)); if (!fn.body) return; - // `inCallback` is true once the walk has entered a per-item iteration callback - // (`items.map((item) => { ... })`), never reset back to false: nesting deeper inside one is - // still inside it. `calleeNames` and `logCalls` keep descending regardless. A try/catch does - // not: a per-item catch is not part of this body's own statement list, and `countStatement` - // already stops at a nested function boundary, so counting it here let `tryStatementCount` - // exceed the entry point's whole `statementCount` and judged a per-item error boundary as - // though it were the route's own. What is refused is kept in `callbackCatches` with its - // evidence instead of dropped, so `error-classification` can fail a refused swallow and sit - // out a refused catch that decides, without ever crediting either as the route's own. - // - // Only an iteration callback is a boundary, not every function-like node: a route's own body - // wrapped in `trace(async () => {...})`, `mutateWithFallback({ pgMutation: async (t) => {...} })` - // or `new ReadableStream({ start: async (c) => {...} })` still runs exactly once, as the route's - // own continuation one layer of nesting away, and its catch is the route's own error handling. - // - // A nested function's statements count towards `statementCount` too, whichever kind it is. - // They are work the route does, and leaving them out let `trace("x", async () => { whole body - // })` collapse a route to one statement, which is inside the triviality rule's limit: the route - // then read as trivial and every check reported not-applicable for it. `wrap-body-in-trace` in - // the mutation corpus is that shape. + // `inCallback` is true once the walk has entered a per-item iteration callback and is never reset, + // since nesting deeper inside one is still inside it. `calleeNames`, `logCalls` and the statement + // count keep descending regardless; a catch does not, and is kept in `callbackCatches` with its + // evidence rather than dropped. Only an iteration callback is a boundary, not every function-like + // node. See README, "The iteration-callback boundary". const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); @@ -1747,8 +1449,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { for (const sink of sinks) sink.hasTryCatch = true; if (node.catchClause) { // Built the same way for a refused catch as for an own one, so the dead-code defence - // and the walk's guaranteed-execution rules apply to both. Which list it lands in is - // walkBody's attribution decision alone. + // applies to both. Which list it lands in is this walk's attribution decision alone. const tryStatementCount = countStatements(node.tryBlock.statements); const clause = catchClauseEvidence(node.catchClause); (inCallback ? callbackCatches : catches).push({ @@ -1875,14 +1576,9 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { const SOURCE_FILE = /\.tsx?$/; /** - * Whether a file name is one the scanner reads at all. - * - * Exported because three other places ask the same question and each had written its own copy: - * `mutationCorpus.test.ts` materializes exactly the files `scanDirectory` reads, and - * `integration.test.ts` and `webappSymbols.test.ts` walk trees of their own. The corpus's - * anti-vacuity thresholds count files and sites the scanner never saw if those predicates drift, - * and `integration.test.ts`'s `entryPoints.length < countRouteModuleFiles(ROUTES)` stops meaning - * anything if its denominator counts files the scanner skips. + * Whether a file name is one the scanner reads at all. Exported because three test files ask the same + * question and each had written its own copy, and a predicate that drifts makes the corpus's + * anti-vacuity thresholds count files the scan never saw. */ export function isScannableFile(fileName: string): boolean { return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); @@ -1893,14 +1589,8 @@ export type RouteModuleFile = { absolutePath: string; relativeName: string }; /** * The route modules under `dir`: every scannable flat file, plus the `route.ts`/`route.tsx` of each - * immediate subdirectory. - * - * Exported so `mutationCorpus.test.ts` can materialize exactly this set rather than re-deriving it. - * Its `readTree` was a verbatim copy of the walk below; the FILE half of that copy was later - * replaced by a call to `isScannableFile` while the DIRECTORY half stayed duplicated, which is the - * usual way this package's duplicates half-die. A corpus that enumerates a different tree from the - * scanner reports file and site counts for files the scan never reads, and those counts are the - * only thing standing between a mutation that reaches nothing and a green test. + * immediate subdirectory. Exported so `mutationCorpus.test.ts` materializes exactly this set rather + * than re-deriving it, for the same reason as `isScannableFile` above. */ export function routeModuleFiles(dir: string): RouteModuleFile[] { const files: RouteModuleFile[] = []; diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts index 86f15402a51..c045e216f0e 100644 --- a/internal-packages/observability-map/src/score.ts +++ b/internal-packages/observability-map/src/score.ts @@ -9,50 +9,30 @@ export type ScoredEntry = { routePath: string; family: Family; sensitive: boolean; - /** - * The route's body is in another module (`EntryPoint.delegating`), so every check here reads - * not-applicable for that reason and the entry is never measured. Carried separately from - * `measured` because "we could not see it" and "nothing happened to apply" are different facts - * and the report has to be able to say which. - */ + /** The route's body is in another module, so every check reads not-applicable for that reason. + * Carried apart from `measured` because "we could not see it" and "nothing happened to apply" are + * different facts and the report has to be able to say which. */ delegating: boolean; - /** Post-suppression: a suppressed check reads `not-applicable` here, with the reason in - * `detail`. This is the display view; every denominator below reads `rawChecks` instead, so a - * suppression is never invisible to a published figure just because its check is not scored. */ + /** Post-suppression display view. Every denominator reads `rawChecks` instead. */ checks: CheckResult[]; - /** Every check exactly as it ran, before a suppression comment can turn a result into - * `not-applicable`. The one true source for any figure that counts applicability: `measured` - * below, and `contextGap`/`auditGap` in `MapReport`, which read this rather than `checks` for - * exactly that reason. */ + /** Every check exactly as it ran. The one true source for any figure that counts applicability. */ rawChecks: CheckResult[]; - /** - * Whether at least one scored check (`SCORED_CHECK_IDS`, so never `audit-trail`) was applicable - * before suppression. A fully-suppressed entry stays measured, at its capped score, so a - * suppression cannot buy removal from every mean by way of removal from this one. - * `false` means nothing was measured here: the 100 in `score` is a vacuous default, not a - * finding, and `buildReport` excludes an unmeasured entry from every mean it computes so that - * default cannot inflate a figure nobody checked. - */ + /** Whether at least one scored check was applicable BEFORE suppression, so a fully suppressed + * entry stays measured at its capped score. False means the 100 in `score` is a vacuous default + * rather than a finding, and every mean excludes it. */ measured: boolean; - /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. Includes - * `audit-trail`: a suppression is real regardless of whether its check feeds the score. */ + /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. */ suppressed: string[]; - /** Ids in a suppression directive that name no check, so they suppress nothing. Carried here so - * the renderers can say so: dropping them silently is what made a typo look like an - * acknowledgement. */ + /** Ids in a directive that name no check, so they suppress nothing. Carried so the renderers can + * say so: dropping them silently made a typo look like an acknowledgement. */ unknownSuppressions: string[]; /** Passed over applicable, across scored checks only. 100 when nothing applies. */ score: number; }; /** - * What one check contributes to the composite, so a reader can see what the global is made of. - * - * The four-check framing presents a composite the number is not: `request-context` applies to - * nearly every entry point and the rest apply to a minority, so most entries score 0 or 100 on one - * boolean. Disclosed rather than weighted, deliberately. Weighting was rejected in the design - * because a coefficient nobody can explain invites argument about the number instead of the - * finding, and that reasoning has not changed. + * What one check contributes to the composite. Disclosed rather than weighted, deliberately: see + * README, "What the score is made of". */ export type CheckContribution = { id: string; @@ -62,12 +42,11 @@ export type CheckContribution = { passed: number; /** Whether the check feeds `global` at all. `audit-trail` does not, see `buildReport`. */ scored: boolean; - /** Entry points where this was the ONLY applicable scored check, so their score is this check's - * verdict and nothing else. Zero for a check that is not scored. */ + /** Entry points this was the ONLY applicable scored check for, so their score is its verdict and + * nothing else. Zero for a check that is not scored. */ sole: number; - /** The global recomputed with this check taken out of the score, so the difference from `global` - * is what the check is worth. Null when the check is not scored, and null when taking it out - * would leave nothing measured. */ + /** The global recomputed with this check out of the scored set, so the difference from `global` is + * what the check is worth. Null when it is not scored, or when nothing would be left measured. */ globalWithout: number | null; }; @@ -79,13 +58,9 @@ export type MapReport = { /** Entry points every scored check reported not-applicable for; excluded from `global`. Counts * only routes the scanner could read: a delegating one is in `delegating` instead. */ unmeasured: number; - /** - * Routes whose body is in another module, by file name. Excluded from `global` for the same - * reason a parse failure is, and reported for the same reason: the denominator is smaller than - * the entry point count and nothing about these routes has been checked. Moving a body into a - * `.server.ts` file is an ordinary refactor, and without this it silently deletes the route from - * the metric while the route reads as having nothing to fix. - */ + /** Routes whose body is in another module, by file name. Excluded from `global` for the same reason + * a parse failure is, and reported for the same reason: the denominator is smaller than the entry + * point count and nothing about these routes has been checked. */ delegating: string[]; /** Per-check applicability, pass rate and worth, in `CHECKS` order. */ checkContributions: CheckContribution[]; @@ -96,25 +71,17 @@ export type MapReport = { byFamily: Record; sensitiveCohort: { n: number; measured: number; mean: number | null }; auditGap: { sensitiveMutations: number; withAudit: number }; - /** - * `request-context` fails 401 of the 412 entry points it applies to, so it is reported as a - * figure rather than as hundreds of identical list entries, the same treatment `audit-trail` - * gets. It stays fully in the score: the gap is real and the score is meant to show it. - */ + /** Reported as a figure rather than as hundreds of identical list entries, the same treatment + * `audit-trail` gets, while staying fully in the score. */ contextGap: { applicable: number; naming: number }; entries: ScoredEntry[]; parseFailures: string[]; }; /** - * What every check reports for a route whose body is in another module. Applied here rather than in - * each check, because it is a fact about what the scan could see and not about any one question: - * the file holds no handler function and no builder call, so there is nothing for a check to read - * and no check may claim a verdict. `request-context` would otherwise fail such a route for leaving - * its failures to the central handler, an accusation about a body this file does not contain. - * - * Because it is answered here, no check tests `ep.delegating` itself. Two did, and both branches - * were unreachable. + * What every check reports for a route whose body is in another module. Answered here rather than in + * each check, because it is a fact about what the scan could see and not about any one question, so + * no check tests `ep.delegating` itself. Two did, and both branches were unreachable. */ const DELEGATED_CHECKS = (): CheckResult[] => CHECKS.map((c) => ({ @@ -156,25 +123,21 @@ export function scoreEntry(ep: EntryPoint): ScoredEntry { suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), unknownSuppressions: unknown, measured: scoredApplicable.length > 0, - // Capped by the pre-suppression ratio: removing a failing check from both the numerator and - // the denominator otherwise raises the ratio, which is how 33 became 50 became 100 before this - // cap existed. See ScoredEntry.measured for why the denominator itself is pre-suppression too. + // Capped by the pre-suppression ratio, or removing a failing check from both numerator and + // denominator raises it, which is how 33 became 50 became 100 before the cap existed. score: Math.min(ratio(visible), ratio(scored)), }; } -/** - * Null for an empty group rather than 100. A family nothing was measured in has no score, and - * rendering the absence as a full green bar said the opposite of what the data said. - */ +/** Null for an empty group rather than 100: rendering the absence as a full green bar said the + * opposite of what the data said. */ const mean = (xs: number[]): number | null => xs.length === 0 ? null : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); /** - * `n` is every entry point in the group; `mean` is taken over the measured subset only, so an - * entry point nothing applied to cannot drag a family's or cohort's figure toward 100. `measured` - * is reported alongside so a reader can tell a family scoring high because it is clean apart from - * a family scoring high because most of it was never measured. + * `n` is every entry point in the group; `mean` is over the measured subset only. `measured` is + * reported alongside so a reader can tell a family scoring high because it is clean from one scoring + * high because most of it was never measured. */ function groupStats(entries: ScoredEntry[]): { n: number; @@ -190,11 +153,9 @@ function groupStats(entries: ScoredEntry[]): { } /** - * The global as it would read with `omitted` taken out of the scored set, so the difference from - * the published global is what that check is worth. Recomputed from `rawChecks` the same way - * `scoreEntry` computes a score, minus the suppression cap: a suppression can only lower an entry's - * score, and lowering both figures by the same rule would leave the difference between them saying - * something about suppressions rather than about the check. + * The global as it would read with `omitted` out of the scored set. Recomputed from `rawChecks` minus + * the suppression cap, because lowering both figures by the same rule would leave the difference + * saying something about suppressions rather than about the check. */ function globalWithout(entries: ScoredEntry[], omitted: string): number | null { const scores: number[] = []; @@ -244,14 +205,9 @@ export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapRepo const sensitive = entries.filter((e) => e.sensitive); - // audit-trail is excluded from the score (see checks/index.ts and scoreEntry above), and is - // reported here as its own architectural figure instead: how many sensitive mutations have an - // audit record, out of how many. Folding it into the score would tank every sensitive route on a - // gap that is the same everywhere, and bury the routes that have their own, fixable problems. - // - // Both gaps read `rawChecks`, pre-suppression, the same reason `measured` does: suppressing the - // one request-context or audit-trail finding on an entry must not shrink these denominators and - // raise the printed percentage, on the same screen as a claim that suppression cannot do that. + // Both gaps read `rawChecks`, pre-suppression: suppressing the one finding on an entry must not + // shrink these denominators and raise the printed percentage, on the same screen as a claim that + // suppression cannot do that. const contextChecks = entries .map((e) => e.rawChecks.find((c) => c.id === "request-context")) .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index ba13454fe17..c9981c46b55 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -3,16 +3,9 @@ import { routePathOf } from "./adapters/remix.js"; /** * Symbols whose presence says the route does something risky: minting or revoking a credential, - * escalating to another user, destroying a tenant. Calling a guard is not one of them: - * `requireAdminApiRequest` was on this list and made 34 of the 67 sensitive entry points sensitive - * purely because they were guarded, which `auth-boundary` then passed them for. A mitigation - * cannot be the hazard, and this list feeds the fix list's primary sort key. - * - * Half of this list used to name nothing. `Set.has` is exact, so `setImpersonation`, `createJWT`, - * `signJWT` and `updateEnvVars`, none of which are exported anywhere in `apps/webapp/app`, matched - * no route at all, while the real escalation `startImpersonation` was absent. Every name here now - * resolves to a declaration in the webapp, and `webappSymbols.test.ts` fails if one stops - * doing so. + * escalating to another user, destroying a tenant. Calling a guard is never one of them, because a + * mitigation cannot be the hazard, and `webappSymbols.test.ts` fails if a name stops resolving in the + * webapp. Both rules and what they cost: README, "Sensitivity, and the names the tool matches on". */ export const SENSITIVE_SYMBOLS = [ // Escalation: acting as another user. @@ -44,39 +37,23 @@ export const SENSITIVE_SYMBOLS = [ ]; /** - * Segments in `SENSITIVE_SEGMENTS` that match no route in the tree today. Kept because they are - * ordinary words for the thing they name, so a route called one of them would be sensitive the day - * it lands, and separated because the rest of the vocabulary is read off the tree and - * `webappSymbols.test.ts` holds it to that. Adding a word here is a deliberate statement that - * it names nothing yet, and shows up in review as one. + * Segments in `SENSITIVE_SEGMENTS` that match no route in the tree today, kept apart because + * `webappSymbols.test.ts` holds the rest of the vocabulary to naming something. Adding a word here is + * a deliberate statement that it names nothing yet, and shows up in review as one. */ export const ANTICIPATED_SEGMENTS = ["payment", "invoices", "secrets"]; /** - * Whole path segments only, so "authorship" does not match "auth". - * - * Every entry is a segment that exists in `apps/webapp/app/routes` today; the vocabulary was read - * off the tree rather than invented, and `webappSymbols.test.ts` fails if a segment stops - * appearing in a route name. Two consequences of that rule are worth stating rather than leaving - * to be rediscovered: there is no `transfer` segment because the webapp has no org or project - * transfer route, and org/project deletion is reached through `DeleteOrganizationService` above - * rather than through a segment, because the four routes that delete are named `orgs` and - * `projects` and `settings`. - * - * Two segments were measured and left out. - * - * `logout` is one route, and both questions the cohort exists to ask are meaningless on it: - * `logout.tsx` destroys the caller's own session, so there is no other party's credential to guard - * and no actor to record beyond the one already leaving. Including it produced one permanent - * `auth-boundary` failure that no change to the route could clear. + * Whole path segments only, so "authorship" does not match "auth". Every entry exists in + * `apps/webapp/app/routes` today and `webappSymbols.test.ts` fails if one stops appearing. * - * `sessions` is the bigger one. It reads as the auth session surface and is not: every `sessions` - * route in this tree is the realtime agent-session product, - * `_app...env.$envParam.sessions._index/route.tsx` renders `SessionsTable`, and - * `api.v1.sessions.$session.close.ts` closes an agent session. Sixteen route files carry the - * segment. The genuine session-management surface is `session-duration`, which is here, and the - * credential minting inside those routes is caught by `mintSessionToken` in the symbol list above, - * which is why `api.v1.sessions.ts` is in the cohort and its fifteen siblings are not. + * Two absences worth knowing rather than rediscovering. `logout` is left out because + * `logout.tsx` destroys the caller's own session, so there is no other party's credential to guard and + * no actor to record, and including it produced one permanent `auth-boundary` failure nothing could + * clear. `sessions` is left out because every `sessions` route in this tree is the realtime + * agent-session product rather than the auth surface, sixteen files of it; the genuine + * session-management surface is `session-duration` below, and the credential minting inside those + * routes is caught by `mintSessionToken` above. */ export const SENSITIVE_SEGMENTS = [ // Credentials, tokens and money: the original vocabulary. @@ -118,14 +95,12 @@ export const SENSITIVE_SEGMENTS = [ ]; /** - * A route-name segment with Remix's layout markers taken off, so the segment vocabulary can be - * written the way a reader would say it. A trailing underscore opts a route out of its parent - * layout (`resources.impersonation_.view-as.ts`) and changes nothing about what the route does. + * A route-name segment with Remix's layout markers taken off. A trailing underscore opts a route out + * of its parent layout and changes nothing about what the route does. */ export function normalizeSegment(segment: string): string { - // Trimmed by hand rather than with /_+$/, which backtracks polynomially on a run of underscores - // and trips CodeQL. Nothing here is attacker-controlled (the input is a filename read off disk), - // so this is about not spending a reviewer's attention on the alert. + // Trimmed by hand rather than with /_+$/, which backtracks polynomially and trips CodeQL. Nothing + // here is attacker-controlled, so this is about not spending a reviewer's attention on the alert. let end = segment.length; while (end > 0 && segment[end - 1] === "_") end--; return segment.slice(0, end); @@ -136,25 +111,23 @@ export type Sensitivity = { sensitive: boolean; reasons: string[] }; export function classifySensitivity(ep: EntryPoint): Sensitivity { const reasons: string[] = []; - // importedNames is file-wide; calleeNames is scoped to the loader/action body. A sensitive - // symbol called only at module scope is caught here only if it is also imported. + // `importedNames` is file-wide and `calleeNames` is body-scoped, so a sensitive symbol called only + // at module scope is caught here only if it is also imported. const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); for (const s of SENSITIVE_SYMBOLS) { if (symbols.has(s)) reasons.push(`calls ${s}`); } - // `routePathOf` turns both flat routes (`api.v1.envvars.ts`) and directory routes - // (`billing/route.tsx`) into real `/`-separated path segments, so this matches whole segments in - // either shape rather than splitting the raw fileName on ".". + // `routePathOf` normalises both flat and directory routes to `/`-separated segments, so this + // matches whole segments in either shape rather than splitting the raw fileName on ".". const segments = routePathOf(ep.fileName) .split("/") .filter((s) => s.length > 0) .map(normalizeSegment); for (const [i, seg] of segments.entries()) { if (!SENSITIVE_SEGMENTS.includes(seg)) continue; - // A waitpoint token is a handle for resuming a run, not a credential. Seven of the eight - // `tokens` matches in the tree were waitpoint routes, so the segment on its own was mostly - // finding the wrong thing. + // A waitpoint token is a handle for resuming a run rather than a credential, and seven of the + // eight `tokens` matches in the tree were waitpoint routes. if ((seg === "token" || seg === "tokens") && segments[i - 1] === "waitpoints") continue; reasons.push(`path segment "${seg}"`); } diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 463706dfa2c..9d23e8021cf 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -4,26 +4,17 @@ import { CHECKS } from "./checks/index.js"; const KNOWN_CHECK_IDS = new Set(CHECKS.map((c) => c.id)); /** - * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` - * does not match a newline, so a suppression on one line cannot pick up a reason from the next. - * - * It was `obs-map-disable-next-line`, which was a lie: a check applies to a whole entry point, so - * the directive did too, and one on the last line of a file switched a check off for everything - * above it. The honest options were to scope it to a line or to rename it, and scoping is not - * available: a `CheckResult` carries no line number, and neither does an `EntryPoint`, so there is - * nothing to match a line against. Scoping it would mean inventing a proximity rule that silently - * drops legitimate suppressions. So the name now says what it does. Real line scoping needs - * positions on the findings, which is scanner work. + * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` does + * not match a newline, so a suppression cannot pick up a reason from the next line. The old + * `obs-map-disable-next-line` spelling and why it was renamed rather than scoped: README, + * "Suppression". */ const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; /** - * Every leaf token in the parsed source: keeps descending through `.getChildren()` rather than - * `ts.forEachChild`, which only returns the child nodes a statement or expression models as its - * own properties and silently skips a bare punctuation or keyword token (a closing brace, a - * semicolon). A comment can sit directly before one of those with nothing else following it, the - * last line inside a block, and `.getChildren()` still reaches it because the token itself is - * still a node with a position. + * Every leaf token in the parsed source, through `.getChildren()` rather than `ts.forEachChild`, which + * silently skips a bare punctuation or keyword token. A comment can sit directly before one of those + * with nothing else following it, on the last line inside a block. */ function leafTokens(node: ts.Node): ts.Node[] { const children = node.getChildren(); @@ -31,23 +22,16 @@ function leafTokens(node: ts.Node): ts.Node[] { } /** - * Node kinds whose text the parser has already claimed as content, so nothing inside their span can - * be trivia however it is spelled. `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw - * lexers over source text from an offset and consult no parse tree at all, so at a leaf-token - * boundary they will happily lex the inside of one of these as a comment: a JSX text node that - * BEGINS with `//` or `/*` is the shape that reached the real tree, in - * `resources.branches.create.tsx`'s `//`. - * - * The four cases in `jsx text is content, not a comment` (`suppression.test.ts`) are the ones - * that fail without `ts.isJsxText` here; the positive control beside them, `still reads a directive - * from a comment in a JSX expression container`, is what stops the filter being widened until it - * eats real comments. `does not suppress from a directive inside a template literal` and the two - * substitution cases cover the template kinds, and `ignores the directive inside a string literal` - * covers the string kind. + * Node kinds whose text the parser has already claimed as content, so nothing inside their span can be + * trivia however it is spelled. `jsx text is content, not a comment` is the four cases that fail + * without `ts.isJsxText` here, and the positive control beside them, `still reads a directive from a + * comment in a JSX expression container`, is what stops the filter being widened until it eats real + * comments. The template and string kinds are covered by + * `does not suppress from a directive inside a template literal` and + * `ignores the directive inside a string literal`. * - * The mutation corpus does NOT cover any of this, and cannot: a suppression can only lower an - * entry's score, because `scoreEntry` caps it at the pre-suppression ratio. Suppression bugs are - * invisible to a harness that watches for the score rising, so they need ordinary unit tests. + * The mutation corpus cannot cover any of this, because a suppression can only lower a score. See + * README, "Reading the directive out of the source". */ function isClaimedContent(node: ts.Node): boolean { return ( @@ -62,21 +46,12 @@ function isClaimedContent(node: ts.Node): boolean { } /** - * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a - * standalone `ts.createScanner`, and then filtered against the spans above. - * - * Both halves are needed. Parsing rather than scanning is what stops a template literal WITH a - * substitution being rescanned as ordinary code after `${x}`, and what makes JSX text a node at all. - * Filtering by span is what stops the two comment-range lexers reading the start of such a node as - * a comment anyway, which they do because they never see the tree the parser built. - * - * The filter is on the range's start offset falling inside a claimed span, not on the gap between a - * token's full start and its start. A gap filter was tried and rejected: it loses a same-line - * trailing comment and a comment inside a JSX expression container, both of which are real. - * - * Both lexers are called at every token boundary, because which one returns a given comment depends - * on whether it shares a line with the token before it (trailing) or comes after a line break - * (leading), not on which directive it happens to be. + * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a standalone + * `ts.createScanner`, and then filtered against the spans above. Both halves are needed, and the + * filter is on the range's start offset rather than on the gap between a token's full start and its + * start: README, "Reading the directive out of the source". Both lexers are called at every token + * boundary, because which one returns a given comment depends on whether it shares a line with the + * token before it. */ function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { const claimed: ts.TextRange[] = []; @@ -126,21 +101,18 @@ function commentLines(source: string, sf: ts.SourceFile): string[] { export type Suppressions = { /** Check id to reason, for ids that name a check in `CHECKS`. */ byId: Map; - /** - * Ids that parsed as a directive but name no check, in source order and deduplicated. A typo - * (`eror-classification`) used to land in the map, match nothing and appear nowhere, so the - * author read the finding as acknowledged while the tool kept reporting it. - */ + /** Ids that parsed as a directive but name no check, deduplicated. A typo (`eror-classification`) + * used to land in the map, match nothing and appear nowhere, so the author read the finding as + * acknowledged while the tool kept reporting it. */ unknown: string[]; }; /** - * Every suppression directive in the source, split by whether its id names a real check. A - * directive without a reason, or outside a comment, is ignored either way. + * Every suppression directive in the source, split by whether its id names a real check. A directive + * without a reason, or outside a comment, is ignored either way. * - * `fileName` picks the parser's script kind: JSX syntax is only legal, and only correctly - * distinguished from a generic type argument list (`(x) => x`), when the file is really a - * `.tsx`. Defaults to a plain `.ts` for callers that only have source text. + * `fileName` picks the parser's script kind, because JSX is only distinguished from a generic type + * argument list (`(x) => x`) when the file is really a `.tsx`. */ export function parseSuppressions(source: string, fileName = "check.ts"): Suppressions { const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts index 15515d1ec36..377c1165dec 100644 --- a/internal-packages/observability-map/src/triviality.ts +++ b/internal-packages/observability-map/src/triviality.ts @@ -3,30 +3,22 @@ import type { EntryPoint } from "./types.js"; /** * Substrings that say the route touches a service, a datastore or the network. Always matched - * against the callee names, and additionally against `TrivialityView.hintText`, which is the whole - * file for the entry-point-wide view and empty for a per-export one. + * against the callee names, and additionally against `TrivialityView.hintText`. */ const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; -/** - * Calls a genuinely trivial body makes: parse the params, build a path, hand back a response. Every - * shape found in the real tree stays at or below three, so anything busier is doing work. Allowing - * a fourth admits `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. - */ +/** Calls a genuinely trivial body makes. Three; a fourth admits + * `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. */ const MAX_CALLS = 3; -/** - * Parse the params, build a path, redirect. Or an environment guard and two returns. Both real - * shapes need three. Allowing a fourth admits the routes that authenticate and then hand off to a - * presenter (`...tasks.stream/route.tsx`), which have real work behind them and belong in the - * report; allowing a fifth admits an admin route that calls a service and hand-rolls its own error - * responses. - */ +/** Statements a genuinely trivial body has. Three; a fourth admits the routes that authenticate and + * then hand off to a presenter, which have real work behind them. */ const MAX_STATEMENTS = 3; /** - * What the rule reads, so the entry-point-wide answer and a single export's answer are the same - * rule over different bodies rather than two rules that can drift. + * What the rule reads, so the entry-point-wide answer and a single export's answer are the same rule + * over different bodies rather than two rules that can drift. Both limits, the reluctance and the + * measured `hintText` decision: README, "Triviality, in detail". */ type TrivialityView = { statementCount: number; @@ -34,48 +26,19 @@ type TrivialityView = { hasTryCatch: boolean; /** Every builder call in scope of this view. A view with one is never trivial. */ initializerCallees: (string | null)[]; - /** - * Text to match the side-effect hints against besides the callee names. - * - * The whole file for the entry-point-wide view, so an import of `prisma` disqualifies it even - * when the query sits somewhere the scanner does not walk. For a per-export view it is that - * export's own callee PATHS instead, and the difference is not a convenience: - * - * - The file's text is a fact about the file, so reading it into one export's verdict is the - * per-file-for-per-export substitution this rule exists to damp. It is also defeatable. - * `log-caller-scope-userid` in the mutation corpus prepends `logger.error(...)` to every body; - * with this term file-wide that put the word `logger` in `auth.github.ts` and turned its - * untouched one-line redirect loader from excused into accused, on a rewrite that changed - * nothing the loader does. - * - Emptying it instead is not the answer either, and that was measured: `calleeNames` keeps only - * a call's last segment, so `prisma.orgMember.findMany` reads as `findMany` and a - * three-statement body that queries the datastore matches no hint at all. Five existing - * `auth-boundary` fixtures went from `fail` to `not-applicable`, which is the check being - * switched off rather than fixed. - * - * The callee paths are body-scoped like the first option wants and name the receiver like the - * second needs. Comments and imports are not in them, which is deliberate: everything in this - * view is something the export actually does. - */ + /** Text to match the side-effect hints against besides the callee names: the whole file for the + * entry-point-wide view, that export's own callee PATHS for a per-export one. Reading the file's + * text into one export's verdict is defeatable by `log-caller-scope-userid`; emptying the term + * instead switches five `auth-boundary` fixtures off. */ hintText: string; }; /** - * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed - * response, or hands off in a single call. Checks report not-applicable for these rather than - * failing, which is what stops `@.ts` being a finding. - * - * Deliberately reluctant. A route wrongly called trivial is exempted and never shows up in the - * report again, so every signal that the body might be doing real work rules triviality out: + * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed response, + * or hands off in a single call. Checks report not-applicable for these rather than failing. * - * - `statementCount` counts a nested function's statements but `calleeNames` descends further, into - * the callee of every call at any depth, so the call count still catches bodies the statement - * count reads as short. - * - An initializer callee means the route is wrapped in a builder, and the config passed to that - * builder (`findResource`, `authorization`) is work the scanner never walks. The visible body is - * not the whole route, so we cannot claim it is trivial. - * - A try/catch is exactly what the error-classification check reads, so a body with one has an - * error path worth reporting on however short it is. + * Deliberately reluctant, because a route wrongly called trivial is exempted and never shows up in + * the report again. */ function isTrivialView(view: TrivialityView): boolean { if (view.statementCount > MAX_STATEMENTS) return false; @@ -100,16 +63,11 @@ export function isTrivial(ep: EntryPoint): boolean { } /** - * The same rule over ONE export's handlers. - * - * Needed because a per-export verdict judged against an entry-point-wide triviality rule accuses - * the wrong half of a file. `auth.github.ts` and `auth.google.ts` are - * `export let loader = () => redirect("/login")` beside an action that calls - * `authenticator.authenticate`: per export the loader is unguarded, and the entry-point-wide rule - * calls the file non-trivial because the ACTION is not, so `auth-boundary` accused a one-line - * redirect stub of missing an auth guard. `checks/index.test.ts` pins both directions of that - * ("reports not-applicable for a redirect-stub loader beside a guarded action" and "fails an export - * whose own body does real work unguarded"). + * The same rule over ONE export's handlers. Needed because a per-export verdict judged against an + * entry-point-wide rule accuses the wrong half of a file: `auth-boundary` accused + * `auth.github.ts`'s one-line redirect loader of missing a guard because the ACTION is not trivial. + * `checks/index.test.ts` pins both directions (`reports not-applicable for a redirect-stub loader + * beside a guarded action`, `fails an export whose own body does real work unguarded`). */ export function isTrivialExport(e: RouteExport): boolean { return isTrivialView({ diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index f11fb4a4a1f..5d1c93ffaec 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -9,70 +9,34 @@ export type CheckResult = { /** * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays - * legible instead of collapsing into one boolean. + * legible. Every field's exact rule and its measured reasoning: README, "Catch evidence, per + * clause". */ export type CatchEvidence = { - /** - * Throwing is the clause's only way out. Two conditions: a `throw` is reached on the clause's - * guaranteed path (the positions certain to execute whenever the clause runs: its own - * statements, a bare nested block's, a `do` body's, a catchless `try`'s tryBlock, a - * single-default `switch`'s clause, an `if (true)` then-arm, and both arms of an `if`/`else` - * together, cut at the first statement that definitely exits, see `catchClauseEvidence` and - * `definitelyExits`), and the clause contains no live `return` anywhere. A throw guarded by a - * condition the walk cannot fold, a loop, a nested caught `try`, a finally block or a callback - * does not count, and neither does one written after something that has already returned. - */ + /** Throwing is the clause's only way out: a throw on the clause's guaranteed path (see + * `catchClauseEvidence`) and no live `return` anywhere. */ rethrows: boolean; - /** A `throw` is reached on that same guaranteed path, whether or not it is the only way out. - * `rethrows` is this AND no reachable `return`. Kept separately so a verdict can say what is true - * of a clause that both throws and returns. */ + /** A throw is reached on that path, whether or not it is the only way out. Kept apart from + * `rethrows` so a verdict can say what is true of a clause that both throws and returns. */ throws: boolean; - /** - * The clause picks what to do from what it caught, on that same guaranteed path: an `if` or - * `switch` whose condition references the caught error binding AND at least one of whose arms - * returns or throws, or a conditional that is the whole value of a `return`/`throw`. - * `if (retries > 0)` does not count, `if (e instanceof Error) { }` does not count, and a - * bindingless `catch { ... }` cannot count at all. An `instanceof` used only to word a message, - * `json({ error: e instanceof Error ? e.message : String(e) })`, does not count either: every - * error still leaves by the same path. - */ + /** The clause picks what to do from what it caught, on that same guaranteed path. */ branches: boolean; - /** - * The guarded region parses something: `JSON.parse`, `request.json()`, a zod `parse`/`safeParse`, - * a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three constructors are read here - * because a `new` expression is not a call, so the call-callee scan that feeds this check never - * sees them; other constructors do not count, or every `new SomePresenter()` in a try would excuse - * its catch. - */ + /** The guarded region parses something. Includes `new URL`/`URLSearchParams`/`RegExp`, which the + * call-callee scan cannot see because a `new` expression is not a call. */ guardsParse: boolean; - /** - * The guarded region does something that could raise at all: a call, a construction, an `await`, - * a member access, a `throw`, an iteration, an `instanceof`. False means `try { 0; }` and little - * else: any call counts, including one that cannot throw, so `try { String(0); }` reads as true. - * See `canRaise` in `scan.ts` for both directions of that, including the destructuring - * declaration it misses. - */ + /** The guarded region does anything that could raise. Any call counts, including one that cannot + * throw, so `try { String(0); }` is true here: `dead-classifying-try-with-call`. */ guardCanRaise: boolean; /** - * The containment twin of `guardCanRaise`: false only when the guarded region provably cannot - * raise, i.e. every statement is an expression over a bare literal, which is `try { 0; }` and - * nothing else. Everything `canRaise`'s whitelist misses (a destructuring declaration, a - * temporal-dead-zone read) stays true here, so `guardCanRaise` implies `guardMayRaise`. What the - * refused-callback arm of `error-classification` reads: a route whose own classifying catch - * `canRaise` cannot see must never be told nothing it owns decides - * (`does not accuse a route that owns a catch of owning none`), while the provably dead - * `try { 0; }` clause still blocks nothing (`still fails a per-item swallow beside a deciding - * catch over a dead guard`). + * The containment twin of `guardCanRaise`: false only for the provably inert `try { 0; }`, so + * can-raise implies may-raise. Read by the refused-callback arm of `error-classification`, where + * a `canRaise` miss would otherwise accuse a route that owns a real classifying catch + * (`does not accuse a route that owns a catch of owning none`). */ guardMayRaise: boolean; - /** - * Everything the guarded region waits for is one of those parses. What separates - * `try { const body = await request.json(); } catch { 400 }` from - * `try { const body = await request.json(); return await handleEverything(body); } catch { 500 }`, - * which the statement count reads as the same size. Synchronous work is not counted here: the - * calls that prepare a parse's input are synchronous, and the swallows this has to catch wait on - * a service. - */ + /** Everything the guarded region waits for is one of those parses. Synchronous work is not + * counted: preparing a parse's input is synchronous, and the swallows this separates out wait on + * a service. */ awaitsOnlyParse: boolean; /** Statements in the guarded try block, counted as `statementCount` counts them. */ tryStatementCount: number; @@ -88,6 +52,10 @@ export type LogCall = { inCatch: boolean; }; +/** + * Body-scoped evidence for one route module. Which fields are per export and why, and what "the + * body" means: README, "How the scanner reads a route". + */ export type EntryPoint = { fileName: string; source: string; @@ -96,56 +64,28 @@ export type EntryPoint = { /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ loaderInitializerCallee: string | null; actionInitializerCallee: string | null; - /** - * Top-level keys of the object literals passed to that call, e.g. `["params", "authorization"]`. - * Empty when the export has no initializer call, and empty when the call takes no object - * literal, so an empty array is "nothing declared here" rather than "no builder". - */ + /** Top-level keys of the object literals passed to that call. Empty means "nothing declared + * here" rather than "no builder". */ loaderBuilderOptions: string[]; actionBuilderOptions: string[]; /** - * The route declares a loader or an action, and the scan resolved neither a handler function nor - * a builder call for any of them: `export { action } from "./handler.server"`, - * `export const action = handleWebhook`. Nothing about the request handling is in this file, so - * every check reports not-applicable and `buildReport` counts the entry point separately from - * the ones nothing happened to apply to. Those are different facts: a redirect stub genuinely has - * nothing to instrument, a delegating route has work the scanner cannot see. - * - * A route that delegates one export and writes the other in the file is NOT delegating by this + * The route declares a loader or an action and the scan resolved neither a handler function nor a + * builder call for any of them, so nothing about the request handling is in this file. A route + * that delegates ONE export and writes the other in the file is not delegating by this * definition, and is judged on the half that is visible. */ delegating: boolean; - /** - * Whether THIS export's handler assigns the caller's own id to an object-literal property, the - * `where: { members: { some: { userId: authentication.userId } } }` and - * `presenter.call({ userId: user.id })` shapes. Read by `auth-scope` as evidence that the handler - * narrowed its work to whoever is asking. See `CALLER_ID_PATH` and `scopesByCallerIn` in - * `scan.ts`. - * - * Split per export because the exposure is per export: a loader that narrows itself to the caller - * says nothing about the action beside it. Property assignments only, so a value read into a - * local first (`const userId = user.id; ... { userId }`) is not seen. - */ + /** Whether THIS export's handler assigns the caller's own id to an object-literal property. + * Property assignments only, so a value read into a local first is not seen. */ loaderScopesByCaller: boolean; actionScopesByCaller: boolean; /** - * Callees whose answer THIS export's handlers demonstrably looked at: the call's result was bound - * to a local and some `if`, `while`, `switch` or conditional in the same handlers reads that - * local. `const user = await getUser(request); if (!user) return redirect("/login");` puts - * `getUser` here; a call whose result is dropped, or bound and never tested, does not appear. - * - * Read by `auth-boundary` for the guards that answer with null instead of throwing, where being - * called is not evidence that the route acted on the answer. - * - * Split per export for the same reason `loaderScopesByCaller` is, and there is no entry-point-wide - * version on purpose: a loader that reads what `getUser` returned says nothing about the action - * beside it, so the union is not a fact any check should be able to reach for. + * Callees whose answer THIS export's handlers demonstrably looked at: bound to a local, and that + * local read by some condition. No entry-point-wide version exists on purpose, so no check can + * let a loader's reading of `getUser` speak for the action beside it. * - * Deliberately coarse. It does not check that the test guards anything, that the local is the one - * tested rather than a same-named one in another scope, or that the branch exits: a route - * that writes `if (!user) { logger.warn("anonymous"); }` and carries on is credited. It separates - * "looked at the answer" from "ignored it", which is the distinction the check needs, and not - * "acted correctly on the answer", which it cannot see. + * Deliberately coarse: it does not check that the test guards anything, so a route writing + * `if (!user) { logger.warn("anonymous"); }` and carrying on is credited. */ loaderCheckedCallees: string[]; actionCheckedCallees: string[]; @@ -153,39 +93,21 @@ export type EntryPoint = { importedNames: string[]; /** * Names of functions called inside the loader/action bodies, or in a same-file helper they call. - * - * Entry-point-wide, and read only by the questions that are themselves entry-point-wide: - * `sensitivity.ts` asks what the file touches, `triviality.ts` counts how much the file does, - * `audit-trail` asks whether the file records anything. A question about ONE export's exposure - * must read `loaderCalleeNames`/`actionCalleeNames` instead. `auth-boundary` read this and + * Entry-point-wide, and read only by the questions that are themselves entry-point-wide. A + * question about ONE export's exposure must read the pair below: `auth-boundary` read this and * credited a file whose loader called a guard for an action that called none. */ calleeNames: string[]; - /** - * The same callee names attributed to the export whose handlers made the call. A handler serving - * both exports (`const { loader, action } = createActionApiRoute({ handler })`) contributes to - * both, and a same-file helper contributes to whichever exports reach it. - * - * Every name here appears in `calleeNames` and every name in `calleeNames` appears in at least one - * of these, because all three are filled from one push in `scanFile`. `scan.test.ts` pins that - * ("every callee name is attributed to an export that exists") and `integration.test.ts` pins it - * again across the real route tree, so the split cannot drift away from the union it came from. - */ + /** The same names attributed to the export whose handlers made the call. A handler serving both + * exports contributes to both. */ loaderCalleeNames: string[]; actionCalleeNames: string[]; - /** - * The same calls as `loaderCalleeNames`/`actionCalleeNames`, each as its whole dotted path - * (`prisma.organization.findFirst` rather than `findFirst`). Read by the per-export triviality - * rule, which has to know that a three-statement body reaches the datastore; the bare name that - * `auth-boundary` matches guards against throws that receiver away. - */ + /** The same calls as whole dotted paths (`prisma.organization.findFirst`), which the per-export + * triviality rule needs to know a short body reaches the datastore. */ loaderCalleeTexts: string[]; actionCalleeTexts: string[]; - /** - * Whether a `try` appears in the loader/action bodies, or in a same-file helper they call. Note - * that this says a `try`, not a catch: a `try`/`finally` sets it while `catches` stays empty and - * every catch-shaped field stays false. Read `catches.length` to ask whether anything is caught. - */ + /** Whether a `try` appears in those bodies. A `try`/`finally` sets this while `catches` stays + * empty, so ask `catches.length` whether anything is caught. */ hasTryCatch: boolean; /** The same fact for one export's handlers alone. Read by the per-export triviality rule. */ loaderHasTryCatch: boolean; @@ -193,30 +115,18 @@ export type EntryPoint = { /** One entry per catch clause in those bodies, in source order. */ catches: CatchEvidence[]; /** - * Catch clauses the scan found but refused to attribute to the route, because they sit inside a - * per-item iteration callback. Still refused for attribution: they never join `catches`, never - * speak for the route's `tryStatementCount`, and never reach a pass. Kept WITH their evidence, - * built by the same `catchClauseEvidence` machinery as an own catch, so `error-classification` - * can judge what a refused catch does rather than where it sits: a refused swallow fails the - * route (`fails a per-item swallow even when the route owns an inert rethrow catch`), a refused - * catch that decides or rethrows caps at not-applicable (`sits out a route whose only catch is a - * deciding per-item boundary`). The count the old field carried is `.length`. + * Catch clauses the scan refused to attribute to the route because they sit inside a per-item + * iteration callback. Never join `catches`, never speak for `tryStatementCount`, never reach a + * pass. Kept WITH their evidence so `error-classification` can judge what a refused catch does + * rather than where it sits. */ callbackCatches: CatchEvidence[]; /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ logCalls: LogCall[]; - /** - * Statement count across loader/action bodies, used by the triviality rule. Includes the - * statements of functions written inline in those bodies, so wrapping a body in a callback does - * not shrink it. A body that delegates to a same-file helper counts that helper's statements too, - * one hop only: work in a helper's own helpers, or in an imported module, is not counted. - */ + /** Statement count across loader/action bodies, used by the triviality rule. */ statementCount: number; - /** - * The same count for one export's handlers alone, counted by the same walk. A handler serving both - * exports is counted once in `statementCount` and once in each of these, so the two do not sum to - * the entry point's total and must not be used as though they did. - */ + /** The same count for one export's handlers alone. A handler serving both exports is counted in + * each, so these do not sum to `statementCount`. */ loaderStatementCount: number; actionStatementCount: number; }; From 70a6bf8e8ab1762c8831e1d1b8ff3dc45b5514f1 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 14:53:56 +0100 Subject: [PATCH 110/117] test(observability-map): apply the comment rule to the tests Same pass over the test files. Each block now states in a line or two which regression the test below pins, and names the corpus entry or sibling test where the pairing is load bearing; the narrative around it is in the README. The KNOWN_GAPS entries keep a reason each, on the id they belong to, so neither expected failure is orphaned. --- .../src/checks/index.test.ts | 92 ++---- .../observability-map/src/cli.test.ts | 9 +- .../src/docstringReferences.test.ts | 52 +--- .../observability-map/src/integration.test.ts | 223 ++++----------- .../src/mutationCorpus.test.ts | 239 +++++----------- .../src/report/prComment.test.ts | 6 +- .../src/report/terminal.test.ts | 7 +- .../observability-map/src/scan.test.ts | 265 ++++++------------ .../observability-map/src/score.test.ts | 30 +- .../observability-map/src/sensitivity.test.ts | 8 +- .../observability-map/src/suppression.test.ts | 16 +- .../src/webappSymbols.test.ts | 57 +--- 12 files changed, 294 insertions(+), 710 deletions(-) diff --git a/internal-packages/observability-map/src/checks/index.test.ts b/internal-packages/observability-map/src/checks/index.test.ts index f0e80af528d..2f6e4e0e391 100644 --- a/internal-packages/observability-map/src/checks/index.test.ts +++ b/internal-packages/observability-map/src/checks/index.test.ts @@ -162,11 +162,8 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); - // I6. NARROW_TRY_STATEMENTS is an absolute count over the try block alone (guardsParse still - // required), not a ratio against the enclosing body, so it holds the exact boundary regardless of - // how big or small the rest of the function is: two statements binds the parsed result and still - // passes, a third means the try has started to cover the handler and fails, even though both - // guard the same parse. + // The exact boundary, held regardless of how big the rest of the function is: two statements binds + // the parsed result and passes, a third means the try has started to cover the handler. it("passes a parse guard that binds its result in exactly two statements", () => { const r = run( "error-classification", @@ -370,11 +367,8 @@ describe("error-classification", () => { expect(r.status).toBe("fail"); }); - // A6. isParseGuard compared the clause against ep.statementCount, the loader and the action and - // every one-hop helper summed together, rather than the statements of the body the clause is - // actually in. So an unrelated sibling handler or a fat helper in the same file diluted the - // denominator and relabelled the same broad swallow as a narrow parse guard. Byte-identical - // action, verdict must not move. + // Against the entry-point statement count, a sibling handler or a fat helper in the same file + // diluted the denominator and relabelled the same broad swallow as a narrow parse guard. it("gives the same verdict to a byte-identical swallow whether or not an unrelated sibling and helper share the file", () => { const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; export async function action({ request }) { @@ -424,12 +418,8 @@ describe("error-classification", () => { expect(withSiblingAndHelper.status).toBe("fail"); }); - // I6. Moving the denominator from the entry point to the enclosing body (A6) closed - // cross-body dilution but not same-body dilution: the rule was still a ratio, "unrelated - // statements dilute", wherever the unrelated statements live. Padding the SAME action with 11 - // inert statements after the try relabelled the identical broad swallow from fail to pass. - // isParseGuard is now an absolute count over the try block alone (NARROW_TRY_STATEMENTS), - // which nothing outside the try can dilute, in the same body or another. + // Same-body dilution, which moving the denominator to the enclosing body did not close: as a + // ratio, padding the SAME action with 11 inert statements took the identical swallow to pass. it("gives the same verdict to a byte-identical swallow whether or not it is padded with inert statements in the same body", () => { const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; export async function action({ request }) { @@ -507,11 +497,8 @@ describe("error-classification", () => { expect(r.status).toBe("not-applicable"); }); - // A7 as revised twice. A per-item error boundary inside a `.map()` callback is still not judged - // as the route's own catch, so it never sets `catches` and never speaks for the route's - // `tryStatementCount`. This catch SWALLOWS what it caught, and nothing the route owns decides, - // so the route still fails: judging refused catches on their evidence must not stop failing the - // relocated swallow, which is the anti-laundering half of the rule. + // The anti-laundering half of the boundary rule: judging refused catches on their evidence must + // not stop failing a swallow relocated behind one. it("fails a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { const source = `import { prisma } from "~/db.server"; export async function action({ request }) { @@ -581,12 +568,9 @@ describe("error-classification", () => { expect(r.detail).toContain("its only catches sit in iteration callbacks"); }); - // The refused-swallow arm is deliberately not conditioned on the route owning no catches. An - // own inert catch is what `wrap-body-in-rethrow`, a preserving corpus entry, adds to every - // route: were the arm gated on `catches.length === 0`, wrapping a per-item-swallow route in - // try/rethrow would read "every catch rethrows" and lift the fail to not-applicable, a rise - // that existed in the pre-evidence code and was masked only by the affected routes scoring 0 on - // every other check. + // Why the refused-swallow arm is not conditioned on the route owning no catches: an own inert catch + // is exactly what `wrap-body-in-rethrow` adds to every route, and the gate would lift this fail to + // not-applicable. it("fails a per-item swallow even when the route owns an inert rethrow catch", () => { const r = run( "error-classification", @@ -707,11 +691,9 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); - // I3. "Takes one way out regardless of what was thrown" is false of a clause that throws for - // some errors, and the strengthened `rethrows` makes such a clause a swallow by this check's - // definition: it decides nothing about the error, but it does not send everything the same way - // either. 16 clauses in the tree flipped `rethrows` this round and every one was eligible for the - // false wording. Whether `fail` is the right verdict for them is a separate question, parked. + // "Takes one way out regardless of what was thrown" is false of a clause that throws for some + // errors, and 16 clauses in the tree were eligible for the wording. Whether `fail` is the right + // verdict for them at all is a separate question, parked. it("does not accuse a clause that throws of taking one way out", () => { const r = run( "error-classification", @@ -741,13 +723,10 @@ describe("error-classification", () => { expect(r.detail).toContain("one way out"); }); - // I4. A route that owns a real classifying catch must never be told it owns none. `canRaise` does - // not list destructuring, and `const { a } = undefined` throws, so the owned catch dropped out of - // `reachable`; with the refused-swallow arm ordered off `reachable` the route was then accused of - // owning nothing that decides, which was simply false. The arm now reads own deciding catches - // through `guardMayRaise`, so the accusation is withheld and the route sits out exactly as it did - // before the arm existed. Asserted on `status`: an earlier version of this test asserted the - // absence of a detail string no arm ever emits, which could not fail. + // `canRaise` does not list destructuring, and `const { a } = undefined` throws, so an owned + // classifying catch drops out of `reachable`; the refused-swallow arm reads `guardMayRaise` instead. + // Asserted on `status`, since an earlier version asserted the absence of a detail string no arm + // emits, which could not fail. it("does not accuse a route that owns a catch of owning none", () => { const r = run( "error-classification", @@ -970,11 +949,8 @@ describe("error-classification", () => { expect(r.status).toBe("pass"); }); - // The verdict end of the walk's guaranteed-execution entries, one pair per entered construct. - // The evidence end is `the walk enters exactly the positions guaranteed to execute` in - // scan.test.ts; these hold the wrapped and unwrapped spellings to the same verdict, modeled on - // the switch pair above. Before the entries existed, every wrapper here turned a passing - // deciding clause into a fail with a detail line accusing it of ignoring the error. + // The verdict end of the walk's guaranteed-execution entries, one pair per entered construct. The + // evidence end is `the walk enters exactly the positions guaranteed to execute` in scan.test.ts. const CLAUSE_WRAPPED = (clauseBody: string) => `import { prisma } from "~/db.server"; export async function loader() { try { @@ -1750,11 +1726,8 @@ describe("auth-boundary: the guard accept-list", () => { expect(r.detail).toContain("no auth guard in the body"); }); - // The two live shapes that made the non-throwing variants worth crediting. Both act on the - // answer, which is why they are on the list; a route that calls one and ignores it is the - // residual, stated on `GUARDS`. - // Round C ruling 2. getUser and getUserId answer with null instead of throwing, so being called - // is not evidence of a boundary. They are credited only when the body reads the answer. + // `getUser` and `getUserId` answer with null instead of throwing, so being called is not evidence + // of a boundary: they are credited only when the body reads the answer. it("does not pass a route that resolves the caller and ignores the answer", () => { const r = run( "auth-boundary", @@ -1834,9 +1807,8 @@ describe("auth-boundary: the guard accept-list", () => { }); /** - * Per-export attribution. Every input `auth-boundary` reads used to be entry-point-wide, so one - * guarded export spoke for the whole file. Each `it` here goes green on the entry-point-wide - * version of exactly one of those inputs, which is why they are separate cases rather than one. + * Per-export attribution. Each `it` here goes green on the entry-point-wide version of exactly one of + * the three inputs, which is why they are separate cases rather than one. */ describe("auth-boundary: a guard credits only the export that calls it", () => { const TOKENS = `import { requireUserId } from "~/services/session.server"; @@ -1954,12 +1926,8 @@ describe("auth-boundary: a guard credits only the export that calls it", () => { expect(r.status).toBe("pass"); }); - /** - * The damper on the attribution, and the reason it is not simply "accuse every unguarded export". - * `auth.github.ts` and `auth.google.ts` are this shape: per export the loader is unguarded, and - * an entry-point-wide triviality rule calls the file non-trivial because the ACTION is not. Both - * routes went pass to fail on the real tree until `isTrivialExport` existed. - */ + /** The damper on the attribution: `auth.github.ts` and `auth.google.ts` went pass to fail on the + * real tree until `isTrivialExport` existed. */ it("reports not-applicable for a redirect-stub loader beside a guarded action", () => { const r = run( "auth-boundary", @@ -1979,12 +1947,8 @@ describe("auth-boundary: a guard credits only the export that calls it", () => { expect(r.detail).toBe("guarded in the body"); }); - /** - * The per-export excuse must read the export's own body and not the file's text. It read - * `ep.source` first, and `log-caller-scope-userid` in the mutation corpus, which prepends - * `logger.error(...)` to every body, put the word `logger` in this file and turned the untouched - * loader from excused into accused. A five-minute corpus run is the wrong place to catch that. - */ + /** The per-export excuse must read the export's own body and not the file's text, or + * `log-caller-scope-userid` puts the word `logger` in this file and accuses the untouched loader. */ it("does not un-excuse a redirect-stub loader because the file mentions a logger", () => { const r = run( "auth-boundary", diff --git a/internal-packages/observability-map/src/cli.test.ts b/internal-packages/observability-map/src/cli.test.ts index 9444d603b09..04d8f9121ad 100644 --- a/internal-packages/observability-map/src/cli.test.ts +++ b/internal-packages/observability-map/src/cli.test.ts @@ -4,12 +4,9 @@ import { join } from "node:path"; import { main, type Io } from "./cli.js"; /** - * A routes tree of this package's own making. These tests used to run against - * `apps/webapp/app/routes` and assert that `/api/v1/token` exists and that `api.v1.runs.ts` is line - * 2 of the output, which is an assertion about the webapp's contents rather than about this CLI. - * A webapp-only pull request renaming a route broke them, and `pr_checks.yml` did not run this - * suite for such a pull request, so the break landed on whoever pushed next. The one deliberate - * real-tree test lives in `integration.test.ts` and asserts only invariants that survive churn. + * A routes tree of this package's own making. Run against `apps/webapp/app/routes`, these asserted the + * webapp's contents rather than this CLI, and a route rename broke them for whoever pushed next. The + * one deliberate real-tree test lives in `integration.test.ts`. */ const ROUTES = mkdtempSync(join(tmpdir(), "obs-map-fixture-routes-")); diff --git a/internal-packages/observability-map/src/docstringReferences.test.ts b/internal-packages/observability-map/src/docstringReferences.test.ts index 64666ac5b95..c8399e3f561 100644 --- a/internal-packages/observability-map/src/docstringReferences.test.ts +++ b/internal-packages/observability-map/src/docstringReferences.test.ts @@ -5,54 +5,25 @@ import { CHECKS } from "./checks/index.js"; import { MUTATIONS } from "./mutations.js"; /** - * Every test name a docstring in `src/` claims to be covered by must exist. - * - * The rule this enforces has been asked for six times in prose and broken six times, most recently - * by a docstring naming `content-is-not-a-comment`, a test that was never written. Prose cannot - * enforce itself, so this does. - * - * What is checked, precisely, because a checker that overstates its reach is the same defect again: - * - * - every backticked kebab-case token in a `src/` comment, e.g. `empty-instanceof-if`. Those are - * never valid JavaScript identifiers, so in this package they are always a check id, a mutation - * corpus id, a test name, or one of the handful of domain words in `NOT_A_TEST_NAME` below. - * - a backticked glob, `dead-*`, which must match at least one corpus id by prefix. - * - every backticked prose phrase of `MINIMUM_TITLE_WORDS` words or more that contains no code - * punctuation, e.g. `jsx text is content, not a comment`. That is what a test title looks like - * and what a code sample does not. - * - * What is NOT checked, and each of these is a place a bad reference can still hide: - * - * - a reference written without backticks. - * - a test title of fewer than `MINIMUM_TITLE_WORDS` words. `throw e` and `new URL` are code, and - * telling a short title from short code needs more than punctuation. - * - a comment with no node after it. `commentText` collects leading ranges only, so a comment on - * the last line of a block or at the end of a file is never scanned at all. Every docstring in - * this package precedes a declaration, which is why the collector was written that way, and it - * is a coverage hole rather than a design choice. - * - a `.test.ts` file, or `mutations.ts`. Tests live in `src/` for colocation, but a docstring in a - * test or in the mutation corpus helper is exempted from this scan by name, the same as it was - * when both lived outside `src/` in a separate `test/` directory. - * - * The kebab half is the half that has actually failed. + * Every test name a docstring in `src/` claims to be covered by must exist. The rule was asked for six + * times in prose and broken six times, so prose does not enforce itself. Exactly what is and is not + * checked, and the coverage holes that leaves: README, "Tests, timeouts and CI". */ const SRC = resolve(__dirname); const TESTS = resolve(__dirname); -/** Excluded from the `files` scan below: every `.test.ts` is a test rather than a source, and - * `mutations.ts` is the mutation-corpus helper, not production source. Both live in `src/` now for - * colocation, so the exclusion has to be by name rather than by directory. */ +/** Excluded from the `files` scan below. Both live in `src/` for colocation, so the exclusion has to + * be by name rather than by directory. */ const MUTATIONS_HELPER = resolve(SRC, "mutations.ts"); -/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added - * here is a deliberate statement that the token names no test, and shows up in review as such. */ +/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added here + * is a deliberate statement that the token names no test, and shows up in review as such. */ const NOT_A_TEST_NAME = new Set([ // A `CheckStatus` value. "not-applicable", // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. "obs-map-disable-next-line", - // The worked example of a mistyped check id in `suppression.ts`. A misspelling of a check id is - // the thing being described, so it names no test by construction. + // The worked example of a mistyped check id, which names no test by construction. "eror-classification", // A route path segment quoted in `sensitivity.ts`, part of the vocabulary that file is about. "session-duration", @@ -73,8 +44,8 @@ function walkFiles(dir: string, suffix: string, out: string[] = []): string[] { return out; } -/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two - * lines reads as one phrase rather than one with a stray asterisk in it. */ +/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two lines + * reads as one phrase rather than one with a stray asterisk in it. */ function commentText(file: string): string { const source = readFileSync(file, "utf8"); const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); @@ -183,8 +154,7 @@ describe("docstrings in src name things that exist", () => { }); // The checker has to be able to fail, or it is decoration. These run the same predicates over an - // invented docstring rather than over `src/`, so the guarantee does not rest on `src/` currently - // happening to contain a bad reference. + // invented docstring, so the guarantee does not rest on `src/` happening to contain a bad reference. it("would reject a docstring naming a test that does not exist", () => { const invented = "see `content-is-not-a-comment` for the proof"; const token = /`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/.exec(invented)![1]!; diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 978d7964488..0f708578b33 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -14,43 +14,21 @@ import { buildReport } from "./score.js"; import { SCORED_CHECK_IDS } from "./checks/index.js"; /** - * This file's deliberate coupling to `apps/webapp/app/routes`. It is not the suite's only one, and - * saying it was is what let the paths filter be written for this file alone: - * `webappSymbols.test.ts` walks all of `apps/webapp/app`, `packages/plugins/src` and - * `internal-packages/rbac/src`, and `mutationCorpus.test.ts` scans the route tree behind an env - * gate. Everything else, including the CLI tests, runs against a fixture tree of this package's own - * making. + * This file's deliberate coupling to `apps/webapp/app/routes`, and not the suite's only one: + * `webappSymbols.test.ts` walks all of `apps/webapp/app` and two more trees, and + * `mutationCorpus.test.ts` scans the route tree behind an env gate. * - * The coupling is acceptable because nothing here names a route or a count: the scan must not - * crash, the entry point count must sit inside a wide band, and parse failures must be zero. Those - * survive routes being added, renamed and deleted, and they are the only things a fixture tree - * cannot tell us, since a fixture only contains shapes somebody thought to write down. - * - * What runs this for a webapp pull request is `.github/workflows/unit-tests-observability-map.yml`, - * called from `pr_checks.yml` behind an `obsmap` paths filter covering the whole of - * `apps/webapp/app` plus the report workflow, and listed in the `all-checks` aggregate so it - * actually gates. The filter is wider than this file's own coupling because the suite's is: - * `webappSymbols.test.ts` walks all of `apps/webapp/app`, and the describes below read - * `observability-map.yml`. A pull request touching this PACKAGE, or `packages/plugins/src` or - * `internal-packages/rbac/src`, reaches the same test by the other road: `internal` already matches - * `internal-packages/**` and `packages/**`, and `unit-tests-internal.yml` runs `turbo run test - * --filter "@internal/*"` over this package too. So every direction is gated and none is gated - * twice; the `obsmap` filter used to name the package as well, which ran this suite twice on every - * PR touching it. - * - * Two shapes were tried and rejected on the way here. Widening `pr_checks.yml`'s `internal` filter - * to the route paths ran all eighteen internal packages, twelve shards with postgres, clickhouse, - * redis and electric, to protect this one test. Putting the job in `observability-map.yml` - * instead was targeted but gated nothing, because `all-checks` needs an explicit list of jobs and - * cannot see another workflow. + * The coupling is acceptable because nothing here names a route or a count: the scan must not crash, + * the entry point count must sit inside a wide band, and parse failures must be zero. Those are the + * only things a fixture tree cannot tell us, since a fixture only contains shapes somebody thought to + * write down. How the whole suite is gated in CI: README, "Tests, timeouts and CI". */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); /** - * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk, which - * looks at flat files and at one `route.ts`/`route.tsx` per directory: this used to be a verbatim - * copy of that walk, which made `entryPoints.length < countCandidates()` a tautology that could - * not fail for any route shape both of them missed. + * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk: as a copy + * of it, `entryPoints.length < countCandidates()` was a tautology that could not fail for any route + * shape both of them missed. */ function countRouteModuleFiles(dir: string): number { let count = 0; @@ -65,25 +43,22 @@ function countRouteModuleFiles(dir: string): number { } beforeAll(() => { - // A hard failure rather than the `if (!existsSync(ROUTES)) return;` these tests opened with: if - // this package moves relative to apps/webapp, the real-tree coverage must disappear loudly. + // A hard failure rather than a silent skip: if this package moves relative to apps/webapp, the + // real-tree coverage must disappear loudly. if (!existsSync(ROUTES)) { throw new Error(`the webapp routes directory is missing: ${ROUTES}`); } }); -// B7. The build emits ESM (`module: ESNext`, and `cli.ts` uses `import.meta`) while `main` and -// `types` advertised it to a package.json with no module type. On node 24 that loads with a -// MODULE_TYPELESS_PACKAGE_JSON warning and a reparse rather than the throw older nodes give, which -// is a warning about an artifact this package tells other packages to import. +// The build emits ESM while `main` and `types` advertised it to a package.json with no module type, +// which on node 24 loads with a MODULE_TYPELESS_PACKAGE_JSON warning and a reparse. describe("the package it advertises", () => { const manifest = JSON.parse( readFileSync(resolve(__dirname, "../package.json"), "utf8") ) as Record; - // Asserted flat rather than behind an `if (!advertised) return`, which is the silent skip this - // round removed from the tests below: the decision was to keep the entry point and declare the - // module type, so dropping the entry point later should have to edit this, not slip past it. + // Asserted flat rather than behind a skip, so dropping the entry point later has to edit this + // rather than slip past it. it("declares the module type its build emits alongside the entry point it advertises", () => { expect(manifest.main).toBe("./dist/src/index.js"); expect(manifest.types).toBe("./dist/src/index.d.ts"); @@ -99,8 +74,8 @@ function read(path: string): string { return readFileSync(path, "utf8"); } -/** Comment lines dropped, for an assertion about what the YAML says rather than what its prose - * happens to mention. */ +/** Comment lines dropped, so an assertion is about what the YAML says rather than what its prose + * mentions. */ const withoutComments = (text: string) => text.replace(/^\s*#.*$/gm, ""); /** One job's block, from its key to the next key at job indent. */ @@ -117,21 +92,10 @@ const gate = (name: string) => job(name).split(" steps:")[0]!; const steps = (block: string) => block.split(/^ {6}- name: /m).slice(1); /** - * The one thing the docstring checker cannot reach. It walks `src/` only, so workflow prose is - * unpoliced, and the C1 defect was exactly that: two steps disagreeing about what a missing - * `/tmp/existing-comment-id` meant, under a comment claiming they agreed. The render step read it - * as "a comment exists" and emitted the resolved state, the upsert step read it as "no id" and - * POSTed, so a transient lookup failure either added a second marker comment beside the stale one - * or announced that findings were gone on a pull request that never had any. - * - * The sentinel pair those two shared is gone with the lookup, which moved into the `changes` job so - * the report job's own gate could read it. What replaces it is structural rather than agreed: the - * report job does not start unless the lookup finished cleanly, and the id it then uses is one job - * output that both steps read, so there is nothing left for two readers to disagree about. - * - * These are text checks over the workflow, not a parse of its semantics, so they catch the wiring - * coming apart and nothing about whether GitHub agrees. Named as such rather than sold as coverage - * of the file. + * The one thing the docstring checker cannot reach, since it walks `src/` only. What the C1 defect was + * and what replaced it: README, "Tests, timeouts and CI". These are text checks over the workflow + * rather than a parse of its semantics, so they catch the wiring coming apart and nothing about + * whether GitHub agrees. */ describe("the report workflow's one source of the comment id", () => { it("does not start the report job at all unless the lookup finished cleanly", () => { @@ -148,8 +112,8 @@ describe("the report workflow's one source of the comment id", () => { ).toEqual([]); }); - // The lookup is what lets the report job be gated, so it has to happen before it, in the job that - // an unrelated pull request pays for anyway. + // The lookup is what lets the report job be gated, so it happens in the cheap job an unrelated pull + // request pays for anyway. it("looks the comment up in the cheap job and not again in the report job", () => { expect(job("changes")).toContain("issues/${PR_NUMBER}/comments"); expect(job("changes")).toContain("pull-requests: read"); @@ -166,17 +130,10 @@ describe("the report workflow's one source of the comment id", () => { }); /** - * Round F. The workflow used to carry a `paths:` filter, and GitHub evaluates one of those per - * workflow, so a pull request whose diff stopped matching did not start the workflow at all: the - * resolved state could not fire and a comment from an earlier push stood for ever showing findings - * that were no longer in the diff. Confirmed on a throwaway pull request whose only route change was - * reverted, and the case that matters is worse, because a pull request touching a route and other - * files whose author reverts only the route change still has a diff, still does not match, and still - * has the stale comment. - * - * So the workflow runs on every pull request and the gating is internal. What that has to preserve - * is the cost: the scans stay gated on the paths, and a pull request with nothing relevant and no - * comment does not start the report job. + * The workflow runs on every pull request with the gating internal, because GitHub evaluates one + * `paths:` filter per workflow and a pull request whose diff stopped matching never started it at all, + * leaving an earlier push's comment standing for ever. See README, "CI". What that has to preserve is + * the cost, which is what these assert. */ describe("the report workflow reconciles a comment the paths no longer reach", () => { it("runs on every pull request rather than only on the paths it watches", () => { @@ -204,8 +161,7 @@ describe("the report workflow reconciles a comment the paths no longer reach", ( expect(render).toMatch(/SCANNED" != "true" \]; then\s+emit --resolved/); }); - // Change 2. The comment is edited in place across pushes, so it has to say which push it reflects. - // The renderer takes the sha and the URL as data; building the URL is the workflow's job because + // The renderer takes the sha and the URL as data; building the URL is the workflow's job, because // the workflow is what has the two shas. it("forwards the head sha and a compare URL for the pull request's range", () => { const render = steps(job("report")).find((step) => step.startsWith("📝 Render comment"))!; @@ -219,17 +175,8 @@ describe("the report workflow reconciles a comment the paths no longer reach", ( }); /** - * Both scan steps used to capture the scanner's stdout with a shell redirect, into files the - * renderer then `JSON.parse`s. Anything else reaching stdout therefore corrupted the report: - * `pnpm --filter` takes its recursive path, and some versions of pnpm announce - * `Scope: N of M workspace projects` on it. A single line of that in head.json fails the parse, - * and the workflow degrades to the stale-report comment on every run, quietly and permanently. - * - * It does not reproduce on the 10.33.2 the workflow pins, which was checked. What is asserted here - * is the shape that cannot have the bug at all rather than the version that happens not to: the - * scanner writes its own file through `--out`, so stdout carries log output and nothing else. - * The same is not yet true of the render step, which has no `--out` to reach for; a banner there - * puts a stray line in a markdown comment instead of breaking a parse, so it is left alone. + * Asserts the shape that cannot have the stdout-capture bug rather than the pnpm version that happens + * not to. Why, and why the render step is left alone: README, "Tests, timeouts and CI". */ describe("the report workflow's two scan steps", () => { it("let the scanner write its own report rather than capturing stdout", () => { @@ -248,13 +195,8 @@ describe("the report workflow's two scan steps", () => { }); /** - * The gating half of the same problem. A test job that nothing waits for is decoration, and the - * first attempt at this was exactly that: a job inside `observability-map.yml`, which reads well - * and gates nothing, because `pr_checks.yml`'s `all-checks` aggregate needs an explicit list of - * jobs and cannot see another workflow. - * - * Text checks again, over two workflow files. They catch the wiring coming apart, not whether - * GitHub agrees, which only a pull request can answer. + * The gating half of the same problem: a test job that nothing waits for is decoration. Text checks + * again, over two workflow files. */ describe("the package's tests are wired into the gate", () => { const PR_CHECKS = resolve(WORKFLOWS, "pr_checks.yml"); @@ -267,11 +209,8 @@ describe("the package's tests are wired into the gate", () => { expect(read(REUSABLE)).toContain("workflow_call"); }); - // Round 5. The filter watched `apps/webapp/app/routes/**` while the suite reads more than that, - // so a rename outside the routes folder matched only `webapp`, ran no job that runs this suite, - // and broke the build for whoever pushed next. Asserted as the whole set the suite reads and no - // other filter covers, rather than as the one path that prompted the filter, because the routes - // entry looked complete right up until it wasn't. + // Asserted as the whole set the suite reads and no other filter covers, rather than as the one path + // that prompted the filter, because the routes entry looked complete right up until it wasn't. it("watches every webapp path the internal filter misses, not just the routes folder", () => { const filter = read(PR_CHECKS).split(" obsmap:")[1]!.split(" cli:")[0]!; // webappSymbols.test.ts walks all of apps/webapp/app, not just routes. @@ -280,8 +219,8 @@ describe("the package's tests are wired into the gate", () => { expect(filter).toContain("'.github/workflows/observability-map.yml'"); }); - // The other two trees webappSymbols.test.ts reads. They belong to `internal`, not here, and this - // pins the reason so the obvious-looking addition has to argue with a test first. + // The other two trees `webappSymbols.test.ts` reads belong to `internal`, so this pins the reason + // and the obvious-looking addition has to argue with a test first. it("leaves the two non-webapp roots it reads to the internal filter", () => { const text = read(PR_CHECKS); const obsmap = text.split(" obsmap:")[1]!.split(" cli:")[0]!; @@ -293,10 +232,8 @@ describe("the package's tests are wired into the gate", () => { expect(internal).toContain("'internal-packages/**'"); }); - // Round E item 6. `internal` matches `internal-packages/**` and `unit-tests-internal.yml` runs - // `turbo run test --filter "@internal/*"`, so naming this package here as well ran the suite - // twice on every pull request touching it. Asserted rather than left to the next reader, because - // the duplicate looks like the obviously right entry to add back. + // Naming this package here as well ran the suite twice on every pull request touching it. Asserted + // rather than left to the next reader, because the duplicate looks like the right entry to add back. it("leaves the package's own paths to the internal filter, so the suite runs once", () => { const text = read(PR_CHECKS); const obsmap = text.split(" obsmap:")[1]!.split(" cli:")[0]!; @@ -319,17 +256,15 @@ describe("the package's tests are wired into the gate", () => { expect(read(REPORT)).not.toContain("run test"); }); - // Round E item 5. The corpus measures the tool's resistance to laundering, which only an edit to - // the tool can weaken, so it does not belong on every route pull request at four and a half - // minutes a run. The nightly is the other half of that trade and is asserted with it: dropping - // the schedule would leave tree drift uncovered rather than covered late. + // The nightly is the other half of the trade and is asserted with it: dropping the schedule would + // leave tree drift uncovered rather than covered late. it("runs the corpus on the package's own paths and on a schedule, not on every route PR", () => { const text = read(REPORT); const corpus = text.split(" mutation-corpus:")[1]!.split(" steps:")[0]!; expect(corpus).toContain("needs.changes.outputs.package == 'true'"); - // The corpus filter alone, which the report's own gate now sits beside: the two are separate - // filter entries, and this one has to stay off the route tree. + // The corpus filter alone, a separate entry from the report's own gate beside it, and this one + // has to stay off the route tree. const filter = text.split(" package:")[1]!.split(" routes:")[0]!; expect(filter).toContain("'internal-packages/observability-map/**'"); expect(filter).not.toContain("apps/webapp/app/routes"); @@ -338,11 +273,9 @@ describe("the package's tests are wired into the gate", () => { expect(text).toContain("cron:"); }); - // Round E item 4. Two reviewers read the README's "merge base" against the workflow's base.sha - // and reported the workflow. The checkout is a pull_request default, so the tree scanned is - // GitHub's test merge commit and base.sha is one of its parents, which makes base.sha the right - // base and the old wording the bug. Pinned so the wording cannot drift back without the workflow - // moving with it. + // Two reviewers read the README's old "merge base" wording against the workflow's base.sha and + // reported the workflow; the wording was the bug. Pinned so it cannot drift back without the + // workflow moving with it. it("describes the base the report workflow actually scans against", () => { expect(read(REPORT)).toContain("github.event.pull_request.base.sha"); const readme = readFileSync(resolve(__dirname, "../README.md"), "utf8"); @@ -353,24 +286,9 @@ describe("the package's tests are wired into the gate", () => { }); /** - * The third road into this suite, after the two workflows above: `turbo run test`, which is what - * `pnpm run test` and `pnpm run test:internal` reach it by. - * - * Turbo keys a task's cache on the package's own files. This suite's real inputs are mostly not - * its own files, they are `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` - * and the workflow files read above, so turbo happily replayed a pass recorded before a route - * changed. Measured rather than argued: a route file with a syntax error in it makes - * `parses every route file and produces a report inside a wide band` fail under vitest, and the - * same tree came back FULL TURBO in 301ms with the failure cached away as a success. - * - * So the task is uncacheable, and this asserts that, because the config is one line and reads like - * a performance oversight to anyone who does not know what the suite reads. - * - * What this does not assert is the rejected alternative. `inputs` can name `../../apps/webapp/...` - * and does bust the cache, but it replaces turbo 1.x's default file set instead of adding to it, - * so it drops the package's own files from the hash unless every one of them is listed too; that - * was measured the same way, by editing `vitest.config.ts` and getting FULL TURBO back. The - * reasoning lives in `turbo.json` next to the config it explains. + * The third road into this suite, `turbo run test`. Asserts the task is uncacheable, because the + * config is one line and reads like a performance oversight to anyone who does not know what the suite + * reads. Measurement and the rejected `inputs` alternative: README, "Tests, timeouts and CI". */ describe("the third road in, turbo", () => { it("keeps its test task out of the turbo cache", () => { @@ -384,9 +302,7 @@ describe("the third road in, turbo", () => { }); describe("counting candidates independently of the scanner", () => { - // The counter is only worth having if it disagrees with the scanner somewhere. It does: the - // scanner attributes nothing to a nested file that is not `route.ts`/`route.tsx`, and the - // counter counts every module file at every depth. + // The counter is only worth having if it disagrees with the scanner somewhere, and it does. it("counts a nested non-route file the scanner does not attribute to any route", () => { const dir = mkdtempSync(join(tmpdir(), "obs-map-count-")); mkdirSync(join(dir, "components")); @@ -403,28 +319,10 @@ describe("counting candidates independently of the scanner", () => { }); /** - * Timeout for the two real-tree tests, which do not fit the suite's 10s default: the first runs a - * ts.Program per route file for the parse diagnostics and walks the tree a second time to count - * candidates, the second scans the tree twice and re-scans every source with the suppression - * directive prepended. - * - * It is a hang detector and nothing else. Neither test asserts anything about how long the scan - * takes, so a number tight enough to be a performance budget would only be a way to fail on a busy - * runner, and a performance budget that flakes gets the whole suite marked unreliable. - * - * The old 30s and 60s were chosen on an idle machine and the 30s one does flake. Measured on an - * 8-core box, this file alone at load average 0.9: 6.3-6.4s for the scan, 10.8-11.2s for the sweep. - * Twenty-four runs of it as two batches of twelve concurrent copies on those same 8 cores: - * 24.2-34.0s for the scan and 27.6-39.7s for the sweep, with one of the first twelve dying on - * "Test timed out in 30000ms". That contention is not hypothetical: - * `.github/workflows/unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"` as - * twelve concurrent shard processes on one runner, and this file executes inside one of them. - * - * The local reproduction is deliberately harsher than CI, which is why it is the thing to size - * against: twelve processes over 8 cores is 1.5 per core where the 32-vCPU runner is 0.375. 120s is - * 3x the worst contended run measured and about 11x the idle sweep. 60s was the other candidate and - * is not enough on those numbers: the sweep already reached 39.7s, which is 1.5x, and a margin that - * thin on a machine nobody controls is how the 30s got here. + * Timeout for the two real-tree tests, which do not fit the suite's 10s default. A hang detector and + * nothing else: neither test asserts anything about how long a scan takes, so a number tight enough to + * be a performance budget would only be a way to fail on a busy runner. The contention measurements + * behind 120s, and why 60s is not enough: README, "Tests, timeouts and CI". */ const TREE_SCAN_TIMEOUT = 120_000; @@ -446,11 +344,8 @@ describe("scanning the real webapp routes", () => { TREE_SCAN_TIMEOUT ); - /** - * The per-export split against the entry-point-wide union it came from, on every real route. - * `scan.test.ts` pins the same property on fixtures; this is the version that sees the shapes - * nobody thought to write down, and the two representations only stay honest while both hold. - */ + /** The per-export split against the union it came from, on every real route. `scan.test.ts` pins the + * same property on fixtures; this is the version that sees the shapes nobody wrote down. */ it( "every callee name is attributed to an export that exists", () => { @@ -477,10 +372,8 @@ describe("scanning the real webapp routes", () => { TREE_SCAN_TIMEOUT ); - // A1, exhaustive: every scored check suppressed on every real route, zero behavioural change. - // The old measured-from-visible logic took this global from 17 to 33 and measured from 412 to - // 176, because every entry whose only applicable checks were suppressed dropped out of the - // mean. Measured must not move: every entry point that had something applicable still does. + // Every scored check suppressed on every real route. The old measured-from-visible logic took the + // global from 17 to 33 and measured from 412 to 176, so `measured` must not move either. it( "suppressing every scored check on every real route does not raise the global", () => { diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index ef2094e0aee..42bee6f733b 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -7,80 +7,42 @@ import { ADDITIVE_IDS, MUTATIONS, type Mutation } from "./mutations.js"; import { CHECKS } from "./checks/index.js"; /** - * The tree-scale mutation corpus. + * The tree-scale mutation corpus. Every laundering shape anyone has found is an entry, each rewrites + * the whole real route tree in a temp copy, and each is held to four assertions: the published global + * does not rise, the mean over the routes measured in both runs does not rise, no individual route + * rises or drops out of the measured set, and the mirror of that, no individual route falls. * - * The tool's central claim is that no semantics-preserving edit to a route raises its score. Three - * rounds argued that claim shape by shape and lost each time. This file turns it into evidence - * instead: every laundering shape anyone has found is a corpus entry, and each entry rewrites the - * whole real route tree in a temp copy and is held to three assertions. - * - * - the published global does not rise. That is the figure the claim is about. - * - the mean over the routes measured in BOTH runs does not rise. Same comparison at full - * precision, with the population held fixed so it measures scores rather than denominators. - * - for a semantics-preserving rewrite, no individual route's score rises and no measured route - * drops out of the measured set. The tree mean can hide a route going up by taking another down; - * `[0].map(...)` is exactly that shape. - * - the mirror of that, for a semantics-preserving rewrite: no individual route's score FALLS on - * the routes measured in both runs. A fall is a false accusation, which is the direction that - * gets the tool switched off, and it went unasserted for three rounds while 19 entries regressed - * 104 routes (see `fallsIn`). Exactly two entries carry a permanent `lowers` exemption, with the - * reason on the entry and the residual shape asserted instead of waived. - * - * Tree scale, not per-fixture, because that is where laundering pays. A shape that moves one - * hand-written fixture by 50 points may move the tree by nothing; a shape that moves the tree is the - * one worth defending. - * - * The honest statement this file supports is "these N mutations are defended, and here they are", - * never "unpaddable". - * - * Runtime is roughly six seconds per entry, which is why the whole file is gated behind - * `OBS_MAP_MUTATION_CORPUS=1`. The `observability-map` workflow sets it, so the gate keeps the - * default suite fast without making this the thing nobody runs. + * Tree scale rather than per fixture, because that is where laundering pays. The honest statement this + * file supports is "these N mutations are defended, and here they are", never "unpaddable". Roughly + * six seconds an entry, which is why the file is gated behind `OBS_MAP_MUTATION_CORPUS=1`. */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; /** - * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, - * so closing the hole later turns this file red until the entry is moved back out deliberately. - * - * `dead-classifying-try-with-call` is the shape `dead-classifying-try` only looked like it closed. - * `canRaise` accepts any call at all, so `try { String(0); }` reads as a clause guarding real work - * and takes the tree from 19 to 44, raising 224 routes, exactly as `try { 0; }` did before it was - * refused. Telling an inert call from one that can throw needs types the scanner does not have. - * The docstrings in `scan.ts`, `types.ts` and `errorClassification.ts` say the rule refuses - * `try { 0; }` and is defeated by one call, rather than claiming the family is closed. - * - * `dead-branch-after-if-true` used to be listed here on a measurement that was wrong. See the round - * A fix 3 report; the short version is that the rejected alternative was implemented with the exit - * flag raised before each statement's own branch check, which makes every deciding statement refuse - * itself. Raising it after is byte-identical on the real tree and closes the shape, so the entry is - * defended now and the `if (true)` family needed no condition folding after all. - * - * `dead-conjunction-instanceof-if` is the sibling of `dead-armed-instanceof-if` that the arm-liveness - * fix does not close. `selectsADistinctPath` now folds a dead ARM; a dead CONDITION still reaches - * the grant, and `e instanceof Error && false` is exactly that, a guard that references the caught - * binding and can never be true. No fold in `scan.ts` can see it, because `literalTruth` treats - * `&&` and `||` as always null on purpose so that a live guard can never be read as dead. Widening - * that fold is a different rule from the one this round fixed and needs its own measurement, so the - * shape is recorded and running rather than left for the next person to rediscover. + * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, so + * closing the hole later turns this file red until the entry is moved back out deliberately. Both + * gaps are described at length in README, "The mutation harness". */ const KNOWN_GAPS = new Set([ + // `canRaise` accepts any call at all, so `try { String(0); }` reads as a clause guarding real work + // and takes the tree from 19 to 44, exactly as `try { 0; }` did before it was refused. Telling an + // inert call from one that can throw needs types the scanner does not have. "dead-classifying-try-with-call", + // `selectsADistinctPath` folds a dead ARM but not a dead CONDITION, and `literalTruth` treats `&&` + // as always null on purpose so a live guard can never be read as dead. Widening that fold is a + // different rule and needs its own measurement. "dead-conjunction-instanceof-if", ]); type SourceFile = { relativeName: string; source: string }; /** - * Route modules exactly as `scanDirectory` enumerates them, because it is the same enumeration and - * no longer a copy of it. `isScannableFile` had already replaced the file half of the copy; the - * directory half survived, so "one `route.ts(x)` per immediate subdirectory" was still written - * twice. A harness that reads a different tree from the scanner reports files and sites the scan - * never saw, and those counts are what the thresholds below rest on. - * - * Read once; every mutation rewrites this list rather than the tree on disk. + * Route modules exactly as `scanDirectory` enumerates them, because it is the same enumeration and no + * longer a copy of it: a harness reading a different tree reports files and sites the scan never saw, + * and those counts are what the thresholds below rest on. Read once; every mutation rewrites this list + * rather than the tree on disk. */ function readTree(dir: string): SourceFile[] { return routeModuleFiles(dir).map((file) => ({ @@ -104,15 +66,14 @@ function materialize(files: SourceFile[]): string { type Measurement = { global: number | null; - /** The unrounded mean the global is a rounding of. A rise of less than half a point is invisible - * in `global` and is still a rise, so the assertions read this. */ + /** The unrounded mean `global` rounds. A rise of less than half a point is invisible in `global` + * and is still a rise. */ exactMean: number; measured: number; entryPoints: number; parseFailures: number; /** Per route file, so a mutation that raises one route while lowering the tree is still caught. - * `[0].map(...)` is exactly that shape: it deletes a route's catches, which takes a failing route - * to 100 and a passing one to nothing, and the two cancel in the global. */ + * `[0].map(...)` is that shape, and the two movements cancel in the global. */ perEntry: Map; }; @@ -146,13 +107,9 @@ function measure(files: SourceFile[]): Measurement { type Rise = { fileName: string; from: number; to: number; before: string; after: string }; /** - * Route files the mutation made look better, worst first. Two ways to qualify, both counted: the - * score went up, or a route that was being measured stopped being measured. The second is a rise - * too. An unmeasured route's score is the vacuous 100 and it leaves every mean, so dropping out of - * the measured set is the most complete form of the thing the property forbids. - * - * A route the mutation removed from the report entirely is not counted here; the entry-point guard - * catches that instead. + * Route files the mutation made look better, worst first. Two ways to qualify: the score went up, or a + * measured route stopped being measured, which is the most complete form of the thing the property + * forbids. A route removed from the report entirely is the entry-point guard's business instead. */ function risesIn(baseline: Measurement, after: Measurement): Rise[] { const rises: Rise[] = []; @@ -175,17 +132,13 @@ function risesIn(baseline: Measurement, after: Measurement): Rise[] { type Fall = { fileName: string; from: number; to: number; before: string; after: string }; /** - * The mirror of `risesIn`: route files the mutation made look WORSE, worst first. A preserving - * edit lowering a route's score is a false accusation, the direction that gets the tool switched - * off, and for three rounds it was structurally invisible here because only rises were asserted. - * 19 of 43 preserving entries were regressing 104 routes when it was first measured. + * The mirror of `risesIn`: route files the mutation made look WORSE. A preserving edit lowering a + * score is a false accusation, and for three rounds it was structurally invisible here, with 19 of 43 + * preserving entries regressing 104 routes when it was first measured. * - * Only routes measured in BOTH runs are compared. A route ENTERING the measured set is not a fall: - * unmeasured routes score the vacuous 100, so a mutation that brings one in at 50 registers a - * 100 -> 50 "fall" that is nothing of the kind. A route LEAVING the measured set is already - * counted by `risesIn`, not double-counted here. `compared` is the size of the both-measured - * population, asserted against `baseline.measured` in every preserving entry so a silently - * shrunken comparison cannot pass. + * Only routes measured in BOTH runs are compared: one ENTERING the measured set would register a + * 100 to 50 fall that is nothing of the kind, and one LEAVING it is already `risesIn`'s business. + * `compared` is asserted against `baseline.measured` so a silently shrunken comparison cannot pass. */ function fallsIn(baseline: Measurement, after: Measurement): { falls: Fall[]; compared: number } { const falls: Fall[] = []; @@ -212,10 +165,8 @@ function checkStatuses(checks: string): Map { } /** Mean score over the routes measured in BOTH runs. The plain mean moves when the measured - * population moves, which a mutation can do without making any route look better: an inert - * try/catch takes 15 trivial routes off the exemption list and into the report, and a route joining - * at 50 raises a tree averaging 15 while itself having gone from an unmeasured 100 to a measured 50. - * Holding the population fixed is what makes the comparison about the scores. */ + * population moves, which a mutation can do without making any route look better, so holding the + * population fixed is what makes the comparison about the scores. */ function commonMean(baseline: Measurement, after: Measurement): { before: number; after: number } { let sumBefore = 0; let sumAfter = 0; @@ -247,17 +198,10 @@ function mutate( } /** - * Deliberately NOT gated behind `OBS_MAP_MUTATION_CORPUS`, unlike everything below it. - * - * This is the guard for the way the corpus actually failed. `auth-scope` was added a round after - * `suppress-every-check` was written and never added to its directive list, so the "a suppression - * cannot raise a score" invariant went untested at tree scale for the 19 routes that check applies - * to, while the entry's own description said "every check". Nothing noticed, because the entry - * still passed: omitting a check from the sweep leaves its failures in place, which lowers the - * score rather than raising it, so the corpus cannot catch its own omission by failing. - * - * A registry assertion can, and it belongs in the default suite so that adding a check without - * extending the corpus turns `pnpm test` red rather than a job nobody runs locally. + * Deliberately NOT gated behind `OBS_MAP_MUTATION_CORPUS`, unlike everything below it: the corpus + * cannot catch its own omission by failing, since omitting a check from the sweep lowers the score + * rather than raising it. Belongs in the default suite so adding a check without extending the corpus + * turns `pnpm test` red rather than a job nobody runs locally. See README, "The mutation harness". */ describe("the corpus keeps up with the check registry", () => { it("suppresses every registered check in the exhaustive sweep", () => { @@ -269,12 +213,10 @@ describe("the corpus keeps up with the check registry", () => { expect(missing).toEqual([]); }); - // Ungated for the same reason as the sweep assertion above: it reads no route tree and costs - // nothing, so gating it would only hide a stale list from the run people actually do. + // Ungated for the same reason as the sweep assertion above: it reads no route tree, so gating it + // would only hide a stale list from the run people actually do. Every corpus entry once removed or + // restructured real signal and none added fake signal, which is where the two largest holes lived. it("covers the additive direction, not only the subtractive one", () => { - // Every corpus entry once removed or restructured real signal, and none added fake signal. The - // two largest holes ever found here lived in that blind spot, so the class is asserted rather - // than left to whoever edits the list next. const ids = new Set(MUTATIONS.map((m) => m.id)); expect(ADDITIVE_IDS.filter((id) => !ids.has(id))).toEqual([]); expect(ADDITIVE_IDS.length).toBeGreaterThanOrEqual(8); @@ -282,16 +224,15 @@ describe("the corpus keeps up with the check registry", () => { }); /** - * Ungated, because a `preserving` entry that changes behaviour is a false negative in the property - * the whole file exists to argue, and the shape is cheaper to state on a two-line fixture than to - * wait for a route to grow it. + * Ungated, because a `preserving` entry that changes behaviour is a false negative in the property the + * whole file exists to argue. */ describe("a preserving mutation preserves what the route does", () => { it("leaves a directive prologue alone when merging comma expressions", () => { const merge = MUTATIONS.find((m) => m.id === "merge-comma-expressions")!; const result = merge.apply("api.v1.a.tsx", '"use client";\nfoo();\nbar();\n'); - // The first two assertions carry the test. Without them `?? ""` let it pass when the mutation - // did not apply at all, and it would still have passed had the mutation deleted the directive. + // The first two assertions carry the test: without them it passed when the mutation did not apply + // at all, and would have passed had the mutation deleted the directive. expect(result).toBeDefined(); expect(result?.source).toContain('"use client";'); expect(result?.source).not.toContain('"use client",'); @@ -302,10 +243,9 @@ describe("a preserving mutation preserves what the route does", () => { const describeCorpus = ENABLED && existsSync(ROUTES) ? describe : describe.skip; /** - * Each entry rescans the whole tree, so the suite's default per-test timeout is far too short. Set - * here rather than left to a `--testTimeout` flag: the flag only helps someone who already knows to - * pass it, and without it the corpus fails as a timeout, which reads as a broken harness rather - * than a slow one. + * Each entry rescans the whole tree. Set here rather than left to a `--testTimeout` flag, which only + * helps someone who already knows to pass it, and without which the corpus fails as a timeout and + * reads as a broken harness rather than a slow one. */ const ENTRY_TIMEOUT_MS = 120_000; @@ -314,29 +254,16 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME let baseline: Measurement | null = null; // In a hook rather than the suite body, which Vitest runs during collection where no test timeout - // applies and a throw has no test name to attach to. The baseline is the single most expensive - // step in the file, so it is the one that must be inside something that can be timed out and - // reported. `beforeAll` takes its own timeout. + // applies and a throw has no test name to attach to. The baseline is the most expensive step here. beforeAll(() => { files = readTree(ROUTES); baseline = measure(files); }, ENTRY_TIMEOUT_MS); /** - * The corpus's own population, against the scanner's. - * - * The whole-body entries wrap what `entryBodies` finds, and that helper read two of the four - * export forms `scan.ts` reads. It missed `export const { action, loader } = builder(...)`, - * `const { action } = builder(...); export { action };` and `export const action = route.action`, - * which is 36 of the tree's entry points: the corpus was testing less than its entry count - * implied, and no assertion could notice, because a mutation that reaches fewer routes lowers the - * score rather than raising it. Same failure mode as the `suppress-every-check` omission above, - * so the answer is the same: assert the population rather than wait for a verdict to move. - * - * `admin.tsx` is the one documented exclusion. Its handler is a concise arrow - * (`async ({ user }) => typedjson({ user })`) with no block for a block wrapper to wrap, which is - * a limit of the rewrite rather than a gap in the enumeration. It is named rather than counted so - * a second one cannot appear silently. + * The one documented exclusion from the population assertion below. `admin.tsx`'s handler is a + * concise arrow with no block for a block wrapper to wrap, which is a limit of the rewrite rather + * than a gap in the enumeration. Named rather than counted so a second one cannot appear silently. */ const CONCISE_ARROW_BODIES = new Set(["admin.tsx"]); @@ -362,9 +289,9 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME it("has a baseline worth mutating", () => { expect(baseline).not.toBeNull(); expect(baseline!.entryPoints).toBeGreaterThan(300); - // The falls assertions compare over the routes measured in both runs and pin that population - // to `baseline.measured`, so the baseline itself has to be big enough that a broken scan - // cannot produce a tiny population the mirror trivially holds over. + // The falls assertions pin their population to `baseline.measured`, so the baseline itself has to + // be big enough that a broken scan cannot produce a tiny population the mirror trivially holds + // over. expect(baseline!.measured).toBeGreaterThan(300); expect(baseline!.global).not.toBeNull(); console.log( @@ -374,24 +301,9 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME }); /** - * How much a mutation must reach before its result means anything. A mutation that silently - * matched nothing would otherwise "pass" by leaving the tree alone, which is the exact failure - * mode that let earlier rounds believe a shape was defended. - * - * Sites, not only files, and sites are what the threshold is really on. A file count says a - * rewrite touched a file, not that it reached anything inside it: eleven entries reported 172 - * files while landing in a position that mattered for 26 of the tree's 260 catch clauses, because - * the splice went after statements that had already returned. `prependToEveryCatch` now splices - * at the head of the clause, so all 260 count, and this threshold is what would notice if a later - * change quietly took that back. - * - * The guard the design asked for, verdict movement, cannot be used, though not for the reason an - * earlier version of this comment gave. Plenty of defended entries move verdicts hard: - * `delete-every-catch` takes the tree from 19 to 8 and `dead-throw-after-switch` to 10. The - * narrower true reason is that the IDEAL defended shape is one the scanner is blind to, and those - * move nothing at all: `dead-if-false` and the ten entries beside it are defended precisely - * because the tree comes out identical. Requiring movement would fail exactly the entries that - * work best. Site count is the reachable version of the same intent. + * How much a mutation must reach before its result means anything, since one that silently matched + * nothing would otherwise pass by leaving the tree alone. On sites and not only files, and why + * verdict movement cannot be the guard instead: README, "The mutation harness". */ const MINIMUM_FILES_TOUCHED = 20; const MINIMUM_SITES_TOUCHED = 40; @@ -406,11 +318,9 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME const after = measure(mutated); - // A mutation that stops the tree parsing, or that hides a route from the scanner, has not - // tested the property: whatever the score does afterwards is measuring a different tree. The - // route guard is exact rather than tolerant, because a route the mutated scan cannot see is - // one `risesIn` and `commonMean` both skip, and a rewrite that makes a route unscannable is - // itself a finding. + // A mutation that stops the tree parsing, or hides a route from the scanner, has not tested the + // property. Exact rather than tolerant, because a route the mutated scan cannot see is one both + // `risesIn` and `commonMean` skip. expect(after.parseFailures).toBe(baseline!.parseFailures); expect(after.entryPoints).toBe(baseline!.entryPoints); expect([...baseline!.perEntry.keys()].filter((f) => !after.perEntry.has(f))).toEqual([]); @@ -446,32 +356,23 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME // and not who is in the denominator. expect(common.after).toBeLessThanOrEqual(common.before + 1e-9); - // Per route, for the preserving half of the corpus. This is the property as stated: an edit - // that does not change what a route does must not make that route look better, whatever it - // does to the tree's mean. The deleting half is exempt on purpose: a route whose only failing - // check was error-classification really does leave the denominator when its catch goes, which - // the design chose over crediting a route for deleting its error handling, and the global - // figure above is where that trade is held to account. + // Per route, for the preserving half. The deleting half is exempt on purpose: a route whose only + // failing check was `error-classification` really does leave the denominator when its catch + // goes, and the global figure above is where that trade is held to account. if (mutation.kind === "preserving") { expect(rises.map((r) => `${r.fileName} ${r.from}->${r.to}`)).toEqual([]); - // The mirror direction. A preserving edit must not make any route look WORSE either: a - // fall here is a false accusation, the direction that gets the tool switched off, and it - // went unasserted for three rounds while 19 entries regressed 104 routes. Together with - // the rises assertion and the entry-point guards above, this pins per-route score - // EQUALITY for a preserving entry. The comparison population is pinned to the whole - // measured baseline first, so a silently shrunken population cannot pass vacuously. + // The mirror direction, which with the rises assertion and the entry-point guards above pins + // per-route score EQUALITY for a preserving entry. Population pinned first, so a silently + // shrunken one cannot pass vacuously. expect(compared).toBe(baseline!.measured); if (mutation.lowers === undefined) { expect(falls.map((f) => `${f.fileName} ${f.from}->${f.to}`)).toEqual([]); } else { - // An exempted entry must still be falling, or the exemption is stale and has to be - // removed deliberately rather than sitting as cover for the next defect. + // An exempted entry must still be falling, or the exemption is stale cover for the next + // defect. expect(falls.length).toBeGreaterThan(0); - // And the falls must have exactly the measured residual shape the exemption was - // granted for: `error-classification` moving pass -> not-applicable, every other - // check's status unchanged, nothing anywhere moving to fail. Anything else is a new - // defect hiding under the exemption. + // And each fall must have exactly the residual shape the exemption was granted for. for (const fall of falls) { const before = checkStatuses(fall.before); const now = checkStatuses(fall.after); diff --git a/internal-packages/observability-map/src/report/prComment.test.ts b/internal-packages/observability-map/src/report/prComment.test.ts index 870128d0bb1..4d57654f79d 100644 --- a/internal-packages/observability-map/src/report/prComment.test.ts +++ b/internal-packages/observability-map/src/report/prComment.test.ts @@ -422,10 +422,8 @@ describe("renderPrComment", () => { }); /** - * The comment is edited in place across pushes, so on its own it says nothing about which push it - * reflects. Every comment the job posts carries the head sha as a link to the pull request's compare - * range, and the commit arrives as data so these renderers stay pure and a local run without it - * still renders. + * The comment is edited in place across pushes, so every comment the job posts carries the head sha as + * a link to the compare range. The commit arrives as data, so these renderers stay pure. */ describe("the commit stamp", () => { const COMMIT = { diff --git a/internal-packages/observability-map/src/report/terminal.test.ts b/internal-packages/observability-map/src/report/terminal.test.ts index 61b7aa9c462..34c3e81b09f 100644 --- a/internal-packages/observability-map/src/report/terminal.test.ts +++ b/internal-packages/observability-map/src/report/terminal.test.ts @@ -194,11 +194,8 @@ describe("rendering honestly when there is nothing to say", () => { expect(out).not.toContain("No audit helper exists"); }); - // Round E item 2. The other half of the branch, which the test above did not pin. A zero used to - // print "No audit helper exists in the webapp", which is false: `models/admin.server.ts` writes - // `prisma.impersonationAuditLog.create(...)` and `AUDIT_SYMBOLS` names the helpers that reach it. - // The full tree reads 3 of 49 so nobody sees the sentence today, and a scan of any subset with no - // impersonation route in it brings the sentence straight back. + // The other half of the branch. A zero used to print "No audit helper exists in the webapp", which + // is false, and a scan of any subset with no impersonation route in it brings it straight back. it("does not claim the webapp has no audit helper when nothing reached one", () => { const unaudited = scanFile( "api.v1.auth.tokens.ts", diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 458fab79ac4..6688fdb26d8 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -494,8 +494,7 @@ describe("scanFile: parse failures", () => { ).toThrow(ParseFailureError); }); - // B8. The detection used to read `sf.parseDiagnostics`, an internal property. These are the - // shapes that prove the public route through `ts.Program` still sees a malformed file. + // The shapes that prove the public route through `ts.Program` still sees a malformed file. it("throws on an unclosed jsx element in a tsx route", () => { expect(() => scanFile( @@ -528,9 +527,8 @@ describe("scanFile: parse failures", () => { } }); - // The change from `sf.parseDiagnostics` to a program-backed lookup is invisible to every test - // above: both spellings find the same malformed files today. What a compiler upgrade can break - // is the private one, and only a source-level guard can fail for that. + // Both spellings find the same malformed files today, so the tests above cannot tell them apart. + // What a compiler upgrade can break is the private one, and only a source-level guard fails for it. it("reads its diagnostics through public typescript api rather than a private field", () => { const source = readFileSync(resolve(__dirname, "./scan.ts"), "utf8"); expect(source).not.toContain("parseDiagnostics"); @@ -658,9 +656,8 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); - // A4. `rethrows` used to be set by any `ThrowStatement` in the clause, reachable or not, so a - // `throw e;` appended after a `return` flipped a swallowing catch from rethrows: false to true - // with no behavioural change, which read as inert instead of a swallow. + // `rethrows` was once set by any `ThrowStatement` at all, so a `throw e;` appended after a `return` + // flipped a swallowing catch to inert with no behavioural change. it("does not set rethrows for a throw that is dead code after a return", () => { const ep = scanFile( "dead-throw.ts", @@ -679,11 +676,8 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); - // C2. `reachableStatements` only handled dead code from statement ordering (A4), not a - // statically-false condition, and `catchClauseEvidence`'s walk descended into every function - // body unconditionally, so a throw or an error test merely REGISTERED in a callback the clause - // constructs (never executed as part of the clause's own synchronous handling) was credited to - // it. All four shapes below must leave a plain swallow (`catch (e) { return null; }`) inert. + // Dead code from a statically-false condition, and a throw merely REGISTERED in a callback the + // clause constructs. All four must leave a plain swallow inert. describe("dead and deferred code inside a catch does not count as evidence", () => { const swallow = (mutation: string) => ` export async function loader() { @@ -701,13 +695,9 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - // Eleven shapes that put a `throw` somewhere it can never run, all of them found by review - // rather than by this suite. An earlier round recognised the first two by folding the literal - // `false` and lost to the other nine. None of them is named in the rule now: a throw counts - // when it is unconditional, and every one of these is guarded by something. There is no claim - // that the list is complete, and a twelfth family arrived the round after it was written, see - // `dead throw written after something that already exited`. `dead-*` in the mutation corpus - // runs the same list over the whole route tree. + // Eleven shapes that put a `throw` somewhere it can never run, none of them named in the rule: a + // throw counts when it is unconditional. No claim the list is complete, and a twelfth family + // arrived the round after, see `dead throw written after something that already exited`. const DEAD_SHAPES: Array<[string, string]> = [ ["if (false)", "if (false) { throw e; }"], ["while (false)", "while (false) { throw e; }"], @@ -742,31 +732,25 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - // Extra input beyond the brief's four, exercising the same "merely registered" mechanism with - // a different callback-taking call, to check the fix is not scoped to `.push` specifically. + // The same "merely registered" mechanism under a different callback-taking call, so the fix is + // not scoped to `.push`. it("does not set rethrows for a throw registered in a setTimeout callback", () => { const ep = scanFile("x.ts", swallow("setTimeout(() => { throw e; }, 0);")); expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); }); - // Positive control: a do/while runs its body at least once regardless of the trailing - // condition, so a throw in one is genuinely unconditional and must still register. This checks - // the while(false) fix was not implemented broadly enough to swallow a real rethrow too. + // Positive control: a do/while runs its body at least once, so a throw in one is genuinely + // unconditional and the while(false) fix must not have swallowed it too. it("still sets rethrows for a throw in a do/while, which runs its body once regardless", () => { const ep = scanFile("x.ts", swallow("do { throw e; } while (false);")); expect(ep!.catches[0]!.rethrows).toBe(true); }); }); - // The mirror of the family above. Each dead spelling earns nothing, and it must also COST - // nothing: a plain containment read was true of the dead statement itself, so prepending one raised the - // `exited` flag and blinded the walk to the real classification below it, turning a pass into a - // swallow verdict on 78 real routes. `containsLiveExit` folds the literal guard and sees no live - // exit, so the deciding statements keep their credit. The spellings are the CORPUS spellings - // from `dead-*` in `mutations.ts`, not the `DEAD_SHAPES` table's: that table's inner-try twin is - // `try { doThing(); } catch { throw e; }`, which is NOT provably dead (`doThing` may throw and - // the rethrow then runs), so conservatively raising the flag after it is correct and it gets no - // twin here. + // The mirror of the family above: each dead spelling earns nothing and must also COST nothing, + // since a containment read of the dead statement raised `exited` and blinded the walk to the real + // classification below it, on 78 real routes. The spellings are the CORPUS ones from `dead-*`, not + // the `DEAD_SHAPES` table's, whose inner-try twin is not provably dead and gets no twin here. describe("dead and deferred code prepended to a deciding catch does not blind it", () => { const deciding = (mutation: string) => ` export async function loader() { @@ -806,17 +790,15 @@ describe("scanFile: catch clause evidence", () => { }); } - // The composed shape: the dead if wrapped in a bare block, which the walk enters. The block's - // own live-exit read has to fold too, or entering it re-raises the flag the fold lowered. + // The dead if wrapped in a bare block, which the walk enters: the block's own live-exit read has + // to fold too, or entering it re-raises the flag the fold lowered. it("keeps branches true past a dead throw in a block around an if (false)", () => { const ep = scanFile("x.ts", deciding("{ if (false) { throw e; } }")); expect(ep!.catches[0]!.branches).toBe(true); }); - // The returns half. A dead `return null;` must not veto the rethrow: `containsReturn` saw the - // return token inside `if (false)` and turned a rethrow-only clause into a swallow verdict, - // which regressed 11 real routes from not-applicable to fail. `dead-if-false-return` in the - // mutation corpus is the tree-scale version. + // The returns half: a dead `return null;` must not veto the rethrow, which regressed 11 real + // routes from not-applicable to fail. `dead-if-false-return` is the tree-scale version. it("still sets rethrows past a dead return in an if (false) arm", () => { const ep = scanFile( "x.ts", @@ -831,8 +813,8 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.rethrows).toBe(true); }); - // Negative controls: the fold only withholds blindness, it must never withhold refusal. - // An always-true guard really can run its throw, so the error test after it stays dead. + // Negative controls: the fold withholds blindness and never refusal, so an always-true guard's + // throw still kills the error test after it. it("still refuses an error test after an always-true spelling that throws", () => { const ep = scanFile( "x.ts", @@ -847,9 +829,8 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(false); }); - // The fall-through slice: `case 1` matches and runs on into `case 2`, so the return is live - // and vetoes the rethrow. Misreading the slice as dead would blind the returns veto, which is - // the direction that hands out credit. + // `case 1` matches and runs on into `case 2`, so the return is live and vetoes the rethrow. + // Misreading the slice as dead blinds the veto, which is the direction that hands out credit. it("reads a switch fall-through onto a live return as live", () => { const ep = scanFile( "x.ts", @@ -865,13 +846,10 @@ describe("scanFile: catch clause evidence", () => { }); }); - // The walk may enter a construct exactly where the entered statements are guaranteed to execute - // whenever the clause body runs. Before these entries existed, relocating a clause's own - // statements inside `if (true)`, a switch default, an if/else or a try/finally put the branch - // evidence out of reach while the returns veto still saw the return, so a deciding clause read - // as a swallow: 83 real routes regressed per corpus entry. Each identity pair here holds the - // wrapped and unwrapped spellings to the same evidence; `checks/index.test.ts` holds them to the - // same verdict. + // The walk enters a construct exactly where the entered statements are guaranteed to execute. + // Without these entries, relocating a clause's own statements inside a wrapper put the branch + // evidence out of reach while the returns veto still saw the return: 83 real routes per corpus + // entry. Each pair holds the wrapped and unwrapped spellings to the same evidence. describe("the walk enters exactly the positions guaranteed to execute", () => { const clauseEvidence = (body: string) => { const ep = scanFile( @@ -960,11 +938,9 @@ describe("scanFile: catch clause evidence", () => { expect(evidence.rethrows).toBe(false); }); - // A finally that leaves itself by `break` or `continue` cancels the try's completion the same - // way a finally return does, so a throw in that tryBlock never escapes the clause. Crediting - // it made `do { try { throw e; } finally { break; } } while (false);` a no-op that minted - // rethrows, and its classifier-hosting variant minted branches on 80 real routes; - // `dead-throw-in-cancelled-try` in the mutation corpus is the tree-scale twin. + // A finally leaving itself by `break` cancels the try's completion the same way a finally return + // does, so a throw in that tryBlock never escapes. Crediting it minted branches on 80 real + // routes; `dead-throw-in-cancelled-try` is the tree-scale twin. it("reads a throw a finally break discards as no rethrow", () => { const evidence = clauseEvidence( "do { try { throw e; } finally { break; } } while (false);\nlogger.error(e);" @@ -1020,17 +996,10 @@ describe("scanFile: catch clause evidence", () => { }); }); - // S1. The other end of the same problem. `reachableStatements` used to cut the statement list - // only on a BARE `return`/`throw`, while the walk descended into blocks and `do` bodies, so a - // `throw e;` written after a nested construct that had already returned was still read as the - // clause rethrowing. Every one of these takes a swallow from `fail` to `not-applicable`, worth 50 - // points a route, and they are semantics-preserving because the throw cannot run. - // - // Two rules answer them together. `definitelyExits` sees through the block, the `do` and the - // `if`/`else`, the `switch` and the `try`/`finally`; the `if (true)` form needs constant folding - // that this file deliberately does not do, and is answered instead by `rethrows` requiring the - // clause to contain no reachable `return` at all. `dead-throw-after-*` in the mutation corpus - // runs all six over the whole route tree. + // The other end of the same problem: cutting the statement list only on a BARE `return`/`throw` + // left a `throw e;` after a nested construct that had already returned reading as a rethrow, worth + // 50 points a route. Two rules answer them together, `definitelyExits` seeing through the wrappers + // and `rethrows` requiring no reachable `return`. `dead-throw-after-*` is the tree-scale family. describe("dead throw written after something that already exited", () => { const exiting = (wrapped: string) => ` export async function loader() { @@ -1059,24 +1028,13 @@ describe("scanFile: catch clause evidence", () => { }); } - // The same wrappers on the branches side, plus three the rethrow list has no use for. An error - // test written after a statement that could already have left the clause is dead code and must - // not read as the clause deciding anything. + // The same wrappers on the branches side, plus three the rethrow list has no use for. This asks a + // weaker question than `definitelyExits` does, and on purpose: "could this have exited", not + // "must it have", which is why a labelled block, a `for...of` and a `while` are all on the list. // - // This asks a weaker question than `definitelyExits` does, and on purpose: "could this have - // exited", not "must it have". That is why `if (true)`, a labelled block, a `for...of` and a - // `while` are all on the list even though none of them is guaranteed to run its body. The flag - // is read through `containsLiveExit`, which folds LITERAL guards only, so every wrapper here - // still counts: `if (true)` descends its then-arm and finds the return, and a loop guarded by - // an identifier keeps the plain containment answer. What the fold withholds is the provably - // dead statement raising the flag itself, which is the twin family in `dead and deferred code - // prepended to a deciding catch does not blind it`. - // - // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the - // END of each statement, after that statement's own branch check. Raising it first makes every - // deciding statement refuse itself, because `if (e instanceof X) return y` contains an exit by - // definition; that variant was measured against the pre-round-C tree and it takes it from 15 to 6, accusing 78 - // routes. This one leaves the real-tree report and all 240 clauses' evidence byte-identical. + // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the END + // of each statement, after that statement's own branch check, or every deciding statement refuses + // itself: measured at 78 routes accused. const BRANCH_EXITED: Array<[string, string]> = [ ...EXITED, ["a labelled block", "outer: { return null; }"], @@ -1100,10 +1058,8 @@ describe("scanFile: catch clause evidence", () => { }); } - // The precision this gives up, pinned so it is a decision and not a surprise: a conditional - // exit before the error test also stops the credit, because the walk cannot tell a guard that - // usually falls through from one that always leaves. No clause in the route tree is this shape, - // which is why the report is byte-identical, but one could be written tomorrow. + // The precision this gives up, pinned so it is a decision and not a surprise: a conditional exit + // before the error test also stops the credit. No clause in the tree is this shape today. it("does not credit an error test written after a conditional return", () => { const ep = scanFile( "x.ts", @@ -1132,8 +1088,7 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches[0]!.branches).toBe(true); }); - // Positive control: nothing before the throw exits, so the throw is real and the clause has no - // other way out. + // Positive control: nothing before the throw exits, so the clause has no other way out. it("still sets rethrows when nothing before the throw returns", () => { const ep = scanFile("x.ts", exiting("logger.error(e);")); expect(ep!.catches[0]!.rethrows).toBe(true); @@ -1153,13 +1108,10 @@ describe("scanFile: catch clause evidence", () => { }); }); - // S3. `definitelyExits` counted a bare `break` and a bare `continue` wherever it found one, and - // both of those target the nearest enclosing construct of their kind rather than the statement - // list the question is about. A `switch` whose clauses all break falls through to the statement - // written after it, so cutting that statement as unreachable accused a route of swallowing an - // error it rethrows, with a detail line saying it "takes one way out regardless of what was - // thrown" about a clause that takes the same way out it arrived by. This is the false-accusation - // direction, so both halves are pinned: what must now stay reachable, and what must still be cut. + // A bare `break` or `continue` targets the nearest enclosing construct of its kind rather than the + // list the question is about, so counting one wherever it appeared accused a route of swallowing an + // error it rethrows. The false-accusation direction, so both halves are pinned: what must stay + // reachable, and what must still be cut. describe("break and continue inside the construct they target", () => { const clause = (body: string) => ` export async function loader() { @@ -1236,11 +1188,9 @@ describe("scanFile: catch clause evidence", () => { }); } - // A `continue` in a switch clause targets the enclosing loop, not the switch, so it is - // inherited through the clause rather than dropped with the `break`. Dropping it would leave - // the throw below reachable, and it is not: the continue goes to the `do`'s condition. - // The labelled jumps beside it leave the `for` entirely, so they are exits wherever they are - // written. The bare `break` is the control that separates the three. + // A `continue` in a switch clause targets the enclosing loop, so it is inherited rather than + // dropped with the `break`. The labelled jumps beside it leave the `for` entirely, and the bare + // `break` is the control that separates the three. const IN_LOOP: Array<[string, boolean]> = [ ["break outer", false], ["continue outer", false], @@ -1268,11 +1218,9 @@ describe("scanFile: catch clause evidence", () => { } }); - // S2. A clause whose try block cannot throw is unreachable, so it is not error handling and - // nothing should be read off it. Crediting one was the largest hole ever found here: prepending - // this to a body takes the real tree from 19 to 44 and raises 224 routes, because the routes - // that catch nothing sat at `not-applicable` and a dead clause moved every one of them to `pass`. - // `dead-classifying-try` in the mutation corpus is the tree-scale version. + // A clause whose try block cannot throw is not error handling. Crediting one was the largest hole + // ever found here: 19 to 44 on the real tree and 224 routes raised. `dead-classifying-try` is the + // tree-scale version. describe("a catch over a try block that cannot throw", () => { const guarding = (guarded: string) => ` export async function loader({ request }) { @@ -1299,8 +1247,8 @@ describe("scanFile: catch clause evidence", () => { }); } - // Positive controls, one per reason `canRaise` recognises, so the predicate is not passing the - // cases above by being false for everything. + // One per reason `canRaise` recognises, so the predicate is not passing the cases above by being + // false for everything. const LIVE: Array<[string, string]> = [ ["a call", "doThing();"], ["a construction", "new Thing();"], @@ -1434,9 +1382,8 @@ describe("scanFile: catch clause evidence", () => { ` ); expect(ep!.hasTryCatch).toBe(true); - // The `throw e` here is guarded by an `if`, so it is not on the clause's straight-line path and - // does not read as a rethrow. The `if` itself does: it reads the binding and one arm throws, so - // the clause decides. The verdict the checks care about is unchanged. + // The `throw e` is guarded by an `if`, so it is not on the straight-line path and does not read as + // a rethrow. The `if` itself decides, so the verdict the checks care about is unchanged. expect(ep!.catches[0]!.rethrows).toBe(false); expect(ep!.catches[0]!.branches).toBe(true); }); @@ -1486,13 +1433,9 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches).toEqual([]); }); - // A7 / C1. The first fix stopped at ANY function-like node, purely lexical, which correctly - // excludes a per-item `.map()` boundary but also deletes the route's own catch when the whole - // body is wrapped in a single-shot callback: `trace(async () => { ...whole body... })`, - // `mutateWithFallback({ pgMutation: async (t) => {...} })`, `new ReadableStream({ start: async (c) - // => {...} })`. All three invoke their callback exactly once, as the route's own continuation. - // The real distinction is per-item iteration versus everything else, so only a callback passed to - // `map`/`forEach`/`filter`/`reduce`/`reduceRight`/`flatMap`/`some`/`every` is a boundary now. + // Stopping at ANY function-like node excluded a per-item `.map()` boundary correctly and also + // deleted the route's own catch whenever the body was wrapped in a single-shot callback. The real + // distinction is per-item iteration versus everything else. describe("inline single-shot wrappers are attributed to the route", () => { it("attributes a catch wrapped in trace(async () => {...})", () => { const ep = scanFile( @@ -1620,9 +1563,8 @@ describe("scanFile: catch clause evidence", () => { expect(ep!.catches).toEqual([]); }); - // A refused catch keeps its evidence, built by the same machinery as an own catch, so - // `error-classification` can judge what it does rather than where it sits. Both flavours are - // pinned: the deciding per-item catch and the inert one. + // A refused catch keeps its evidence, so `error-classification` can judge what it does rather + // than where it sits. Both flavours are pinned, the deciding per-item catch and the inert one. it("populates evidence for a refused per-item catch that decides", () => { const ep = scanFile( "x.ts", @@ -1884,9 +1826,8 @@ describe("scanFile: per-catch evidence", () => { ` ); expect(ep!.catches).toHaveLength(2); - // `if (e instanceof Response) throw e;` branches (it reads the binding and one arm throws) and - // does not rethrow (the throw is guarded, so it is not on the clause's own path). The action's - // catch does neither. What this test is for is that the two clauses stay separate. + // The loader's clause branches and does not rethrow; the action's does neither. What this test is + // for is that the two clauses stay separate. expect(ep!.catches.filter((c) => !c.rethrows && c.branches)).toHaveLength(1); expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); }); @@ -2162,9 +2103,8 @@ describe("scanFile: branches requires the if/switch condition to examine the err }); }); -// A3. `referencesBinding` matched any identifier with the binding's text, including a property -// name, an object literal key and a name re-declared in a nested scope. So a clause that never -// really inspects the error still counted as deciding on it. +// `referencesBinding` once matched any identifier with the binding's text, including a property name, +// an object literal key and a name re-declared in a nested scope. describe("scanFile: branches requires a genuine read of the binding, not a lookalike", () => { it("does not set branches for an `if` that only reads a same-named property", () => { const ep = scanFile( @@ -2261,12 +2201,9 @@ describe("scanFile: branches requires a genuine read of the binding, not a looka }); }); -// I5. The shadow check only fired inside `referencesBinding`'s own top-down search from an -// if/switch's condition, so it only caught shadowing NESTED inside that condition, exactly what -// the tests above exercise. A shadowing scope that instead WRAPS the if (a for-of loop, a nested -// catch with the same name) was invisible, because nothing walked up from the if to notice it. -// catchClauseEvidence now tracks shadowing as it descends, the same way `inCallback` tracks a -// callback boundary: once a scope re-declares the binding, everything nested inside stays shadowed. +// The shadow check once only caught shadowing NESTED inside the condition, so a scope that WRAPS the +// if was invisible. `catchClauseEvidence` tracks shadowing as it descends instead: once a scope +// re-declares the binding, everything nested inside stays shadowed. describe("scanFile: a binding shadowed by an enclosing scope, not just a nested one", () => { const swallow = (mutation: string) => ` export async function loader() { @@ -2348,8 +2285,7 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested }); // Two shapes that read the real binding and are still not credited, for reasons that are not - // shadowing. Both are precision the straight-line rule gives up on purpose, and both are - // recorded here so a later reader can tell a deliberate limit from a bug. + // shadowing: precision the straight-line rule gives up on purpose. it("does not credit an if whose arm does not take the error anywhere", () => { const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); expect(ep!.catches[0]!.branches).toBe(false); @@ -2364,11 +2300,9 @@ describe("scanFile: a binding shadowed by an enclosing scope, not just a nested }); }); -// S3. The ternary path checked only that the condition tested the error, never that the two arms -// went anywhere different, while the `if`/`switch` path had checked exactly that since the round -// before. Rewriting `return X;` as `return e instanceof Error ? (X) : (X)` was therefore worth 50 -// points a route for a change that decides nothing, and it is semantics-preserving. -// `same-arms-ternary` in the mutation corpus is the tree-scale version. +// The ternary path once checked only that the condition tested the error, so rewriting `return X;` as +// `return e instanceof Error ? (X) : (X)` was worth 50 points a route for a change that decides +// nothing. `same-arms-ternary` is the tree-scale version. describe("a ternary on the error has to send its arms somewhere different", () => { const returning = (value: string) => ` export async function loader() { @@ -2403,12 +2337,9 @@ describe("a ternary on the error has to send its arms somewhere different", () = expect(ep!.catches[0]!.branches).toBe(true); }); - // S6. The same three cases on the throw path, which read none of them. The throw arm of the - // shared branch check was unreachable, because the walk sets `rethrows` and cuts the path first, - // so a thrown ternary was never offered to `selectsAnErrorPath` and every one of these clauses - // read as inert. Reading it in the throw arm, before the path is cut, uses the same predicate, - // so the arm test arrives with it. `wrap-body-in-same-arms-throw-ternary` is the tree-scale - // version of the refusal. + // The same three cases on the throw path, which read none of them: the walk sets `rethrows` and + // cuts the path first, so a thrown ternary was never offered to `selectsAnErrorPath`. + // `wrap-body-in-same-arms-throw-ternary` is the tree-scale version of the refusal. const throwing = (value: string) => ` export async function loader() { try { @@ -2470,14 +2401,9 @@ describe("a ternary on the error has to send its arms somewhere different", () = }); }); -// The liveness gap in the branch predicate. `selectsADistinctPath` asked a plain containment -// question, so an arm holding an exit that can never run read as an arm that takes the error -// somewhere. `catch (e) { if (e instanceof Error) { if (false) { return null; } } return json(x, -// { status: 500 }); }` is the same swallow as the clause without the `if`, and it was worth 50 -// points a route. The same eleven dead spellings had already been folded out of -// `catchClauseEvidence`'s `exited` flag by `containsLiveExit`, and this predicate beside it kept -// the containment read. `dead-armed-instanceof-if` in the mutation corpus is the tree-scale -// version: global 19 -> 27 and 80 routes raised, before the fix. +// The liveness gap in the branch predicate: under a plain containment read, an arm holding an exit +// that can never run read as an arm that takes the error somewhere, worth 50 points a route. +// `dead-armed-instanceof-if` is the tree-scale version, global 19 to 27 and 80 routes raised. describe("an arm whose only exit is dead decides nothing", () => { const swallow = (mutation: string) => ` export async function loader() { @@ -2517,20 +2443,11 @@ describe("an arm whose only exit is dead decides nothing", () => { }); } - // The positive controls. The fold is subtractive against containment, so anything it cannot - // prove dead reads exactly as it did, including a guard whose truth is not decidable from the - // token alone. Without these the fix could be "always false" and the four cases above would - // still pass. - // - // `an arm guarded by a condition that does not fold` is also the pin on the alternative that was - // measured and rejected: asking the arm to `definitelyExits` rather than to hold a live exit. - // That is the "guaranteed" reading, it refuses all four shapes above, and it accuses - // `admin.api.v1.orgs.$organizationId.environments.staging.ts` on the real tree, taking the global - // from 19 to 18. That clause recognises Prisma's P2002, re-reads the conflicting row and returns - // `{ status: "updated" }`, rethrowing everything else: a textbook classification whose arm - // happens to fall through to the rethrow when the re-read finds nothing. Accusing it of taking - // one way out regardless of what was thrown is simply false, and a new false accusation is the - // direction that gets the tool switched off. + // Positive controls: without these the fix could be "always false" and the four cases above would + // still pass. `an arm guarded by a condition that does not fold` is also the pin on the rejected + // alternative, asking the arm to `definitelyExits` rather than to hold a live exit, which refuses + // all four shapes above and falsely accuses a real classifying clause in + // `admin.api.v1.orgs.$organizationId.environments.staging.ts`, taking the global from 19 to 18. const LIVE_ARMS: Array<[string, string]> = [ ["a plain returning arm", "if (error instanceof Error) { return badRequest(); }"], [ @@ -2552,11 +2469,9 @@ describe("an arm whose only exit is dead decides nothing", () => { } }); -// C4a. `export const { action, loader } = createActionApiRoute(...)` produced no entry point at -// all: `scanFile` skipped a non-identifier binding name at the export site, so the route was -// absent from the denominator rather than parsed, failed or unmeasured. The two-step spelling -// already worked, because `collectLocalDeclarations` reads the binding pattern and the export -// clause resolves through it, so exactly half the shape was wired. +// `export const { action, loader } = createActionApiRoute(...)` produced no entry point at all, so +// the route was absent from the denominator rather than parsed, failed or unmeasured. The two-step +// spelling already worked, so exactly half the shape was wired. describe("scanFile: a destructured export declaration", () => { const BUILDER = `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";`; diff --git a/internal-packages/observability-map/src/score.test.ts b/internal-packages/observability-map/src/score.test.ts index a8d6943d0e0..0691f103a4a 100644 --- a/internal-packages/observability-map/src/score.test.ts +++ b/internal-packages/observability-map/src/score.test.ts @@ -321,20 +321,14 @@ ${BUSY_AND_FAILING}` expect(after.measured).toBe(2); }); - // M9. A prior version of this test asserted suppressed.score <= plain.score for a check that was - // passing, which the pre-existing per-entry Math.min cap already guarantees on its own: removing - // a passing (maximal) result from a ratio can only lower or hold it, at every level, with or - // without A1's fix, so the assertion passed unchanged against the pre-fix code and proved - // nothing about the aggregate mechanism A1 actually changed. Deleted rather than kept as - // decoration; "does not raise the score" for a failing suppression is exercised, with real - // discriminating power, by the two tests above. + // A test suppressing a PASSING check used to sit here and was deleted rather than kept as + // decoration: the per-entry cap guarantees it on its own, so it passed against the pre-fix code. }); /** - * The invariant, both ways round. The README states one direction, removing error handling must - * lower the score, and that alone could not see the free-points path: adding a catch that only - * rethrows changes nothing about how the route behaves, and used to move it from not-applicable to - * pass, worth 50 points a route and 27 points across the tree. + * The invariant both ways round. Removing error handling must lower the score, and that direction + * alone could not see the free-points path: adding a catch that only rethrows used to move a route + * from not-applicable to pass, worth 27 points across the tree. */ describe("no-op error handling must not pay", () => { const BODY = `const rows = await prisma.thing.findMany(); @@ -563,16 +557,10 @@ export async function action() { }); /** - * `contextGap` and `auditGap` are the same arithmetic `checkContributions` already does for every - * check, written out again by hand for two named ids: `map(find).filter(status)` for the context - * figure, `filter(some)` for the audit one, and a third spelling of "passed" for each. Three - * implementations of "applicable, and how many of those passed", and nothing said they had to - * agree, on the two figures the report puts in front of a reader as headline numbers. - * - * Pinned rather than shared. Collapsing them would mean the gap figures reading their check's row - * out of `checkContributions`, which is a fine refactor and a wider blast radius than the property - * is worth: what matters is that they cannot disagree, and an assertion says that without moving - * any code the renderers read. + * `contextGap` and `auditGap` are the arithmetic `checkContributions` already does, written out again + * by hand for two named ids, on the two figures the report puts in front of a reader as headlines. + * Pinned rather than shared: what matters is that three spellings of "applicable, and how many + * passed" cannot disagree, and an assertion says that without moving code the renderers read. */ describe("the hand-rolled gap figures agree with the per-check contributions", () => { const SOURCE = `import { prisma } from "~/db.server"; diff --git a/internal-packages/observability-map/src/sensitivity.test.ts b/internal-packages/observability-map/src/sensitivity.test.ts index bb89f427de1..c54d4d95730 100644 --- a/internal-packages/observability-map/src/sensitivity.test.ts +++ b/internal-packages/observability-map/src/sensitivity.test.ts @@ -173,11 +173,9 @@ describe("what sensitivity must not mean", () => { }); }); -// C2. The classifier covered tokens, billing, impersonation and envvars and missed the entire -// surface where authorization bugs live, so `auth-boundary` and `audit-trail` never looked at -// membership, the login surface, API keys or the two billing settings the bare `billing` segment -// does not match. The vocabulary below is read off `apps/webapp/app/routes`; -// `test/webappSymbols.test.ts` is what holds it to that. +// The classifier once covered tokens, billing, impersonation and envvars and missed the entire +// surface where authorization bugs live. The vocabulary is read off `apps/webapp/app/routes`, and +// `webappSymbols.test.ts` holds it to that. describe("classifySensitivity: the access-control surface", () => { const flags = (fileName: string) => classifySensitivity(ep(fileName, `export async function loader() { return 1; }`)).sensitive; diff --git a/internal-packages/observability-map/src/suppression.test.ts b/internal-packages/observability-map/src/suppression.test.ts index 62a29facbb4..2ebe8150b68 100644 --- a/internal-packages/observability-map/src/suppression.test.ts +++ b/internal-packages/observability-map/src/suppression.test.ts @@ -118,12 +118,8 @@ describe("suppressedChecks", () => { expect(m.size).toBe(0); }); - // I3. A standalone `ts.createScanner` has no parser state, so it still granted two suppressions - // nobody wrote: it never rescans a template as a continuation after a `${...}` substitution, so - // text after the `}` reads as ordinary code and a `//` in it is a real comment to the scanner; - // and a scanner created in `LanguageVariant.Standard` has no JSX context, so a `//` inside JSX - // text reads as a line comment mid-URL. Reading comments off the actual parsed tree closes both: - // a template's literal segments and a JSX text node are real nodes, never trivia. + // A standalone `ts.createScanner` has no parser state, so it granted two suppressions nobody wrote: + // it never rescans a template as a continuation after a substitution, and it has no JSX context. it("does not suppress from a directive after a template substitution", () => { const m = suppressedChecks( "const msg = `${name} // obs-map-disable error-classification -- via substitution`;\n" + @@ -142,11 +138,9 @@ describe("suppressedChecks", () => { expect(m.size).toBe(0); }); - // S4. The case above passes without any JSX handling at all, because the comment-range lexers - // only find a comment at the exact offset they are asked about and the `//` there is mid-text. - // These are the shapes that actually needed the fix: JSX text that BEGINS with a comment marker, - // which is what the lexers see when they are pointed at the start of a JsxText node. Removing - // `ts.isJsxText` from `isClaimedContent` makes all four fail and nothing else in the suite. + // The case above passes without any JSX handling, because the lexers only find a comment at the + // exact offset they are asked about. These four are the shapes that needed the fix, JSX text that + // BEGINS with a marker, and removing `ts.isJsxText` fails all four and nothing else in the suite. describe("jsx text is content, not a comment", () => { const page = (body: string) => `const name = "x"; export default function Page() { diff --git a/internal-packages/observability-map/src/webappSymbols.test.ts b/internal-packages/observability-map/src/webappSymbols.test.ts index 3a709d2d913..7cc7ed72dd0 100644 --- a/internal-packages/observability-map/src/webappSymbols.test.ts +++ b/internal-packages/observability-map/src/webappSymbols.test.ts @@ -12,39 +12,17 @@ import { } from "./sensitivity.js"; /** - * Every name and every path segment the tool matches on must exist in the codebase it is pointed - * at. + * Every name and every path segment the tool matches on must exist in the codebase it is pointed at. + * Half of `SENSITIVE_SYMBOLS` named nothing before this test, and the guard list has the same failure + * mode with a worse consequence: a guard name resolving nowhere makes a route that can never pass. * - * This is the test the last round did not have, and the cost of not having it was measured: half of - * `SENSITIVE_SYMBOLS` named nothing. `Set.has` is exact, so `setImpersonation`, `createJWT`, - * `signJWT` and `updateEnvVars` matched no route in the tree, while `startImpersonation`, the real - * escalation, was absent from the list. Nothing failed, nothing was reported, and the symbol half - * of the classifier was quietly doing almost nothing. `auth-boundary`'s guard list has the same - * failure mode with a worse consequence, since a guard name that resolves to nothing turns into a - * route that can never pass rather than a route that can never fail. + * Checked: every guard name and sensitive symbol is DECLARED under one of `ROOTS`, as a function, + * class, interface, type, enum, variable or member name; members count because several guards are + * reached through an object and `calleeName` records the property. And every sensitive path segment + * appears as a segment of a real route file name. * - * What is checked: - * - * - every guard name and every sensitive symbol is DECLARED somewhere under one of `ROOTS`. A - * declaration is a function, class, interface, type, enum or variable name, or a member name on a - * class, interface or object literal. Members count because several guards are reached through an - * object: `rbac.authenticateSession`, `authenticator.isAuthenticated`, and `calleeName` in - * `scan.ts` records the property for a member call, so that is the form the check sees. - * - every sensitive path segment appears as a segment of a real route file name. - * - * What is NOT checked, and each is a place a wrong entry can still hide: - * - * - that the declaration found is the one meant. `authenticateAdmin` is a local helper inside - * `admin.api.v1.platform-notifications.ts`; a second route declaring its own no-op function of - * that name would be credited by `auth-boundary`. Names cannot carry that guarantee, and the - * alternative, a module-resolving import graph, is a different kind of analysis from anything - * else in this package. - * - that a guard actually guards. `resolveAuthenticatedEnv` declares fine and authenticates - * nothing, which is why it is not on the list; keeping it off is a hand-read judgement this test - * cannot make. - * - anything outside `ROOTS`. A guard declared only by a dependency is listed in - * `EXTERNAL_GUARDS` and not resolved at all; see the comment there for why that is a list - * rather than a path into `node_modules`. + * Not checked, each a place a wrong entry can hide: that the declaration found is the one meant, that + * a guard actually guards, and anything outside `ROOTS`. */ const REPO = resolve(__dirname, "../../.."); @@ -62,19 +40,10 @@ const ROOTS = [ ]; /** - * Guard names declared by a dependency rather than by us, and therefore deliberately unchecked. - * - * Both are methods on remix-auth's `Authenticator`, reached as `authenticator.authenticate(...)` - * and `authenticator.isAuthenticated(...)`; the whole login surface is built on them. An earlier - * version of this test resolved them by reading - * `apps/webapp/node_modules/remix-auth/build/authenticator.d.ts` directly. That is a path into an - * installed tree: a hoisting change, a version bump that moves `build/`, or a fresh clone with a - * different install layout turns a real assertion into a confusing environmental failure, and a - * test that fails for environmental reasons teaches people to ignore it. - * - * So they are listed instead, which is a smaller claim honestly made. The test still fails if a - * guard name is neither declared in first-party source nor on this list, so a name that resolves - * nowhere cannot be added silently; what it no longer does is prove these two exist. + * Guard names declared by a dependency rather than by us, and therefore deliberately unchecked. Both + * are remix-auth's, and resolving them meant reading a path inside `apps/webapp/node_modules`, which + * an install-layout change turns into a confusing environmental failure. Listing them is a smaller + * claim honestly made: the test still fails on a name that is neither first-party nor listed. */ const EXTERNAL_GUARDS = new Set(["authenticate", "isAuthenticated"]); From be72dd5d9239444b595a9e21f6ca8e091e7ac6c9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 15:22:46 +0100 Subject: [PATCH 111/117] docs(observability-map): split the README and cut its history The README carried four jobs at once after the inline commentary moved into it: what the tool measures, how to read the report, how the scanner decides, and a running account of what earlier rounds got wrong. Split it. README.md keeps the reader-facing job (running it, CI, what the score means, the five checks, headline figures, not-applicable, suppression, known limits, layout); INTERNALS.md takes the scanner reasoning, the mutation harness, the reporting arithmetic and the CI wiring, for someone changing the tool. Cut the history throughout. What a fix changed and why the old shape was broken is in the commits and the ledger; a rejected alternative is kept only where someone would otherwise re-propose it, and then in one line. Every inline 'README,
' pointer is repointed and every one resolves to a heading that exists. 964 lines becomes 336 + 494. The real-tree report is byte-identical. --- .../observability-map/INTERNALS.md | 494 +++++++++ internal-packages/observability-map/README.md | 988 ++++-------------- .../src/checks/authBoundary.ts | 5 +- .../observability-map/src/checks/authScope.ts | 4 +- .../src/checks/errorClassification.ts | 4 +- .../src/checks/requestContext.ts | 2 +- .../src/docstringReferences.test.ts | 2 +- .../observability-map/src/integration.test.ts | 11 +- .../src/mutationCorpus.test.ts | 7 +- .../observability-map/src/mutations.ts | 4 +- .../observability-map/src/report/prComment.ts | 2 +- .../observability-map/src/scan.ts | 20 +- .../observability-map/src/sensitivity.ts | 3 +- .../observability-map/src/suppression.ts | 5 +- .../observability-map/src/triviality.ts | 2 +- .../observability-map/src/types.ts | 4 +- 16 files changed, 715 insertions(+), 842 deletions(-) create mode 100644 internal-packages/observability-map/INTERNALS.md diff --git a/internal-packages/observability-map/INTERNALS.md b/internal-packages/observability-map/INTERNALS.md new file mode 100644 index 00000000000..270726522d6 --- /dev/null +++ b/internal-packages/observability-map/INTERNALS.md @@ -0,0 +1,494 @@ +# Internals + +How the scanner decides what it decides. Read [README.md](./README.md) first for what the tool +measures and how to run it; this file is for changing it. + +Every rule here refuses a shape that would otherwise mint free points, and every rejected +alternative written down is one somebody has proposed. + +## How the scanner reads a route + +`scan.ts` produces one `EntryPoint` per route module, carrying only body-scoped evidence. Three +rules decide what "the body" means, and every finding rests on them. + +**One hop, same file only.** A loader that delegates to a helper declared in the same file has that +helper's statements, try/catch and callees counted as its own. A helper's own helpers are not +followed, the visited set stops a cycle, and nothing imported from another module is ever opened. + +**Nested functions count as work.** A statement inside a callback written in the body is still a +statement the route runs. Leaving them out lets `trace("x", async () => { whole body })` collapse a +route to one statement, inside the triviality limit, so every check reports not-applicable for it +(`wrap-body-in-trace`). + +**Per export, not per file.** Six fields come in `loaderX`/`actionX` pairs, and the union is only +offered where the question itself is file-wide. The split refuses a family of false passes: a file +whose loader calls `requireUser` and whose action calls nothing reading as "guarded in the body", or +a file whose loader is `createLoaderApiRoute(...)` crediting its hand-written action with the +builder's authentication. `routeExports.ts` is the one enumeration both per-export checks read, +because `auth-scope` and `auth-boundary` each grew their own `[loader, action]` literal and only one +of them got each fix. + +`calleeNames` is the union and stays entry-point wide because the three questions that read it are +file-wide: what the file touches, how much it does, whether it records anything. There is +deliberately no entry-point-wide `checkedCallees`, so no check can reach for a union that would say +a loader's reading of `getUser` speaks for the action beside it. One push site in `scanFile` fills +the whole-entry list and each owning export's list, so the two cannot drift (`every callee name is +attributed to an export that exists`, pinned on fixtures and again over the real tree). + +Two fields exist because the bare callee name is not enough. `calleeName` keeps only the last +segment, so `prisma.organization.findFirst` arrives as `findFirst` with the receiver gone; +`calleeTexts` keeps the whole dotted path, which is how the per-export triviality rule knows a +three-statement body reaches the datastore. `auth-boundary` matches the bare name on purpose, so a +guard call cannot be hidden by its receiver. + +### Catch evidence, per clause + +`CatchEvidence` is one record per catch clause rather than a set of booleans per entry point, +because 39 routes have more than one catch and 17 mix a narrow parse guard with a broad handler, and +an aggregate lets the well-behaved clause speak for the swallow beside it. + +- `rethrows`: throwing is the clause's only way out, i.e. a throw is reached on the clause's + guaranteed path AND the clause contains no live `return` anywhere. +- `throws`: a throw is reached on that path, whether or not it is the only way out. Kept separately + so a verdict can say what is true of a clause that both throws and returns; the detail line "takes + one way out regardless of what was thrown" is true only of a clause that never throws. +- `branches`: the clause picks what to do from what it caught. An `if` or `switch` whose condition + references the caught binding and at least one of whose arms returns or throws, or a conditional + that is the whole value of a `return`/`throw`. `if (retries > 0)` does not count, + `if (e instanceof Error) { }` does not count, a bindingless `catch { }` cannot count at all, and an + `instanceof` used only to word a message does not count either, because every error still leaves + by the same path. +- `guardsParse`: the guarded region parses something. `JSON.parse`, `request.json()`, a zod + `parse`/`safeParse`, a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three + constructors are read as `ts.isNewExpression` because a `new` expression is not a call and the + call-callee scan never sees them. Crediting any constructor would let `new BranchesPresenter()` + excuse a catch guarding ordinary work, true of 77 try blocks in the tree. +- `guardCanRaise`: the region does anything that could reach the clause. False means `try { 0; }` and + little else, because any call counts, including one that cannot throw. +- `guardMayRaise`: the containment twin, false only when the region provably cannot raise. Everything + `canRaise`'s whitelist misses stays true here, so `guardCanRaise` implies `guardMayRaise`. +- `awaitsOnlyParse`: everything the region waits for is one of those parses, or a read of the body it + parses. +- `tryStatementCount`: statements in the guarded block, counted as `statementCount` counts them. + +`canRaise` is a whitelist and it misses real raising code, which is the safe direction but does +matter: a destructuring declaration (`const { a } = undefined` throws), a temporal-dead-zone read, a +coercion that raises, and a `delete` on a frozen object all read as unable to raise. So the +refused-swallow arm of `error-classification` reads the route's own deciding catches through +`guardMayRaise` and never through `guardCanRaise`, ordering it off can-raise being what accuses a +route that owns a real classifying catch of owning none. + +"Does this route catch anything" is `catches.length`, never `hasTryCatch`. A `try`/`finally` with no +catch leaves `hasTryCatch` true and `catches` empty, and nothing is swallowed there: the error +propagates once the cleanup has run. + +## The dead-code defence + +Both of the catch-clause answers are read off the clause's guaranteed path. The governing rule: the +walk may enter a construct exactly where the entered statements are guaranteed to execute whenever +the clause body runs, so no credit can come from code a semantics-preserving edit could have added +dead. + +Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` that has no catch +clause and whose finally contains no jump out of itself, the sole clause of a single-default +`switch`, the then-arm of an `if` whose condition is exactly the literal `true` keyword, and both +arms of an `if`/`else` with per-arm states merged by intersection. + +Not entered, deliberately: a bare `if` without an else, loops other than `do`, labelled statements, +function-like nodes, nested catch clauses, finally blocks, and the tryBlock of a `try` that has a +catch clause, where a throw is intercepted by the nested catch rather than escaping. + +Do not replace the rule with a list of statically-false shapes to refuse. Asking for the throw to be +unconditional refuses eleven spellings, from `if (false)` and `for (;false;)` through +`switch (1) { case 2: }` and `for (const k in {})` to `if (1 === 2)`, each worth 50 points a route, +without naming any of them. `dead-*` in the corpus is the tree-scale proof, one entry per shape. + +`rethrows` asks for one thing more: no `return` anywhere in the clause, or a `throw error;` written +after a statement that already exited reads as a rethrow, in seven spellings (`dead-throw-after-*`). +The cost is real, since `catch (e) { if (transient) throw e; return null; }` no longer reads as a +rethrow and so fails rather than sitting out. That is the direction to be wrong in, the reverse +handing out points. + +### Two folds, pointing opposite ways + +There are two literal folds in `scan.ts` and unifying them would be a bug. + +`containsLiveWhere` folds any literal guard `literalTruth` can decide, and it is strictly subtractive +against a plain containment read: wherever the truth cannot be decided, every hit containment would +have found is still found. That is what lets its two callers read it for opposite purposes. In +`catchClauseEvidence`'s `exited` flag a hit BLINDS the walk to whatever follows, and containment +blinds it on a provably dead statement, so prepending one to a deciding clause turns its pass into a +swallow verdict on 78 routes. In `selectsADistinctPath` a hit GRANTS a branch, and containment grants +one for an arm whose only exit is dead (`dead-armed-instanceof-if`, 80 routes and the tree from 19 to +27). Subtracting dead hits only ever un-blinds in the first case and only ever withholds in the +second. + +The walk's own entry tickets fold nothing but the literal `true` keyword. `!!1`, `1` and `!false` are +deliberately not entry tickets, because entry GRANTS credit and a wrong grant pays, where +`literalTruth`'s wider folding only ever withholds blindness. Do not unify the two. + +`literalTruth` treats `&&`, `||`, an identifier, a call, a bigint and a template with substitutions +as undecidable on purpose, so a live guard can never be read as dead. The cost is +`dead-conjunction-instanceof-if`, a corpus expected failure: `e instanceof Error && false` both +references the caught binding and can never be true, and no fold in the file can see it. Widening the +fold is a different rule with its own measurement. + +The `exited` flag is raised at the END of each statement, after that statement's own branch check. A +deciding statement contains an exit by definition, so raising it first makes every such statement +refuse itself, measured at 78 routes losing their pass. The ordering leaves the real-tree report and +all 240 clauses' evidence byte-identical. + +### A finally that cancels the try + +A finally block that completes abruptly supersedes the try's and the catch's completion, so an exit +written in either never leaves the statement. Two places read that, in opposite directions. + +`catchClauseEvidence` refuses to enter a catchless try whose finally holds a jump out of itself, +because entry grants rethrow credit and the throw would never escape the clause. The refusal is a +containment read, over-approximate on purpose: a jump that only may run still refuses (`refuses the +tryBlock when the finally only may break`, and `dead-throw-in-cancelled-try` at tree scale, worth 80 +routes and 8 global points). + +`containsLiveWhere` then folds the same statement to its finally's own statements, so a refused +statement cannot blind the walk to the real classification below it (`keeps the classification after +a finally-break no-op`). A finally holding a `return` is covered by the explicit `containsLiveReturn` +read instead, because `try { throw e; } finally { return null; }` genuinely swallows. + +### The residual both branch tests share + +Two arms that produce the same outcome by different spellings still read as a real decision. +`if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides nothing, as +does the `if` with no `else` whose arm returns what the statement after it returns. Telling those +apart needs the produced values compared for meaning rather than for text. The textual comparison is +the cheapest thing that catches the copy-paste form, which is the one a mutation produces. + +## Parse guards, and the narrow-try count + +A catch clause counts as a parse guard, rather than as the route's error handling, when the try block +parses, waits for nothing except that parse, and is short. All three conditions are load bearing. + +`awaitsOnlyParse` is what a statement count cannot express. +`try { const body = await request.json(); return await handleEverything(body); } catch { 500 }` is +two statements, one of them a parse, and the whole handler inside it: the count reads it as narrow +and it is the `otel.v1.logs.ts` swallow written compactly. Asking what the block waits for separates +them, and unlike the count it does not care how the statements are punctuated or how deeply the work +is nested. Awaiting is the signal rather than calling, because the calls that prepare a parse's input +are ordinary synchronous string work (`matchPattern.slice(4)` before a `new RegExp`), and requiring +every CALL to be a parse refuses four of the tree's clearest guards. Two residuals follow: a block +that does its non-parse work synchronously still reads as a guard, and `guardedWork` looks for a +`ts.AwaitExpression`, which `for await (...)` and `await using` are not. + +`NARROW_TRY_STATEMENTS` is 2, so the guarded operation can bind its result +(`const stripped = ...; new RegExp(stripped);`) and a third statement means the try has started to +cover the handler. It is an absolute count and not a ratio against the enclosing body, because a +ratio is diluted by anything else in the same body: padding the action with unrelated statements +after the try relabels the same broad swallow as a narrow guard, moving the denominator without +touching the clause (`inert-statements-after-try`). + +The count is paddable, which is why it is one condition of three rather than the load-bearing one. +`countStatement` counts declarators and comma operands rather than semicolons, so +`const a = f(), b = g(), c = h();` is three and `a(), b(), c()` is three (`merge-declarations`, +`merge-comma-expressions`), and a third way nobody has written down would work. + +Two rejected alternatives, both measured. Requiring the clause to answer with a 4xx credits, on its +own, the 11 widest swallows in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try +answers every failure with a 400 carrying the internal error message; added on top of the rest it +costs three narrow guards their pass for computing a fallback value rather than answering a request. +And a narrow guard is not a way to qualify as classification on its own: the eleven entry points that +limb would clear hold six real swallows, including a silent run cancellation and two credential paths +reporting a database failure to the browser as a 400 with an internal message in it. + +## The iteration-callback boundary + +`items.map((item) => { try {...} })` is a fresh catch per element, so its clause is not the route's +own error handling. `trace(async () => {...})`, `mutateWithFallback({ pgMutation: ... })` and +`new ReadableStream({ start: ... })` all invoke their callback exactly once, so theirs is. The +structural signal is the method name, a list of eight, because nothing in a syntactic scan can tell +`users.map` from `Result.map`. + +Three rules keep the cheap direction from paying. A refused catch is kept WITH its evidence, built by +the same machinery as an own catch, and judged on what it does rather than on where it sits. A +refused swallow fails the route whenever nothing the route owns decides, and that arm is deliberately +not conditioned on the route owning no catches, so an own inert rethrow catch cannot lift a refused +swallow out of the verdict. A route whose only catches are refused and none of them swallows sits out +at not-applicable and never passes, which keeps a prepended dead deciding `.map` from minting a pass +on the 261 catchless routes (`dead-deciding-map`). + +That is what makes the name list survivable. Relocating a swallow behind the boundary still fails +(`still fails a swallow wrapped in a non-array receiver's .map(...)`), and relocating a decision +earns at most the route's exit from the denominator. A receiver that is an array literal of one +element or none is refused outright, since it cannot iterate. + +The other direction costs precision. A per-item callback under a callee the list does not know, +`pMap(items, cb)` or `Array.prototype.map.call(items, cb)`, is attributed to the route, so a +per-element catch that decides can carry it to a pass. No mutation of a real route produces it, since +a route has to already be iterating for the shape to exist, which makes it a wrong verdict waiting +for a route rather than a laundering path, and is why the list is worth extending when a new +iteration helper shows up. + +## What auth-scope reads as scoping + +Three conditions, all load bearing. With only the middle one, prepending +`const __unused = { anything: user.id };` to every body raises `settings.sso` and `settings.team`, +the only two findings `auth-scope` has ever produced and both confirmed cross-org exposures +(`dead-caller-scope-object`, `dead-caller-scope-userid`). + +- The value has to be the caller's own id, anchored at both ends: the root is one of the auth + bindings a builder hands the handler and the last segment is an identity field, so `user.name` is + not a scope and neither is `run.userId`, which is a resource's owner. +- The property NAME has to be an identity field. Of the ten names that take a caller-id value in the + route tree, `sub`, `value` and `consumerId` are the three that are not, and `anything: user.id` is + what a mutation writes. +- The object has to be handed, through any depth of nesting, to a call that could narrow a read with + it. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. + +The third condition is a denylist of sinks rather than an allowlist of query callees, and that is a +measurement: 72 distinct callees are handed a caller id across the route tree, from +`prisma.project.findFirst` through `presenter.call` to bare `regenerateApiKey`, and no name pattern +separates those from `sendToPlain`. An allowlist would accuse whichever route named its helper next, +which is the failure this check cannot afford. The sinks refused are the log line and the response +body, both of which take the very `{ userId: user.id }` object a query filter takes: loggers account +for 13 of the caller-id sites and the two response serializers for 2 more. The shape is in the tree +already, in `engine.v1.dev.runs...attempts.start`, which logs `{ environmentId: ... }` beside the +`runStore.findRun` that earns its credit honestly (`log-caller-scope-userid`). + +A callee with no readable name of its own is credited, because refusing it would ACCUSE the route and +under-crediting beats accusing a route that is fine. `String({ userId: user.id })` therefore reads as +scoping, the same way `try { String(0); }` reads as error handling and for the same reason. + +`authorization: undefined`, `null` and `false` are read as not declared, because +`apiBuilder.server.ts` gates every option behind `if (option)` and declaring one is what the check +credits. + +An `ability.can(...)` call in the handler is deliberately not a third way to be scoped. +`apps/webapp/CLAUDE.md` says why: the OSS fallback ability is permissive +(`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and +`buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an +ability check enforces the role while the membership-scoped query is the tenant floor. + +## Sensitivity, and the names the tool matches on + +Two rules hold the vocabulary honest, and tests rather than convention enforce both. + +Calling a guard can never be what makes a route sensitive, because a mitigation cannot be the hazard +and the reading is circular: a guard name on the symbol list marks every route that calls it +sensitive, and `auth-boundary` then passes all of them for calling it (`does not treat calling the +admin guard as what makes a route sensitive`). + +Every name and every segment has to exist. `src/webappSymbols.test.ts` resolves every sensitive +symbol, every path segment and every entry in `auth-boundary`'s guard list against `apps/webapp/app` +and the two packages the webapp authenticates through, and fails if one stops resolving, half the +symbol list having named nothing at all without it. The one exception is `ANTICIPATED_SEGMENTS`, +three words that name no route yet and are held to naming none. + +That test is also why `auth-boundary`'s guard list is names rather than the patterns it replaced. +`/^(require|authenticate)/` cleared a sensitive route on any callee beginning `require`, so +`requireSsoEntitlement`, a plan check, cleared the org SSO settings page; `/Authenticated/` passed +`resolveAuthenticatedEnv` on ten routes, a `findFirst` by environment id that authenticates nothing. +Both are corpus entries (`fake-require-guard`, `fake-authenticated-lookup`): under the patterns they +took the tree from 18 to 19 and raised five routes, and under the accept-list they raise nothing. + +## Triviality, in detail + +Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder +wrapping it, and nothing in the calls or the hint text naming a datastore or a service. + +Both limits are 3 because both real shapes need three: parse the params, build a path, redirect, or +an environment guard and two returns. A fourth call admits +`_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls; a fourth statement +admits the routes that authenticate and hand off to a presenter; a fifth admits an admin route that +calls a service and hand-rolls its own error responses. + +The rule is deliberately reluctant, because a route wrongly called trivial is exempted and never +shows up in the report again. So `calleeNames` descends into the callee of every call at any depth +while `statementCount` stops at a nested function, which means the call count still catches bodies +the statement count reads as short. A builder means the config passed to it (`findResource`, +`authorization`) is work the scanner never walks, so the visible body is not the whole route. And a +try/catch is exactly what `error-classification` reads, so a body with one has an error path worth +reporting on however short it is. + +One rule, two views, so the entry-point-wide answer and a single export's answer cannot drift. The +per-export view exists because a file-wide triviality rule accuses the wrong half of a file: +`auth.github.ts` is `export let loader = () => redirect("/login")` beside an action that calls +`authenticator.authenticate`, so a file-wide rule calls it non-trivial for the ACTION and +`auth-boundary` accuses a one-line redirect stub of missing an auth guard. `checks/index.test.ts` +pins both directions (`reports not-applicable for a redirect-stub loader beside a guarded action`, +`fails an export whose own body does real work unguarded`). + +The two views differ in one term, measured both ways. The entry-point-wide view matches the +side-effect hints against the whole file, so an import of `prisma` disqualifies it even when the +query sits somewhere the scanner does not walk. The per-export view matches that export's own callee +PATHS instead, because matching the file's text is defeatable: `log-caller-scope-userid` prepends a +`logger.error(...)` to every body, which with a file-wide term puts the word `logger` in +`auth.github.ts` and turns its untouched redirect loader from excused into accused. Emptying the term +is not the answer either, since `calleeNames` keeps only a call's last segment, so +`prisma.orgMember.findMany` reads as `findMany` and a three-statement body that queries the datastore +matches no hint at all, which takes five `auth-boundary` fixtures from `fail` to `not-applicable`. +The callee paths are body-scoped and name the receiver, which is what both readings needed. + +## Reading the directive out of the source + +The suppression directive is read from a real parsed `ts.SourceFile`, and then filtered against the +spans the parser has already claimed as content. Both halves are needed. + +Parsing rather than scanning is what stops a template literal with a substitution being rescanned as +ordinary code after `${x}`, and what makes JSX text a node at all. Filtering by span is what stops +the two comment-range lexers reading the start of such a node as a comment anyway, which they do +because `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw lexers over source text from +an offset and consult no parse tree. A JSX text node that BEGINS with `//` or `/*` is the shape that +reached the real tree, in `resources.branches.create.tsx`'s `//`. + +The filter is on the range's start offset falling inside a claimed span, not on the gap between a +token's full start and its start, a gap filter losing a same-line trailing comment and a comment +inside a JSX expression container. Both lexers are called at every token boundary, because which one +returns a given comment depends on whether it shares a line with the token before it. Leaf tokens are +walked through `.getChildren()` rather than `ts.forEachChild`, which skips bare punctuation and +keyword tokens, and a comment can sit directly before one of those as the last line inside a block. + +The mutation corpus cannot cover any of this: a suppression can only lower an entry's score, because +`scoreEntry` caps it at the pre-suppression ratio, so suppression bugs are invisible to a harness +watching for the score rising. They need ordinary unit tests. `jsx text is content, not a comment` is +the four cases that fail without the JSX filter, and the positive control beside it, `still reads a +directive from a comment in a JSX expression container`, is what stops the filter being widened until +it eats real comments. + +## The mutation harness + +Every mutation is a TEXT rewrite driven by AST positions, never a reprint. A reprint would change +formatting everywhere and make a failure impossible to read; splicing at node positions leaves the +rest of the file byte-identical, so a corpus failure can be diffed down to the one construct that +moved. Overlapping edits are dropped inner-first, which is what "the outer rewrite won" means. + +Nothing is ever executed. Semantics-preserving means preserving the observable behaviour of the route +as written, which is what the scanner claims to measure, not that the mutated tree compiles against +its real types. + +**Splices go at the HEAD of a catch clause, not the tail.** 234 of the tree's 260 clauses end in a +`return` or a `throw`, so an appended shape is dead by ordering before the rule under test ever looks +at it. At the head every clause is reachable, and the shapes spliced this way are dead wherever they +sit, so moving them does not make the rewrite any less preserving. + +**The harness's population is asserted against the scanner's.** A mutation reaching fewer routes +lowers the score rather than raising it, so no invariant here can notice the harness missing an +export form. `wraps a body in every non-delegating entry point the scanner finds` pins it, with +`admin.tsx` the one named exclusion, its handler being a concise arrow with no block for a block +wrapper to wrap. The same failure mode is why the registry assertion and the additive-class +assertion are ungated while everything else in the file needs `OBS_MAP_MUTATION_CORPUS=1`: omitting a +check from a sweep leaves its failures in place, which lowers the score, so the corpus cannot catch +its own omission by failing. + +**The corpus deliberately disagrees with the scanner about where a handler sits.** `mutations.ts` +keeps its own copy of the builder handler shapes rather than importing them, because sharing the +scanner's notion would let a bug in that notion hide a laundering shape. Which exports exist is not a +judgement, though, which is the distinction above. + +**The anti-vacuity threshold is on sites, not only files,** a file count saying a rewrite touched a +file rather than that it reached anything inside it. Verdict movement cannot be the guard instead: +the IDEAL defended shape is one the scanner is blind to, so `dead-if-false` and the ten entries +beside it are defended precisely because the tree comes out identical, and requiring movement would +fail exactly the entries that work best. + +**A `lowers` exemption is a per-entry field with a reason, not a skip list.** An exempted entry must +still be falling, or the exemption is stale, and its falls must have exactly the measured residual +shape it was granted for: `error-classification` moving pass to not-applicable, every other check +unchanged, nothing moving to fail. Exactly two entries carry one, both non-array-receiver iteration +wrappers. + +**A `KNOWN_GAPS` entry runs as `it.fails`,** so closing the hole later turns the file red until the +entry is moved out deliberately. Two are open, both described above: +`dead-classifying-try-with-call` and `dead-conjunction-instanceof-if`. + +## Reporting + +Every denominator reads `rawChecks`, pre-suppression; `checks` is the display view. Suppressing the +one `request-context` or `audit-trail` finding on an entry must not shrink the gap denominators and +raise the printed percentage, on the same screen as a claim that suppression cannot do that. An +entry's score is capped by what it would have scored unsuppressed. + +`score` is 100 for an entry no scored check applied to, a placeholder rather than a verdict. +Rendering it as a figure turns a route refactored down to a trivial body into a 67-point improvement, +and a trivial route gaining real work into the pull request's worst regression, so the PR comment's +cell says "not measured" instead. `globalWithout` recomputes from `rawChecks` minus the suppression +cap, because lowering both figures by the same rule would leave the difference between them saying +something about suppressions rather than about the check. + +`hasDelta` has to be true whenever `renderPrComment` would say something different, because anything +it misses is a change the pull request silently does not report. It covers the global, the per-entry +score, measured state and suppression set, an entry added or removed, a check failing at head that +did not at base, the parse failure count, the unknown suppression warnings, the audit and context +gaps, `delegating` and `checkContributions`. The per-entry suppression set and the two gaps are the +terms that run the dangerous way, since suppressing an already-failing check moves no score, no +measured flag and no new failure, so without them a pull request whose entire purpose was silencing +findings posts nothing. What is defended is that the union is complete, not that each term is load +bearing: three terms are shadowed by another today and kept because which shadows which depends on +the shape of the change. `MapReport.suppressions` is deliberately left out, its totals being summed +from the very per-entry arrays the loop compares. + +Two sections of the PR comment grow with the tree and both are capped, because GitHub's comment limit +is 65,536 characters and a 422 loses the whole comment to the section warning about a typo: a +mistyped directive applied tree wide renders 87,938 characters. The delegated list is capped at +fifteen rather than ten because a file name is one comma-separated item rather than a line naming +every known check, and the longest route file name in the tree is 130 characters. + +The `AUDIT` line has one shape for every count and no branch on the count, the count being correct +and the branch being where a false sentence gets written. A suppression whose id names no check is +carried through to both renderers rather than dropped, because dropping it silently makes a typo look +like an acknowledgement. + +## Tests, timeouts and CI + +**The docstring checker.** `docstringReferences.test.ts` enforces that every test a docstring in +`src/` names exists, the rule having been asked for six times in prose and broken six times. It reads +every backticked kebab-case token, every backticked glob against the corpus ids by prefix, and every +backticked prose phrase of five words or more with no code punctuation. It does not read a reference +written without backticks, a title of fewer than five words, a comment with no node after it (leading +ranges only, so a comment on the last line of a block is never scanned), or a `.test.ts` file or +`mutations.ts`, both exempted by name. Three negative controls run the same predicates over an +invented docstring, so the guarantee does not rest on `src/` happening to contain a bad reference. + +**`TREE_SCAN_TIMEOUT` is a hang detector, not a budget.** Neither real-tree test asserts anything +about how long a scan takes, so a number tight enough to be a performance budget is only a way to +fail on a busy runner. Measured on an 8-core box: 6.3s for the scan and 10.8s for the sweep +uncontended, rising to 34.0s and 39.7s under twelve concurrent copies, with one run dying on a 30s +timeout. `unit-tests-internal.yml` runs twelve concurrent shard processes on one runner, so that +contention is what CI does. 120s is 3x the worst contended run measured; 60s is not enough. + +**Parse failures come from a `ts.Program`,** not from the diagnostics array the parser hangs on the +source file, which is internal and which the compiler is free to rename. An undetected parse failure +shrinks the denominator and inflates the score, so it must not be the kind of thing a compiler +upgrade can switch off silently. The host hands the program the source file we already have, so +nothing is parsed twice; the cost is the program machinery, about 850ms to about 1450ms on a full +scan of the real tree. + +**The turbo task is uncacheable,** because its real inputs are mostly not its own files: they are +`apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and the workflow files, so +turbo replays a pass recorded before a route changed and caches a failure as a success. `turbo.json` +carries the reasoning and the rejected `inputs` alternative beside the config. + +Three roads reach the suite and all three are asserted. `pr_checks.yml` calls +`unit-tests-observability-map.yml` behind an `obsmap` paths filter and lists it in the `all-checks` +aggregate, without which the job gates nothing, `all-checks` needing an explicit list of jobs and +being unable to see another workflow. The filter watches all of `apps/webapp/app` plus the report +workflow, because the suite reads more than the routes folder and a rename outside it matched only +`webapp`, ran no job, and broke the build for whoever pushed next. It deliberately does NOT name this +package or the two non-webapp roots, since `internal` already matches `internal-packages/**` and +`packages/**` and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, so naming +them here runs the suite twice on every pull request touching the package. Widening `internal` to the +route paths instead runs all eighteen internal packages with postgres, clickhouse, redis and electric +to protect one test. + +The report workflow's own text is asserted from `integration.test.ts`, because it is the one thing +the docstring checker cannot reach. What those checks pin: the comment lookup sits in the cheap +`changes` job so the report job's gate can read it, the report job does not start unless the lookup +finished cleanly, and the id both steps use is one job output, so no two steps can disagree about +what a missing id means and turn a transient lookup failure into a duplicate comment or a false +all-clear. Both scan steps write their own file through `--out` rather than capturing stdout, +because +`pnpm --filter` takes its recursive path and some versions announce +`Scope: N of M workspace projects` on it, and a single line of that in head.json fails the renderer's +`JSON.parse` and degrades the workflow to the stale-report comment permanently. What is asserted is +the shape that cannot have the bug rather than the pinned 10.33.2 that happens not to. + +The corpus runs on the package's own paths and on a schedule rather than on every route pull request, +because it measures the tool's resistance to laundering, which only an edit to the tool can weaken, +and it costs four and a half minutes. The nightly covers tree drift late rather than not at all. diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md index 9b7f448706b..fe337cfc185 100644 --- a/internal-packages/observability-map/README.md +++ b/internal-packages/observability-map/README.md @@ -2,14 +2,15 @@ Scores every webapp entry point on whether it could explain itself during an incident, and prints the ones worth fixing. An entry point is a Remix `loader` or `action` under -`apps/webapp/app/routes`, 427 of them at the time of writing. +`apps/webapp/app/routes`, 427 of them today. -The number it prints today is 19 out of 100. That is not a bug, and the rest of this file is mostly -about why you should believe it. +The score is 19 out of 100. It is low because the webapp does not attach tenant identity to its +failures: **11 of the 412 measured entry points name an environment, project, organization or user +on a failure path.** Everything else, when it breaks at 3am, gives you the route and the request id +and nothing about whose request it was. -Every figure below was re-derived from a run of the tool on the tree as it stands. Every invariant -below names the test that holds it, because on this branch a claim written down without one has -turned out to be false more often than not. +Every figure here comes from a run against the tree as it stands. How the scanner reaches a verdict, +and what it refuses to decide, is in [INTERNALS.md](./INTERNALS.md). ## Running it @@ -20,114 +21,114 @@ pnpm --filter @internal/observability-map run map /api/v1/token # one route, w ``` The whole-tree run also writes `observability-map.json` at the repo root, which `--no-write` -suppresses. The single-route mode takes either the route path the report prints (`/api/v1/token`) or -the file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an -ambiguous prefix warns and names the alternatives rather than silently picking one -(`src/cli.test.ts`: `prefers an exact match over the routes it is a prefix of`, `warns when a -prefix matches more than one route rather than silently taking the first`). +suppresses. Single-route mode takes either the route path the report prints (`/api/v1/token`) or the +file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an +ambiguous prefix warns and names the alternatives rather than picking one. ## CI -A PR that touches `apps/webapp/app/routes` or this package gets a sticky comment scanning head +A pull request touching `apps/webapp/app/routes` or this package gets a sticky comment scanning head against the tip of the base branch, with the score, what changed, and the current fix list. Every -comment names the head commit it was rendered for, as a link to the PR's compare range, because the +comment names the head commit it was rendered for, as a link to the compare range, because the comment is edited in place across pushes and otherwise says nothing about which push it reflects. It -is report-only: nothing here fails the build or blocks a merge, and the gate stays deferred until a -later phase decides to add one. See `.github/workflows/observability-map.yml`. - -The workflow itself runs on every PR, and the paths above are a gate inside it rather than a `paths:` -filter on the trigger. GitHub evaluates one of those per workflow, so a PR whose diff stops matching -does not start the workflow at all, and the comment an earlier push left then stands for ever showing -findings that are no longer in the diff. The case that matters is not a PR reverted to nothing: it is -one touching a route and other files whose author reverts the route change and keeps the rest, which -still has a diff and still does not match. So a PR with a comment and nothing left to compare gets -the comment reconciled to its resolved state without scanning anything, and a PR with neither pays -for one cheap job that reads the paths and looks for a comment. - -This paragraph used to say "merge base", and two reviewers read that against -`github.event.pull_request.base.sha` and reported the workflow as the thing that was wrong. It is -the other way round. `actions/checkout` on a `pull_request` event checks out GitHub's test merge +is report-only: nothing here fails the build or blocks a merge. See +`.github/workflows/observability-map.yml`. + +The workflow runs on every pull request and applies the path list as a gate inside the job rather +than as a `paths:` filter on the trigger. GitHub evaluates one of those per workflow, so a pull +request whose diff stops matching never starts the workflow at all, and the comment an earlier push +left then stands for ever showing findings that are no longer in the diff. The case that matters is +a pull request touching a route and other files whose author reverts the route change and keeps the +rest, which still has a diff and still does not match. So a pull request with a comment and nothing +left to compare gets the comment reconciled to its resolved state without scanning anything, and one +with neither pays for a single cheap job that reads the paths and looks for a comment. + +The base is `github.event.pull_request.base.sha` rather than a merge base, which reviewers have +reported as a bug twice. `actions/checkout` on a `pull_request` event checks out GitHub's test merge commit, whose two parents are `base.sha` and the PR head, so the head tree being scanned already contains everything on the base branch up to `base.sha`. Diffing that against `base.sha` isolates -the PR's own work, which is what the comment is for. A real merge base would be the wrong base -here: it would leave the intervening base-branch commits in the head tree and out of the base tree, -and attribute all of them to the pull request. `git merge-base HEAD ` would not even do -that, since `base.sha` is a parent of `HEAD` and the command returns `base.sha` unchanged. +the pull request's own work. A real merge base would leave the intervening base-branch commits in +the head tree and out of the base tree, and attribute all of them to the pull request. -## What 19 means +## What the score means It is the mean score of the 412 entry points that had at least one applicable check, where an entry's score is the share of its applicable checks that passed. -Read that definition carefully before reading the number, because it has a property that will -mislead you otherwise. **Changing which routes a check applies to moves the score without anything -in the webapp changing.** It happened in the round that added this paragraph: widening the sensitive -cohort from 26 routes to 67 gave `auth-boundary` 39 more routes to look at, 36 of which already -passed it, and the global went from 15 to 19. Not one line of `apps/webapp` changed. The same thing -runs in reverse: narrowing a check, or a refactor that takes routes out of the denominator, lowers -or raises it for reasons that are about the tool. So a movement is only evidence about the codebase -once you have checked the CHECKS block below for an applicability change. Compare fix lists, not -scores. It is low because the webapp does -not attach tenant identity to its failures: **11 of 412 entry points name an environment, project, -organization or user on a failure path.** Everything else, when it breaks at 3am, tells you the -route and the request id and nothing about whose request it was. - -The score was 76 until we stopped crediting routes for the error handling they do not do. Emptying -every catch clause in the tree used to score it 100, which meant the metric paid you for deleting -error handling. - -The property behind that is now a test corpus rather than a claim. `src/mutationCorpus.test.ts` -applies 44 semantics-preserving or handling-deleting rewrites to the whole route tree in a temp copy -and asserts three things for each: the published global does not rise, the mean over the routes -measured in both runs does not rise, and for a semantics-preserving rewrite no individual route's -score rises or drops out of the measured set. Every laundering shape a reviewer has found on this -branch is an entry in it, `src/mutations.ts` holds them, and each entry says which it is. - -Two of them are worth naming because they are the ones the design turns on. Deleting every catch -clause in the tree drops the score from 19 to 8, so the metric does not pay you for removing error -handling. Wrapping every body in `try { ... } catch (e) { throw e }` leaves the global at 19 and -raises no route, so it does not pay you for adding error handling that does nothing either. - -The rewrites come in two directions and both matter. A subtractive one takes real signal away or -moves it about: delete the catches, wrap the body, merge the statements. An additive one puts fake -signal in: a classifying catch over a try that does nothing, a test whose two arms are the same, a -rethrow that can never run, a call whose name starts with `require`. The corpus had only the -subtractive half for a while, and the two largest holes ever found here were both additive. - -One of those is still open and the corpus says so. A catch over `try { 0; }` is refused, but -`canRaise` accepts any call, so `try { String(0); }` reads as real error handling: it takes the tree -from 19 to 44 and raises 224 routes. Telling an inert call from one that can throw needs types the -scanner does not have. - -The honest statement is "these 43 rewrites are defended, here they are, and here is the one that is -not", not "unpaddable". One entry, `dead-classifying-try-with-call`, runs as an expected failure -with the residual written out beside it. The corpus takes about four and a half minutes, so it is -gated behind `OBS_MAP_MUTATION_CORPUS=1` and run as its own CI job rather than in `pnpm test`. If -you change this package, run it: +One property of that definition will mislead you otherwise. **Changing which routes a check applies +to moves the score without anything in the webapp changing.** Widening the sensitive cohort from 26 +routes to 67 gave `auth-boundary` 39 more routes to look at, 36 of which already passed it, and the +global went from 15 to 19 without a line of `apps/webapp` changing. Narrowing a check, or a refactor +that takes routes out of the denominator, runs the same way in reverse. A movement is evidence about +the codebase only once you have checked the CHECKS block below for an applicability change. Compare +fix lists, not scores. + +The number is deliberately unflattering, and one platform change would move most of it. Nothing +central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line +through AsyncLocalStorage and forwards errors to Sentry, and the route builders log +`logBoundaryError(message, error, url)`. If the auth path pushed `environmentId` through +`trace(...)`, several hundred entry points would flip at once, and `request-context` would want +rethinking rather than celebrating. + +### What the number cannot tell you + +`request-context` checks that a failure-path log names a tenant field. It does not check that the +value is real. Adding a synthetic `environmentId: "obs-map"` field to the first object argument of +all 139 in-catch log calls, with no other change, takes the global from 19 to 29 and the CONTEXT +figure from 11 to 98. + +That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check reads +syntax: a field name, a call, a binding reference. None can tell a genuine tenant id from a +hardcoded string with the right key. A reviewer owns whether the value behind the field is real, the +same way a Lighthouse accessibility score checks that an `alt` attribute exists and not that its +text describes the image. The number tells you where to look. It does not tell you what you will +find there. + +### What stops it being gamed + +`src/mutationCorpus.test.ts` applies 44 semantics-preserving or handling-deleting rewrites to the +whole route tree in a temp copy and asserts three things for each: the published global does not +rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving +rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a +reviewer has found is an entry in `src/mutations.ts`, and each entry says which kind it is. + +Two entries are the ones the design turns on. Deleting every catch clause in the tree drops the +score from 19 to 8, so the metric does not pay you for removing error handling. Wrapping every body +in `try { ... } catch (e) { throw e }` leaves the global at 19 and raises no route, so it does not +pay you for adding error handling that does nothing either. + +One hole is open and the corpus says so. A catch over `try { 0; }` is refused, but `canRaise` +accepts any call, so `try { String(0); }` reads as real error handling: it takes the tree from 19 to +44 and raises 224 routes. Telling an inert call from one that can throw needs types the scanner does +not have. That entry, `dead-classifying-try-with-call`, runs as an expected failure with the +residual written out beside it, so the claim is "43 rewrites are defended and here is the one that +is not", never "unpaddable". + +The corpus takes about four and a half minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` +and runs as its own CI job rather than in `pnpm test`. Run it if you change this package: ```bash OBS_MAP_MUTATION_CORPUS=1 pnpm --filter @internal/observability-map exec vitest run \ src/mutationCorpus.test.ts --disable-console-intercept ``` -So the number is deliberately unflattering, and one platform change would move most of it. Nothing -central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line -through AsyncLocalStorage and forwards errors to Sentry, and the route builders log -`logBoundaryError(message, error, url)`. If the auth path ever pushed `environmentId` through -`trace(...)`, several hundred entry points would flip at once, and this check would want rethinking -rather than celebrating. - ## The five checks -- **error-classification**: does every catch clause decide what it caught, by branching on the - error or by guarding a parse it can answer for. A clause that only rethrows decides nothing and - is read as though there were no catch, so it neither passes nor fails. +- **error-classification**: does every catch clause decide what it caught, by branching on the error + or by guarding a parse it can answer for. A clause that only rethrows decides nothing and is read + as though there were no catch, so it neither passes nor fails. - **auth-boundary**: does a route handling credentials, access control, sessions, billing or impersonation check who is asking. -- **auth-scope**: a route builder authenticates the request, and its `authorization` option is - optional, so a route can be authenticated and scoped to nobody. This asks whether a sensitive - builder-wrapped route also narrows itself to the caller, in every export, by declaring - `authorization` or by filtering on the caller's own id. +- **auth-scope**: does a sensitive builder-wrapped route also narrow itself to the caller, in every + export, by declaring `authorization` or by filtering on the caller's own id. All nine route + builders authenticate the request, but their `authorization` option is optional and + `apiBuilder.server.ts` runs the RBAC gate inside `if (authorization)`, so a route can be + authenticated and scoped to nobody. That is the cross-org IDOR class `apps/webapp/CLAUDE.md` + names. It applies to 19 routes and 17 pass; both failures resolve their target organization from + the URL slug with no membership filter and put nothing but an ability check in front of it + (`_app.orgs.$organizationSlug.settings.sso/route.tsx` in its loader, + `_app.orgs.$organizationSlug.settings.team/route.tsx` in its action). The fix in each is + `members: { some: { userId } }` on the lookup. - **request-context**: when this entry point's failure is reported, is the tenant named. - **audit-trail**: does a sensitive mutation leave a record of who did it. Three routes do, all of them impersonation paths reaching `prisma.impersonationAuditLog.create` in @@ -135,10 +136,18 @@ rather than celebrating. `audit-trail` is excluded from the score. The other four are in it. +`auth-boundary`, `auth-scope` and `audit-trail` only look at routes `src/sensitivity.ts` calls +sensitive, and that cohort is the fix list's primary sort key, so what goes in it decides what a +reader sees first. 67 routes are in it today: credentials and tokens, envvars, billing and the two +billing settings the bare `billing` segment does not match, impersonation, membership and invites +and roles and the team page, the login surface, API keys, and org or project deletion. Calling a +guard never makes a route sensitive, and `src/webappSymbols.test.ts` fails if a symbol or path +segment in the vocabulary stops resolving in the webapp. + ## What the score is made of -The check list describes a composite the number mostly is not, so the report discloses the -shape instead of hiding it behind a weight. Today: +The check list describes a composite the number mostly is not, so the report discloses the shape +instead of hiding it behind a weight. Today: ```text CHECKS @@ -149,157 +158,73 @@ CHECKS audit-trail 49 applicable, 3 pass, 0 sole, not in the score ``` -`sole` is the figure that says the most: 223 of the 412 measured entry points have exactly one -applicable scored check, so their score is 0 or 100 on a single boolean. Read the family bars with -that in mind. They do not compare families on observability in general; they mostly compare them on -whether someone wrote a tenant field into a catch log. +`sole` says the most: 223 of the 412 measured entry points have exactly one applicable scored check, +so their score is 0 or 100 on a single boolean. Read the family bars with that in mind. They do not +compare families on observability in general; they mostly compare them on whether someone wrote a +tenant field into a catch log. -Weighting was considered and rejected in the design, and that reasoning has not changed: a -coefficient nobody can explain invites argument about the number instead of about the finding. The -block above is in the terminal report and in the JSON as `checkContributions` -(`src/score.test.ts`: `per-check contribution`; `src/report/terminal.test.ts`: `reporting what the score -is made of`). +Weighting was considered and rejected, because a coefficient nobody can explain invites argument +about the number instead of about the finding. The block above is in the terminal report and in the +JSON as `checkContributions`. ## Two findings are headlines, not list entries -`audit-trail` fails 46 of 49, and `request-context` fails 401 of 412. Printing either one per route +`audit-trail` fails 46 of 49 and `request-context` fails 401 of 412. Printing either one per route would bury the route-specific findings under the same sentence repeated hundreds of times, so both are reported as a figure: the `AUDIT` and `CONTEXT` lines. 328 entry points fail nothing except `request-context` and appear only in that figure, which leaves 76 in the fix list. An entry that fails `request-context` *and* another scored check keeps both findings and stays in the list, so `/account/tokens` still shows the whole picture. `audit-trail` does not count as "another" for this -purpose: it is already a headline, so a route failing only `request-context` and `audit-trail` +purpose, being already a headline, so a route failing only `request-context` and `audit-trail` collapses too (28 do today, all of them sensitive). 42 of those 328 are sensitive, so the `CONTEXT` line says how many. Read them out of `observability-map.json`, where every entry keeps its full check results, rather than assuming the -list is the whole story. +fix list is the whole story. `request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score is meant to show it. Only the presentation collapses. -## Not applicable is not a pass - -An entry with no applicable scored check is `measured: false`, and it is left out of every mean the -report computes (`src/score.test.ts`: `excludes an unmeasured entry point from the global mean`, -`excludes an unmeasured entry point from its family mean too`). Its `score` field reads 100, which -is a placeholder for "nothing was measured here", not a verdict, and nothing averages it. This -matters because the alternative, letting unmeasured entries into the mean at 100, would let the tool -look better the less it understood. The header prints every count (`412 measured, 15 unmeasured`) so -the denominator is never hidden, and a family with nothing measured renders as `not measured` rather -than as a full green bar (`src/report/terminal.test.ts`: `renders a family with nothing measured as not -measured, not as 100`). - -15 of those 427 routes are unmeasured because `isTrivial` (`src/triviality.ts`) rules them out before -any check runs. Trivial means a body of three statements or fewer, three or fewer calls, no -try/catch, no builder wrapping it, and nothing in the calls or the source naming a datastore or a -service (`prisma`, `logger`, `fetch`, `redis`, and the like). Parse the params, build a path, -redirect: nothing there for a check to find evidence in either way. Exclusion is a denominator exit, -not a credit: a trivial route's `score` is the same placeholder 100 that an unmeasured entry always -carries, and it is left out of every mean for the same reason. - -## A route whose body is somewhere else - -`export { action } from "./handler.server"` and `export const action = handleWebhook` are not -trivial routes. A redirect stub genuinely has nothing to instrument; a delegating route has work the -scanner cannot see. Both used to produce the same verdict, so moving a body into a `.server.ts` -file, an ordinary refactor, deleted the route from the metric while the report said nothing. - -Delegating routes are now counted apart from the unmeasured ones, listed on a `DELEGATED` line and -carried in the JSON as `delegating`, the same treatment a parse failure gets and for the same -reason: the denominator is smaller than the entry point count and nothing about these routes has -been checked (`src/score.test.ts`: `refactoring a body out of the route file`, -`a route that delegates its body to another module`; `src/report/terminal.test.ts`: `reporting a route whose -body is in another module`). There are none in the tree today, which is exactly why it would have -gone unnoticed when someone wrote one. - ## When a check declines to judge The rule every applicability decision follows: **would this evidence necessarily be visible in the body if it existed?** -A log call inside a catch would be, because the catch is right there in the body being read. So its +A log call inside a catch would be, because the catch is right there in the body being read, so its absence is evidence of absence and `request-context` fails the route. A guard on work that happens -inside an imported helper would not be, because neither the work nor the guard is in the body. So -`auth-boundary` reports not-applicable with a detail saying it could not verify, rather than -accusing the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls -`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, -a file this tool never opens. Five of the 67 sensitive routes sit out for this reason today. +inside an imported helper would not be, because neither the work nor the guard is in the body, so +`auth-boundary` reports not-applicable with a detail saying it could not verify rather than accusing +the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls +`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, a +file this tool never opens. Five of the 67 sensitive routes sit out for this reason. -The failure mode this rule exists to prevent is a fix list whose top three entries are all wrong. -That happened, twice, and both times the cause was a check asserting something the evidence did not -support. +The failure mode the rule exists to prevent is a fix list whose top three entries are all wrong, +which is what a check asserting more than its evidence supports produces. -## Sensitivity, and the names the tool matches on +## Not applicable is not a pass -`auth-boundary`, `auth-scope` and `audit-trail` only look at routes `src/sensitivity.ts` calls -sensitive, and that cohort is the fix list's primary sort key, so what goes in it decides what a -reader sees first. 67 routes are in it today: credentials and tokens, envvars, billing and the two -billing settings the bare `billing` segment does not match, impersonation, membership and invites -and roles and the team page, the login surface, API keys, and org or project deletion. - -Two rules hold the vocabulary honest. - -Calling a guard can never be what makes a route sensitive. `requireAdminApiRequest` was on the -symbol list once and made 34 of the then 67 sensitive routes sensitive purely for being guarded, -which `auth-boundary` then passed every one of them for (`src/sensitivity.test.ts`: `does not treat -calling the admin guard as what makes a route sensitive`). - -Every name and every segment has to exist. Half the symbol list once named nothing at all: -`Set.has` is exact, and `setImpersonation`, `createJWT`, `signJWT` and `updateEnvVars` are exported -nowhere in the webapp, so the symbol half of the classifier was quietly doing almost nothing. -`src/webappSymbols.test.ts` resolves every sensitive symbol, every path segment and every entry in -`auth-boundary`'s guard list against `apps/webapp/app` and the two packages the webapp -authenticates through, and fails if one stops resolving. The one exception is -`ANTICIPATED_SEGMENTS`, three words that name no route yet and are held to naming none. - -The same test is what stops `auth-boundary`'s guard list rotting. That check used to match -`/^(require|authenticate)/`, so any callee at all beginning `require` cleared a sensitive route: -`requireSsoEntitlement`, a plan check, cleared the org SSO settings page, and a local -`requireValidParams(request)` would clear whatever route was written next. It also matched -`/Authenticated/`, which passed `resolveAuthenticatedEnv` on ten routes, a `findFirst` by -environment id that authenticates nothing. Both shapes are corpus entries now -(`fake-require-guard`, `fake-authenticated-lookup`): under the patterns they took the tree from 18 -to 19 and raised five routes, and under the accept-list they raise nothing. - -## Authenticated is not the same as scoped - -All nine route builders authenticate the request, which is why `auth-boundary` passes a -builder-wrapped route. Their `authorization` option is optional and `apiBuilder.server.ts` runs the -RBAC gate inside `if (authorization)`, so a route can be authenticated and scoped to nobody. That is -the cross-org IDOR class `apps/webapp/CLAUDE.md` names: "A PAT route must resolve its target -org/project scoped to the caller's membership. Skipping it opens cross-org access." - -`auth-scope` is the check that can say "authenticated but not scoped" as a finding in its own -right. It applies to 19 routes and 17 pass. It reads two things as scoping, and requires EVERY -builder-wrapped export of a file to have one of them: an `authorization` option with a real value, -or a query in that export's own handler filtered on the caller's own id -(`userId: authentication.userId`, `userId: user.id`). - -An `ability.can(...)` call in the handler is deliberately not a third way. The same CLAUDE.md -passage says why: the OSS fallback ability is permissive -(`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and -`buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an -ability check enforces the role while the membership-scoped query is the tenant floor. - -The two routes that fail both resolve their target organization from the URL slug with no -membership filter, and put nothing but an ability check in front of it: -`_app.orgs.$organizationSlug.settings.sso/route.tsx` in its loader, whose `resolveOrg` is -`findFirst({ where: { slug } })`, and `_app.orgs.$organizationSlug.settings.team/route.tsx` in its -action, whose org id comes from `resolveOrgIdFromSlug`. Both were hand-read. The fix in each is to -put `members: { some: { userId } }` on the lookup. - -That per-export rule is the load-bearing half. Both of those files scope themselves in their OTHER -export, so an entry-point-wide reading passed them, and the exposure is per export. - -One thing to know before reading a score on any of these 19 routes. `auth-scope` is only applicable -when the route uses a builder, and `auth-boundary` passes any route that uses a builder, so -**`auth-scope` applicable structurally implies `auth-boundary` pass**: all 19 carry the same -`auth-boundary` detail, "authenticated by the builder". That free point is a third or a quarter of -each of their scores. The 19 average 59.7 as scored and 44.6 with `auth-boundary` taken out, and -`settings.team`, a confirmed cross-org exposure, scores 25 rather than 0 because of it. The score is -not wrong, since the builder does authenticate. It is just less informative here than it looks, and -the finding is the thing to read. +An entry with no applicable scored check is `measured: false`, and it is left out of every mean the +report computes (`src/score.test.ts`: `excludes an unmeasured entry point from the global mean`, +`excludes an unmeasured entry point from its family mean too`). Its `score` field reads 100, a +placeholder for "nothing was measured here" that nothing averages, because the alternative of +letting unmeasured entries into the mean at 100 would let the tool look better the less it +understood. The header prints every count (`412 measured, 15 unmeasured`) so the denominator is +never hidden, and a family with nothing measured renders as `not measured` rather than as a full +green bar. + +15 of the 427 routes are unmeasured because `isTrivial` rules them out before any check runs. +Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder +wrapping it, and nothing in the calls or the source naming a datastore or a service (`prisma`, +`logger`, `fetch`, `redis`, and the like). Parse the params, build a path, redirect: nothing there +for a check to find evidence in either way. Exclusion is a denominator exit, not a credit. + +A route whose body is somewhere else is a different case. `export { action } from "./handler.server"` +and `export const action = handleWebhook` are not trivial: a redirect stub genuinely has nothing to +instrument, while a delegating route has work the scanner cannot see, and treating them alike would +delete a route from the metric whenever someone moved a body into a `.server.ts` file. Delegating +routes are counted apart from the unmeasured ones, listed on a `DELEGATED` line and carried in the +JSON as `delegating`, the same treatment a parse failure gets and for the same reason. There are +none in the tree today, which is exactly why the case needed writing down before someone wrote one. ## Suppression @@ -307,43 +232,18 @@ the finding is the thing to read. // obs-map-disable auth-boundary -- public by design, see ADR 12 ``` -The reason is mandatory: a suppression without one is ignored (`src/suppression.test.ts`: `ignores -a suppression with no reason`). The directive is read from comments only, so a string literal -quoting it does not switch a check off (`does not suppress from a directive quoted inside a string -literal`, and six more for template literals and JSX text). +The reason is mandatory: a suppression without one is ignored. The directive is read from comments +only, so a string literal quoting it does not switch a check off. -It applies to the whole entry point, not to the line under it. It was called -`obs-map-disable-next-line`, which was untrue in a way that mattered: a directive on the last line -of a file switched a check off for everything above it. Genuine line scoping is not available, -because a finding is attached to an entry point and carries no line number to match against, so the -name was corrected instead. The old spelling is not honoured (`does not honour the old -next-line -spelling`). +It applies to the whole entry point, not to the line under it. Line scoping is not available, +because a finding is attached to an entry point and carries no line number to match against, which +is why the old `obs-map-disable-next-line` spelling is not honoured. A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing check holds the number still rather than improving it (`src/score.test.ts`: `does not raise the -score when a failing check is suppressed`; `scoring 100 and 0, suppressing every check on the -failing entry leaves the global at 50`), and `suppress-every-check` is the tree-scale version in -the corpus. What you buy is removal from the worklist with a reason on the record. The report prints how many suppressions are -in force so the practice stays visible. - -## The gaming boundary - -`request-context` checks that a failure-path log names a tenant field. It does not check that the -value is real. A codemod that added `environmentId` to every in-catch `logger.error` call, wiring it -up to the wrong variable or a constant, would move the score exactly as far as one that wired it up -correctly. Measured on the real tree: adding a synthetic `environmentId: "obs-map"` field to the -first object argument of all 139 in-catch log calls, with no other change, takes the global from 19 -to 29 and the CONTEXT figure from 11 to 98. That measurement is a one-off script rather than a -corpus entry, because the corpus asserts that the score must not rise and this rewrite is supposed -to. - -That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check here reads -syntax: a field name, a call, a binding reference. None of them can tell a genuine tenant id from a -hardcoded string with the right key. What a reviewer owns is whether the value behind the field is -real, the same way a Lighthouse accessibility score checks that an `alt` attribute exists and not -that its text describes the image. The number tells you where to look. It does not tell you what -you will find there. +score when a failing check is suppressed`). What you buy is removal from the worklist with a reason +on the record. The report prints how many suppressions are in force so the practice stays visible. ## Known limits @@ -355,8 +255,8 @@ Read these before trusting a specific verdict. entry points; the other 5 hand their work to an imported helper and are reported as unverified rather than unguarded. - **A guard is matched by name, not by what it does.** The accept-list is 29 names read off the - webapp, plus two `SOFT_GUARDS`. `src/webappSymbols.test.ts` proves each one is declared - somewhere; nothing proves the declaration it found is the guard we meant. `authenticateAdmin` and + webapp, plus two `SOFT_GUARDS`. `src/webappSymbols.test.ts` proves each one is declared somewhere; + nothing proves the declaration it found is the guard we meant. `authenticateAdmin` and `authenticatePlainRequest` are local helpers inside one route file each, so a second route declaring its own no-op function of either name would be credited. - **Two guard names are only checked as far as being read.** `getUser` and `getUserId` answer with @@ -365,16 +265,22 @@ Read these before trusting a specific verdict. is whether the test guards anything: `if (!user) { logger.warn("anonymous"); }` followed by the work reads the same as returning. - **`authenticate` and `isAuthenticated` are unresolved on purpose.** They are remix-auth's, and - resolving them meant reading a path inside `apps/webapp/node_modules`, which fails confusingly on + resolving them means reading a path inside `apps/webapp/node_modules`, which fails confusingly on an install-layout change. They are listed in `EXTERNAL_GUARDS` instead, so the resolution test still rejects a name that is neither first-party nor listed. +- **`auth-scope` applicable structurally implies `auth-boundary` pass.** The check only applies to a + builder-wrapped route, and `auth-boundary` passes any builder-wrapped route, so all 19 carry the + same `auth-boundary` detail, "authenticated by the builder". That free point is a third or a + quarter of each of their scores: the 19 average 59.7 as scored and 44.6 with `auth-boundary` taken + out, and `settings.team`, a confirmed cross-org exposure, scores 25 rather than 0. Read the + finding rather than the score. - **`auth-scope` cannot tell a caller-id filter from a caller-id actor argument.** `presenter.call({ userId: user.id })` narrows the query; `generatePortalLink({ organizationId, - userId: user.id })` just records who asked. Both read as scoping. Separating them means following - the argument into the callee, so the four helpers credited this way - (`ApiKeysPresenter`, `TeamPresenter`, `regenerateApiKey`, `DeleteOrganizationService`, all of - which do `members: { some: { userId } }` and throw) were hand-read instead. No route in the tree - passes on an actor argument alone. + userId: user.id })` records who asked. Both read as scoping. Separating them means following the + argument into the callee, so the four helpers credited this way (`ApiKeysPresenter`, + `TeamPresenter`, `regenerateApiKey`, `DeleteOrganizationService`, all of which do + `members: { some: { userId } }` and throw) were hand-read instead. No route in the tree passes on + an actor argument alone. - **`auth-scope` reads property assignments in that export's own handler.** A handler that pulls the id into a local first, `const userId = user.id; ... { userId }`, or that builds its filter in a same-file helper, scopes itself and is not seen, so it would be reported as unscoped. @@ -382,12 +288,9 @@ Read these before trusting a specific verdict. route whose action is builder-wrapped and whose loader is a plain `export async function loader` is judged on the action alone, and the pass detail, "every builder-wrapped export has an authorization gate", is true of what it read while reading as a claim about the whole route. Ten - routes in the tree mix the two, and one of them is sensitive, so it is the only one the check runs - on: `_app.orgs.$organizationSlug.settings._index/route.tsx`, whose builder-wrapped action carries - the pass and whose plain loader filters on `members: { some: { userId } }` and is scoped. That was - hand-read; nothing in the check saw it. Widening the check to a hand-written export means deciding - first whether that export is authenticated at all, which is `auth-boundary`'s question rather than - this one. + routes in the tree mix the two, and the one sensitive enough for the check to run on is + `_app.orgs.$organizationSlug.settings._index/route.tsx`, whose builder-wrapped action carries the + pass and whose plain loader filters on `members: { some: { userId } }`. That was hand-read. - **Three login-flow routes fail `auth-boundary` correctly and unhelpfully.** `/auth/sso`, `/api/v1/authorization-code` and `/api/v1/token` are unauthenticated by design: the caller is anonymous at that point, which is the whole purpose. The check's statement about them is true and @@ -396,560 +299,29 @@ Read these before trusting a specific verdict. - **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route can be reported as recording nothing while it records plenty. -- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot - say whether a clause does anything besides rethrow, so `error-classification` withholds credit - rather than granting it. Crediting it would reopen the free-points path a single `logger.error` - line wide. +- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot say + whether a clause does anything besides rethrow, so `error-classification` withholds credit rather + than granting it and reopening the free-points path a single `logger.error` line wide. - **Only the first object-literal argument is read** for identifier fields, and only its property names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither does a second object. -- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` - is a fresh boundary per element, so its clause is not read as the route's own error handling. The +- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` is + a fresh boundary per element, so its clause is not read as the route's own error handling. The test is the method name, which cannot tell `users.map` from `Result.map`. Being wrong there costs - precision rather than points: a refused catch fails the route rather than excusing it, so no - wrapper can turn a swallow into a not-applicable by getting the boundary rule to refuse it. + precision rather than points: a refused catch fails the route rather than excusing it. - **A route that delegates only one of its two exports is judged on the other.** `export { action } from "./x"` beside a loader written in the file is not counted as delegating, so half the route is scored and half is invisible. -- **`try { String(0); }` still buys a pass.** The open corpus entry, above. It is the largest single +- **`try { String(0); }` still buys a pass.** The open corpus entry above, and the largest single hole known in the tool: measured live, it takes the tree from 19 to 44 and raises 224 routes. -- **A forged tenant field buys a pass too.** The gaming boundary above, restated here because it - belongs on this list: `request-context` reads the field name, never the value, so a codemod - writing `environmentId: "obs-map"` into every in-catch log call takes the global from 19 to 29. - Unlike the entry above this one is not a bug to fix, since no syntactic check can tell a real - tenant id from a constant, but it bounds what the number can mean either way. +- **A forged tenant field buys a pass too.** `request-context` reads the field name, never the + value, so a codemod writing `environmentId: "obs-map"` into every in-catch log call takes the + global from 19 to 29. Unlike the entry above this one is not a bug to fix, since no syntactic + check can tell a real tenant id from a constant, but it bounds what the number can mean either + way. - **The score is a mean of means over a heuristic.** Read the fix list, the two headline figures and the CHECKS block. Watching the single number for small movements will mislead you. -## How the scanner reads a route - -`scan.ts` produces one `EntryPoint` per route module, carrying only body-scoped evidence. Three -rules decide what "the body" means, and every finding in this tool rests on them. - -**One hop, same file only.** A loader that delegates to a helper declared in the same file has that -helper's statements, try/catch and callees counted as its own. A helper's own helpers are not -followed, the visited set stops a cycle, and nothing imported from another module is ever opened. - -**Nested functions count as work.** A statement inside a callback written in the body is still a -statement the route runs. Leaving them out let `trace("x", async () => { whole body })` collapse a -route to one statement, which is inside the triviality limit, so every check reported -not-applicable for it. `wrap-body-in-trace` in the corpus is that shape. - -**Per export, not per file.** Six fields come in `loaderX`/`actionX` pairs, and the union is only -offered where the question itself is file-wide. This split is the fix for a whole family of false -passes, all the same shape: a file whose loader called `requireUser` and whose action called -nothing read as "guarded in the body", and a file whose loader was `createLoaderApiRoute(...)` -credited its hand-written action with the builder's authentication. `routeExports.ts` is the one -enumeration both per-export checks read, because `auth-scope` and `auth-boundary` each grew their -own `[loader, action]` literal and only one of them got each fix. - -`calleeNames` is the union and stays entry-point wide because the three questions that read it are: -`sensitivity.ts` asks what the file touches, `triviality.ts` asks how much the file does, -`audit-trail` asks whether the file records anything. There is deliberately no entry-point-wide -`checkedCallees`, so no check can reach for a union that would say a loader's reading of `getUser` -speaks for the action beside it. - -The split cannot drift from the union it came from: one push site in `scanFile` fills the whole-entry -list and each owning export's list, `scan.test.ts` pins the property on fixtures and -`integration.test.ts` pins it again over the real tree (`every callee name is attributed to an -export that exists`). - -Two fields exist because the bare callee name is not enough. `calleeName` keeps only the last -segment, so `prisma.organization.findFirst` arrives as `findFirst` with the receiver that says what -is being called gone; `calleeTexts` keeps the whole dotted path, which is how the per-export -triviality rule knows a three-statement body reaches the datastore. `auth-boundary` matches the bare -name on purpose, so a guard call cannot be hidden by its receiver. - -### Catch evidence, per clause - -`CatchEvidence` is one record per catch clause rather than a set of booleans per entry point, -because 39 routes have more than one catch and 17 mix a narrow parse guard with a broad handler. -Under the old aggregate booleans a single well-behaved catch spoke for the swallow next to it. - -- `rethrows`: throwing is the clause's only way out, i.e. a throw is reached on the clause's - guaranteed path AND the clause contains no live `return` anywhere. -- `throws`: a throw is reached on that path, whether or not it is the only way out. Kept separately - so a verdict can say what is true of a clause that both throws and returns. The detail line "takes - one way out regardless of what was thrown" is only true of a clause that never throws, and saying - it of a clause that does was a false accusation on 16 clauses in the tree. -- `branches`: the clause picks what to do from what it caught. An `if` or `switch` whose condition - references the caught binding and at least one of whose arms returns or throws, or a conditional - that is the whole value of a `return`/`throw`. `if (retries > 0)` does not count, - `if (e instanceof Error) { }` does not count, a bindingless `catch { }` cannot count at all, and an - `instanceof` used only to word a message does not count either, because every error still leaves by - the same path. -- `guardsParse`: the guarded region parses something. `JSON.parse`, `request.json()`, a zod - `parse`/`safeParse`, a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three - constructors are read as `ts.isNewExpression` because a `new` expression is not a call and the - call-callee scan never sees them. Any constructor at all would mean `new BranchesPresenter()` - excuses a catch guarding ordinary work, which was true of 77 try blocks in the tree. -- `guardCanRaise`: the region does anything that could reach the clause. False means `try { 0; }` and - little else, because any call counts, including one that cannot throw. -- `guardMayRaise`: the containment twin, false only when the region provably cannot raise. Everything - `canRaise`'s whitelist misses stays true here, so `guardCanRaise` implies `guardMayRaise`. -- `awaitsOnlyParse`: everything the region waits for is one of those parses, or a read of the body it - parses. -- `tryStatementCount`: statements in the guarded block, counted as `statementCount` counts them. - -`canRaise` is a whitelist and it misses real raising code, which is the safe direction but does -matter: a destructuring declaration (`const { a } = undefined` throws), a temporal-dead-zone read, a -coercion that raises, and a `delete` on a frozen object all read as unable to raise. That is why the -refused-swallow arm of `error-classification` reads the route's own deciding catches through -`guardMayRaise` and never through `guardCanRaise`. Ordering it off can-raise accused a route that -owns a real classifying catch of owning none, which was flatly untrue (`does not accuse a route that -owns a catch of owning none`). - -"Does this route catch anything" is `catches.length`, never `hasTryCatch`. A `try`/`finally` with no -catch leaves `hasTryCatch` true and `catches` empty: nothing is swallowed there, the error propagates -once the cleanup has run, and reading the old flag as a catch put -`admin.api.v1.runs-replication.status.ts` at the top of the first rendered fix list. - -## The dead-code defence - -Both of the catch-clause answers are read off the clause's guaranteed path. The governing rule: the -walk may enter a construct exactly where the entered statements are guaranteed to execute whenever -the clause body runs, so no credit can ever come from code a semantics-preserving edit could have -added dead. - -Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` that has no catch -clause and whose finally contains no jump out of itself, the sole clause of a single-default -`switch`, the then-arm of an `if` whose condition is exactly the literal `true` keyword, and both -arms of an `if`/`else` with per-arm states merged by intersection. - -Not entered, deliberately: a bare `if` without an else, loops other than `do`, labelled statements, -function-like nodes, nested catch clauses, finally blocks, and the tryBlock of a `try` that has a -catch clause, where a throw is intercepted by the nested catch rather than escaping. - -That rule replaced a list of statically-false shapes an earlier round kept extending, and the list -was losing. `if (false)` and `while (false)` were recognised; `for (;false;)`, `if (true) {} else`, -`switch (1) { case 2: }`, `try {} catch`, `for (const x of [])`, `for (const k in {})`, `if ("")`, -`if (!true)` and `if (1 === 2)` were not, each worth 50 points a route. Asking for the throw to be -unconditional refuses all eleven without naming any of them. `dead-*` in the corpus is the -tree-scale proof, one entry per shape. - -`rethrows` asks for one thing more: no `return` anywhere in the clause. Without it a `throw error;` -written after a statement that already exited read as a rethrow, in seven spellings. -`dead-throw-after-*` in the corpus covers them. The cost is real and worth stating: -`catch (e) { if (transient) throw e; return null; }` no longer reads as a rethrow, so it fails rather -than sitting out. That is the direction to be wrong in, since the reverse hands out points. - -### Two folds, pointing opposite ways - -There are two literal folds in `scan.ts` and unifying them would be a bug. - -`containsLiveWhere` folds any literal guard `literalTruth` can decide, and it is strictly -subtractive against a plain containment read: wherever the truth cannot be decided, every hit -containment would have found is still found. That is what lets its two callers read it for opposite -purposes. In `catchClauseEvidence`'s `exited` flag a hit BLINDS the walk to whatever follows, and -containment blinded it on a provably dead statement, so prepending one to a deciding clause turned -its pass into a swallow verdict on 78 real routes. In `selectsADistinctPath` a hit GRANTS a branch, -and containment granted one for an arm whose only exit was dead: `dead-armed-instanceof-if`, measured -at 80 routes and the tree from 19 to 27. Subtracting dead hits only ever un-blinds in the first case -and only ever withholds in the second. - -The walk's own entry tickets fold nothing but the literal `true` keyword. `!!1`, `1` and `!false` are -deliberately not entry tickets, because entry GRANTS credit and a wrong grant pays, where -`literalTruth`'s wider folding only ever withholds blindness. Do not unify the two. - -`literalTruth` treats `&&`, `||`, an identifier, a call, a bigint and a template with substitutions -as undecidable on purpose, so a live guard can never be read as dead. The cost of that is -`dead-conjunction-instanceof-if`, a corpus expected failure: `e instanceof Error && false` both -references the caught binding and can never be true, and no fold in the file can see it. Widening the -fold is a different rule with its own measurement. - -The `exited` flag is raised at the END of each statement, after that statement's own branch check. A -deciding statement contains an exit by definition, so raising it first makes every such statement -refuse itself, measured at 78 routes losing their pass. This ordering leaves the real-tree report and -all 240 clauses' evidence byte-identical. - -### A finally that cancels the try - -A finally block that completes abruptly supersedes the try's and the catch's completion, so an exit -written in either never leaves the statement. Two places read that, in opposite directions. - -`catchClauseEvidence` refuses to enter a catchless try whose finally holds a jump out of itself, -because entry grants rethrow credit and the throw would never escape the clause. The refusal is a -containment read, and it is over-approximate on purpose: a jump that only may run still refuses -(`refuses the tryBlock when the finally only may break`). `dead-throw-in-cancelled-try` in the corpus -is the tree-scale shape, worth 80 routes and 8 global points when measured. - -`containsLiveWhere` then folds the same statement to its finally's own statements, so a refused -statement cannot blind the walk to the real classification below it (`keeps the classification after -a finally-break no-op`). A finally holding a `return` is covered by the explicit `containsLiveReturn` -read instead, because `try { throw e; } finally { return null; }` genuinely swallows. - -### The residual both branch tests share - -Two arms that produce the same outcome by different spellings still read as a real decision. -`if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides nothing, -and so does the `if` with no `else` whose arm returns what the statement after it returns. Telling -those apart needs the produced values compared for meaning rather than for text, which is a -different kind of analysis from anything else in the file. The textual comparison is the cheapest -thing that catches the copy-paste form, which is the one a mutation produces. - -## Parse guards, and the narrow-try count - -A catch clause counts as a parse guard, rather than as the route's error handling, when the try block -parses, waits for nothing except that parse, and is short. All three conditions are load bearing and -two of them are corrections. - -`awaitsOnlyParse` is what a statement count cannot express. -`try { const body = await request.json(); return await handleEverything(body); } catch { 500 }` is -two statements, one of them a parse, and the whole handler inside it: the count reads it as narrow -and it is the `otel.v1.logs.ts` swallow written compactly. Asking what the block waits for separates -them, and unlike the count it does not care how the statements are punctuated or how deeply the work -is nested. Awaiting is the signal rather than calling, because the calls that prepare a parse's input -are ordinary synchronous string work (`matchPattern.slice(4)` before a `new RegExp`), and requiring -every CALL to be a parse refuses four of the tree's clearest guards. - -Two residuals follow from awaiting being the signal, both in the round A fix 3 report. A block that -does its non-parse work synchronously still reads as a guard. And `guardedWork` looks for a -`ts.AwaitExpression`, which `for await (...)` and `await using` are not. Neither occurs in the tree -and neither is reachable by rewriting a real route, since both need work that is not there to begin -with. - -`NARROW_TRY_STATEMENTS` is 2, so the guarded operation can bind its result -(`const stripped = ...; new RegExp(stripped);`) and a third statement means the try has started to -cover the handler. The idiom it was hand-read against: 55 of 427 entry points, 11 of the failures at -the time, all eleven the deliberate `try { body = await request.json() } catch { 400 }` shape. - -It is an absolute count and not a ratio against the enclosing body, because a ratio is diluted by -anything else in the same body: padding the action with unrelated statements after the try relabelled -the same broad swallow as a narrow guard, moving the denominator without touching the clause. -`inert-statements-after-try` in the corpus is that shape. - -What the count is not is unpaddable, which an earlier docstring and a commit subject both claimed. -`countStatement` counts declarators and comma operands rather than semicolons, so -`const a = f(), b = g(), c = h();` is three and `a(), b(), c()` is three; that is what -`merge-declarations` and `merge-comma-expressions` check. A third way nobody has written down would -work, which is why the count is no longer the only condition and no longer the load-bearing one. - -Two rejected alternatives, both measured. Requiring the clause to answer with a 4xx credits, on its -own, 11 clauses guarding four to thirty statements, the widest swallows in the tree, including -`admin.api.v1.workers.ts`, whose 28-statement try answers every failure with a 400 carrying the -internal error message; added on top of the rest it costs three routes their pass, all three narrow -guards computing a fallback value rather than answering a request. And a narrow guard is not a way to -qualify as classification on its own: reading all eleven entry points that limb would clear found six -real swallows, including a silent run cancellation and two credential paths reporting a database -failure to the browser as a 400 with an internal message in it. - -## The iteration-callback boundary - -`items.map((item) => { try {...} })` is a fresh catch per element, so its clause is not the route's -own error handling. `trace(async () => {...})`, `mutateWithFallback({ pgMutation: ... })` and -`new ReadableStream({ start: ... })` all invoke their callback exactly once, so theirs is. The -structural signal is the method name, which is a list of eight, because nothing in a syntactic scan -can tell `users.map` from `Result.map`. - -Being wrong here is asymmetric, and the direction that used to pay no longer does. A refused catch is -kept WITH its evidence, built by the same machinery as an own catch, and judged on what it does -rather than on where it sits. A refused swallow fails the route whenever nothing the route owns -decides, and that arm is deliberately not conditioned on the route owning no catches, so an own inert -rethrow catch cannot lift a refused swallow out of the verdict. A route whose only catches are refused -and none of them swallows sits out at not-applicable and never passes, which is what keeps a prepended -dead deciding `.map` from minting a pass on the 261 catchless routes. `dead-deciding-map` holds that -at tree scale. - -That is what makes the name list survivable. `Result.map(...)` is a corpus entry that passes rather -than a hole: relocating a swallow behind the boundary still fails -(`still fails a swallow wrapped in a non-array receiver's .map(...)`), and relocating a decision earns -at most the route's exit from the denominator. A receiver that is an array literal of one element or -none is refused outright, since it cannot iterate. - -The other direction still costs precision. A per-item callback under a callee the list does not know, -`pMap(items, cb)` or `Array.prototype.map.call(items, cb)`, is attributed to the route, so a -per-element catch that decides can carry it to a pass. No mutation of a real route produces it: a -route has to already be iterating for the shape to exist. It is a wrong verdict waiting for a route -to be written that way rather than a laundering path, and it is why the list is worth extending when a -new iteration helper shows up in the tree. - -## What auth-scope reads as scoping - -Three conditions, and the first version had only the middle one, which made the check free to defeat. -Prepending `const __unused = { anything: user.id };` to every body raised `settings.sso` and -`settings.team`, the only two findings `auth-scope` has ever produced and both confirmed cross-org -exposures. `dead-caller-scope-object` and `dead-caller-scope-userid` are the two halves of that -shape. - -- The value has to be the caller's own id, anchored at both ends: the root is one of the auth - bindings a builder hands the handler and the last segment is an identity field, so `user.name` is - not a scope and neither is `run.userId`, which is a resource's owner. -- The property NAME has to be an identity field. Of the ten names that take a caller-id value in the - route tree, `sub`, `value` and `consumerId` are the three that are not, and `anything: user.id` is - what a mutation writes. -- The object has to be handed, through any depth of nesting, to a call that could narrow a read with - it. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. - -The third condition is a denylist of sinks rather than an allowlist of query callees, and that is a -measurement. 72 distinct callees are handed a caller id across the route tree, running from -`prisma.project.findFirst` through `presenter.call` and `new DeleteProjectService().call` to bare -`regenerateApiKey`. No name pattern separates those from `sendToPlain`, so an allowlist would accuse -whichever route named its helper next, and a wrong accusation is the failure this check cannot -afford. The sinks refused are the log line and the response body, both of which take the very -`{ userId: user.id }` object a query filter takes: loggers account for 13 of the caller-id sites and -the two response serializers for 2 more. The shape is already in the tree rather than hypothetical, -in `engine.v1.dev.runs...attempts.start`, which logs `{ environmentId: ... }` beside the -`runStore.findRun` that earns its credit honestly. `log-caller-scope-userid` covers it at tree scale. - -A callee with no readable name of its own is credited, because refusing it would ACCUSE the route, -and under-crediting beats accusing a route that is fine. `String({ userId: user.id })` therefore -reads as scoping, the same way `try { String(0); }` reads as error handling and for the same reason. - -`authorization: undefined`, `null` and `false` are read as not declared, because -`apiBuilder.server.ts` gates every option behind `if (option)` and declaring one is what the check -credits. - -## Triviality, in detail - -Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder -wrapping it, and nothing in the calls or the hint text naming a datastore or a service. - -Both limits are 3 because both real shapes need three: parse the params, build a path, redirect, or -an environment guard and two returns. Allowing a fourth call admits -`_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls; allowing a fourth -statement admits the routes that authenticate and then hand off to a presenter, which have real work -behind them; allowing a fifth admits an admin route that calls a service and hand-rolls its own error -responses. - -The rule is deliberately reluctant, because a route wrongly called trivial is exempted and never -shows up in the report again. So `calleeNames` descends into the callee of every call at any depth -while `statementCount` stops at a nested function, which means the call count still catches bodies -the statement count reads as short. A builder means the config passed to it (`findResource`, -`authorization`) is work the scanner never walks, so the visible body is not the whole route. And a -try/catch is exactly what `error-classification` reads, so a body with one has an error path worth -reporting on however short it is. - -One rule, two views, so the entry-point-wide answer and a single export's answer cannot drift. The -per-export view exists because a per-export verdict judged against a file-wide triviality rule -accuses the wrong half of a file: `auth.github.ts` is -`export let loader = () => redirect("/login")` beside an action that calls -`authenticator.authenticate`, and the file-wide rule called it non-trivial because the ACTION is not, -so `auth-boundary` accused a one-line redirect stub of missing an auth guard. `checks/index.test.ts` -pins both directions (`reports not-applicable for a redirect-stub loader beside a guarded action` and -`fails an export whose own body does real work unguarded`). - -The two views differ in one term and the difference was measured both ways. The entry-point-wide view -matches the side-effect hints against the whole file, so an import of `prisma` disqualifies it even -when the query sits somewhere the scanner does not walk. A per-export view matches that export's own -callee PATHS instead. Reading the file's text into one export's verdict is the per-file-for-per-export -substitution the rule exists to damp, and it is defeatable: `log-caller-scope-userid` prepends a -`logger.error(...)` to every body, which with the term file-wide put the word `logger` in -`auth.github.ts` and turned its untouched one-line redirect loader from excused into accused. Emptying -the term instead is not the answer either: `calleeNames` keeps only a call's last segment, so -`prisma.orgMember.findMany` reads as `findMany` and a three-statement body that queries the datastore -matches no hint at all, which took five existing `auth-boundary` fixtures from `fail` to -`not-applicable`. The callee paths are body-scoped like the first option wants and name the receiver -like the second needs. - -## Reading the directive out of the source - -The suppression directive is read from a real parsed `ts.SourceFile`, and then filtered against the -spans the parser has already claimed as content. Both halves are needed. - -Parsing rather than scanning is what stops a template literal with a substitution being rescanned as -ordinary code after `${x}`, and what makes JSX text a node at all. Filtering by span is what stops -the two comment-range lexers reading the start of such a node as a comment anyway, which they do -because `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw lexers over source text from -an offset and consult no parse tree. A JSX text node that BEGINS with `//` or `/*` is the shape that -reached the real tree, in `resources.branches.create.tsx`'s `//`. - -The filter is on the range's start offset falling inside a claimed span, not on the gap between a -token's full start and its start. A gap filter was tried and rejected: it loses a same-line trailing -comment and a comment inside a JSX expression container, both of which are real. Both lexers are -called at every token boundary, because which one returns a given comment depends on whether it -shares a line with the token before it. - -Leaf tokens are walked through `.getChildren()` rather than `ts.forEachChild`, which skips bare -punctuation and keyword tokens. A comment can sit directly before one of those with nothing else -following it, the last line inside a block. - -The mutation corpus does not cover any of this and cannot: a suppression can only lower an entry's -score, because `scoreEntry` caps it at the pre-suppression ratio, so suppression bugs are invisible -to a harness that watches for the score rising. They need ordinary unit tests, which is what -`suppression.test.ts` is: `jsx text is content, not a comment` is the four cases that fail without -the JSX filter, and the positive control beside it, `still reads a directive from a comment in a JSX -expression container`, is what stops the filter being widened until it eats real comments. - -## The mutation harness - -Every mutation is a TEXT rewrite driven by AST positions, never a reprint. A reprint would change -formatting everywhere and make a failure impossible to read; splicing at node positions leaves the -rest of the file byte-identical, so a corpus failure can be diffed down to the one construct that -moved. Overlapping edits are dropped inner-first, which is what "the outer rewrite won" means. - -Neither kind of entry is ever executed. Semantics-preserving here means preserving the observable -behaviour of the route as written, which is what the scanner claims to measure. It is not a claim -that the mutated tree compiles against its real types. - -**Splices go at the HEAD of a catch clause, not the tail.** 234 of the tree's 260 clauses end in a -`return` or a `throw`, so an appended shape was dead by ordering before the rule under test ever -looked at it: eleven entries reported touching 172 files while exercising 26 clauses. At the head -every clause is reachable. The shapes spliced this way are dead wherever they sit, so moving them -does not make the rewrite any less preserving. - -**The harness's population is asserted against the scanner's.** `entryBodies` read two of the four -export forms `scan.ts` reads, missing `export const { action, loader } = builder(...)`, -`const { action } = builder(...); export { action };` and `export const action = route.action`, which -is 36 of the tree's entry points. No assertion could notice, because a mutation reaching fewer routes -lowers the score rather than raising it. `wraps a body in every non-delegating entry point the scanner -finds` pins it now, with `admin.tsx` the one named exclusion: its handler is a concise arrow with no -block for a block wrapper to wrap. - -The same failure mode is why the registry assertion and the additive-class assertion are ungated -while everything else in the file needs `OBS_MAP_MUTATION_CORPUS=1`. `auth-scope` was added a round -after `suppress-every-check` was written and never added to its directive list, so the suppression -invariant went untested at tree scale for 19 routes while the entry's description said "every -check". Omitting a check from a sweep leaves its failures in place, which lowers the score, so the -corpus cannot catch its own omission by failing. - -**The corpus deliberately disagrees with the scanner about where a handler sits.** `mutations.ts` -keeps its own copy of the builder handler shapes rather than importing them, because sharing the -scanner's notion would let a bug in that notion hide a laundering shape. Which exports exist is not -a judgement, though, and there the harness was simply behind, which is the distinction above. - -**The anti-vacuity threshold is on sites, not only files.** A file count says a rewrite touched a -file, not that it reached anything inside it. The guard the design asked for, verdict movement, -cannot be used, and not for the reason an earlier note gave: plenty of defended entries move verdicts -hard (`delete-every-catch` takes the tree from 19 to 8), but the IDEAL defended shape is one the -scanner is blind to, and `dead-if-false` and the ten entries beside it are defended precisely because -the tree comes out identical. Requiring movement would fail exactly the entries that work best. - -**A `lowers` exemption is a per-entry field with a reason, not a skip list.** An exempted entry must -still be falling, or the exemption is stale, and its falls must have exactly the measured residual -shape it was granted for: `error-classification` moving pass to not-applicable, every other check -unchanged, nothing moving to fail. Exactly two entries carry one, both non-array-receiver iteration -wrappers. - -**A `KNOWN_GAPS` entry runs as `it.fails`,** so closing the hole later turns the file red until the -entry is moved out deliberately. Two are open: `dead-classifying-try-with-call`, the shape -`dead-classifying-try` only looked like it closed, and `dead-conjunction-instanceof-if`, the sibling -the arm-liveness fix does not close. Both are described above. -`dead-branch-after-if-true` used to be listed on a measurement that was wrong; raising the exit flag -after each statement's branch check rather than before is byte-identical on the real tree and closes -the shape, so the `if (true)` family needed no condition folding after all. - -## Reporting - -The score's own arithmetic has three rules that the report is built to keep honest. - -Every denominator reads `rawChecks`, pre-suppression. `checks` is the display view. Suppressing the -one `request-context` or `audit-trail` finding on an entry must not shrink the gap denominators and -raise the printed percentage, on the same screen as a claim that suppression cannot do that. An -entry's score is capped by what it would have scored unsuppressed, which is how 33 became 50 became -100 before the cap existed. - -`score` is 100 for an entry no scored check applied to, and that is a placeholder rather than a -verdict. Rendering it as a figure turned a route refactored down to a trivial body into a 67-point -improvement, and a trivial route gaining real work into the PR's worst regression, so the PR -comment's cell says "not measured" instead. `globalWithout` recomputes from `rawChecks` minus the -suppression cap, because lowering both figures by the same rule would leave the difference between -them saying something about suppressions rather than about the check. - -`hasDelta` has to be true whenever `renderPrComment` would say something different, because anything -it misses is a change the pull request silently does not report. So it covers every figure the -comment renders: the global, the per-entry score, measured state and suppression set, an entry added -or removed, a check failing at head that did not at base, the parse failure count, the unknown -suppression warnings, the audit and context gaps, `delegating` and `checkContributions`. The -per-entry suppression set and the two gaps are the half that was missing, and it ran the dangerous -way: suppressing an already-failing check moves no score, no measured flag and no new failure, so a -pull request whose entire purpose was silencing findings posted nothing. What is defended is that the -union of the terms is complete rather than that each term is load bearing. Four are individually -reachable with a test each; the global, the removed-entry check and the per-entry score are shadowed -by another term today and are kept because which term shadows which depends on the shape of the -change. `MapReport.suppressions` is the one term deliberately left out, because its totals are summed -from the very per-entry arrays the loop compares. - -Two sections of the PR comment grow with the tree and both are capped, because GitHub's comment limit -is 65,536 characters and a 422 loses the whole comment to the section warning about a typo. A -mistyped directive applied tree wide rendered 87,938 characters. The delegated list is capped at -fifteen rather than ten because a file name is one comma-separated item rather than a line naming -every known check, and the longest route file name in the tree is 130 characters. - -The `AUDIT` line has one shape for every count and no branch on the count, because the branch carried -the bug: a zero used to print "No audit helper exists in the webapp", which is false. The count was -already correct, so the sentence was the only wrong thing. - -A suppression whose id names no check is carried through to both renderers rather than dropped, -because dropping it silently is what made a typo look like an acknowledgement. - -## Tests, timeouts and CI - -`docstringReferences.test.ts` enforces that every test name a docstring in `src/` claims to be -covered by exists. The rule was asked for six times in prose and broken six times, most recently by a -docstring naming a test that was never written, so prose does not enforce itself. What is checked: -every backticked kebab-case token, every backticked glob against the corpus ids by prefix, and every -backticked prose phrase of five words or more with no code punctuation. What is not: a reference -written without backticks, a title of fewer than five words, a comment with no node after it -(leading ranges only, so a comment on the last line of a block is never scanned), and a `.test.ts` -file or `mutations.ts`, both exempted by name. The kebab half is the half that has actually failed. -Three negative controls run the same predicates over an invented docstring, so the guarantee does not -rest on `src/` currently happening to contain a bad reference. - -The real-tree tests are gated by `TREE_SCAN_TIMEOUT`, which is a hang detector and nothing else. -Neither test asserts anything about how long a scan takes, so a number tight enough to be a -performance budget would only be a way to fail on a busy runner, and a performance budget that flakes -gets the whole suite marked unreliable. The old 30s was chosen on an idle machine and does flake. -Measured on an 8-core box, this file alone at load average 0.9: 6.3s to 6.4s for the scan, 10.8s to -11.2s for the sweep. Twenty-four runs as two batches of twelve concurrent copies on those same 8 -cores: 24.2s to 34.0s for the scan and 27.6s to 39.7s for the sweep, with one of the first twelve -dying on a 30s timeout. That contention is not hypothetical: `unit-tests-internal.yml` runs twelve -concurrent shard processes on one runner. The local reproduction is harsher than CI on purpose, -twelve processes over 8 cores against the 32-vCPU runner's 0.375 per core, so 120s is 3x the worst -contended run measured. 60s was the other candidate and is not enough on those numbers. - -Parse failures come from a `ts.Program`, not from the diagnostics array the parser hangs on the -source file, which is internal and which the compiler is free to rename. An undetected parse failure -shrinks the denominator and inflates the score, so it must not be the kind of thing a compiler -upgrade can switch off silently. The host hands the program the source file we already have, so -nothing is parsed twice; the cost is the program machinery, and a full scan of the real tree went -from about 850ms to about 1450ms over five runs of each. - -The suite's turbo task is uncacheable. Its real inputs are mostly not its own files, they are -`apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and the workflow files, so -turbo replayed a pass recorded before a route changed: a route file with a syntax error in it fails -under vitest and came back FULL TURBO in 301ms with the failure cached as a success. `inputs` naming -`../../apps/webapp/...` does bust the cache but replaces turbo 1.x's default file set instead of -adding to it, which drops the package's own files from the hash. The reasoning lives in `turbo.json` -beside the config it explains. - -Three roads reach this suite and all three are asserted. `pr_checks.yml` calls -`unit-tests-observability-map.yml` behind an `obsmap` paths filter and lists it in the `all-checks` -aggregate, without which a test job gates nothing: the first attempt put the job inside -`observability-map.yml`, which reads well and gates nothing, because `all-checks` needs an explicit -list of jobs and cannot see another workflow. The filter watches all of `apps/webapp/app` plus the -report workflow, because the suite reads more than the routes folder and a rename outside it matched -only `webapp`, ran no job, and broke the build for whoever pushed next. It deliberately does NOT name -this package or the two non-webapp roots: `internal` already matches `internal-packages/**` and -`packages/**` and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, so naming -them here ran the suite twice on every PR touching the package. Widening `internal` to the route -paths instead was tried and rejected, since it runs all eighteen internal packages with postgres, -clickhouse, redis and electric to protect one test. - -The report workflow's own text is asserted from `integration.test.ts`, because it is the one thing the -docstring checker cannot reach and the C1 defect was exactly that: two steps disagreeing about what a -missing comment id meant, under a comment claiming they agreed. The render step read it as "a comment -exists" and emitted the resolved state, the upsert step read it as "no id" and POSTed, so a transient -lookup failure either added a second marker comment beside the stale one or announced findings were -gone on a pull request that never had any. The sentinel pair those two shared is gone: the lookup -moved into the cheap `changes` job so the report job's gate could read it, the report job does not -start unless the lookup finished cleanly, and the id both steps use is one job output. Those are text -checks over the workflow rather than a parse of its semantics, so they catch the wiring coming apart -and nothing about whether GitHub agrees. - -Both scan steps write their own file through `--out` rather than capturing stdout with a shell -redirect. `pnpm --filter` takes its recursive path and some versions announce -`Scope: N of M workspace projects` on it; a single line of that in head.json fails the renderer's -`JSON.parse` and the workflow degrades to the stale-report comment on every run, quietly and -permanently. It does not reproduce on the pinned 10.33.2, which was checked, and what is asserted is -the shape that cannot have the bug rather than the version that happens not to. The render step has -no `--out` to reach for, and a banner there puts a stray line in a markdown comment instead of -breaking a parse, so it is left alone. - -The corpus runs on the package's own paths and on a schedule rather than on every route pull request. -It measures the tool's resistance to laundering, which only an edit to the tool can weaken, and it -costs four and a half minutes. The nightly is the other half of that trade: dropping the schedule -would leave tree drift uncovered rather than covered late. - ## Layout `scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts index 546f19e6860..ee5b014a40d 100644 --- a/internal-packages/observability-map/src/checks/authBoundary.ts +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -11,8 +11,9 @@ const ID = "auth-boundary"; * A guard the route only imports and never calls does not count, and neither does one the OTHER * export calls. * - * Names rather than the three patterns it replaces, all of which over-matched: README, "Sensitivity, - * and the names the tool matches on". `webappSymbols.test.ts` fails if a name stops resolving, but + * Names rather than the three patterns it replaces, all of which over-matched: INTERNALS.md, + * "Sensitivity, and the names the tool matches on". `webappSymbols.test.ts` fails if a name stops + * resolving, but * cannot check that a declaration with the right name is the guard we meant, which is the residual * the two local helpers below carry. */ diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts index ecb6d889fc4..c4d42d16f61 100644 --- a/internal-packages/observability-map/src/checks/authScope.ts +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -28,8 +28,8 @@ function builderExports(ep: EntryPoint): BuilderExport[] { /** * Whether a route the builder authenticated is also narrowed to the caller. The IDOR class it - * measures, the two ways to be scoped, and why `ability.can(...)` is deliberately not a third: - * README, "Authenticated is not the same as scoped". + * measures: README, "The five checks". The two ways to be scoped, and why `ability.can(...)` is + * deliberately not a third: INTERNALS.md, "What auth-scope reads as scoping". * * There is no triviality test here because `isTrivial` answers false for any route with an * initializer callee, and no delegating test because `scoreEntry` answers for every check before any diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts index 22044700b57..8a6dcb30cce 100644 --- a/internal-packages/observability-map/src/checks/errorClassification.ts +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -31,7 +31,7 @@ export const BUILDERS = new Set([ * An absolute count and not a ratio against the enclosing body, or padding the body relabels the * same broad swallow as a narrow guard (`inert-statements-after-try`). The count is NOT unpaddable, * which an earlier docstring claimed: it is one condition of three and no longer the load-bearing - * one. See README, "Parse guards, and the narrow-try count". + * one. See INTERNALS.md, "Parse guards, and the narrow-try count". */ const NARROW_TRY_STATEMENTS = 2; @@ -88,7 +88,7 @@ function swallows(clause: CatchEvidence): boolean { * and is defeated by one inert call, so `try { String(0); }` reads as classification and takes the * tree from 19 to 44. `dead-classifying-try-with-call` in the mutation corpus is that shape, * running as an expected failure. Read the rule as "refuses `try { 0; }`", never as "an unreachable - * catch cannot be credited". Everything else here: README, "The dead-code defence". + * catch cannot be credited". Everything else here: INTERNALS.md, "The dead-code defence". */ export const errorClassification = { id: ID, diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts index da3f1aa2014..f0e0087b28a 100644 --- a/internal-packages/observability-map/src/checks/requestContext.ts +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -32,7 +32,7 @@ function logLevel(callee: string): string { * is the whole point rather than an oversight: its failures go to the global handler, which names no * tenant. Passing those routes, as this check used to, meant deleting every catch clause in the tree * scored it 100, and excusing them as not-applicable is the same mistake in quieter clothes. What the - * platform attaches centrally and why none of it is a tenant: README, "What 19 means". + * platform attaches centrally and why none of it is a tenant: README, "What the score means". */ export const requestContext = { id: ID, diff --git a/internal-packages/observability-map/src/docstringReferences.test.ts b/internal-packages/observability-map/src/docstringReferences.test.ts index c8399e3f561..40385047798 100644 --- a/internal-packages/observability-map/src/docstringReferences.test.ts +++ b/internal-packages/observability-map/src/docstringReferences.test.ts @@ -7,7 +7,7 @@ import { MUTATIONS } from "./mutations.js"; /** * Every test name a docstring in `src/` claims to be covered by must exist. The rule was asked for six * times in prose and broken six times, so prose does not enforce itself. Exactly what is and is not - * checked, and the coverage holes that leaves: README, "Tests, timeouts and CI". + * checked, and the coverage holes that leaves: INTERNALS.md, "Tests, timeouts and CI". */ const SRC = resolve(__dirname); diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index 0f708578b33..ae9b93a4eeb 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -21,7 +21,7 @@ import { SCORED_CHECK_IDS } from "./checks/index.js"; * The coupling is acceptable because nothing here names a route or a count: the scan must not crash, * the entry point count must sit inside a wide band, and parse failures must be zero. Those are the * only things a fixture tree cannot tell us, since a fixture only contains shapes somebody thought to - * write down. How the whole suite is gated in CI: README, "Tests, timeouts and CI". + * write down. How the whole suite is gated in CI: INTERNALS.md, "Tests, timeouts and CI". */ const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); @@ -93,7 +93,8 @@ const steps = (block: string) => block.split(/^ {6}- name: /m).slice(1); /** * The one thing the docstring checker cannot reach, since it walks `src/` only. What the C1 defect was - * and what replaced it: README, "Tests, timeouts and CI". These are text checks over the workflow + * and what replaced it: INTERNALS.md, "Tests, timeouts and CI". These are text checks over the + * workflow * rather than a parse of its semantics, so they catch the wiring coming apart and nothing about * whether GitHub agrees. */ @@ -176,7 +177,7 @@ describe("the report workflow reconciles a comment the paths no longer reach", ( /** * Asserts the shape that cannot have the stdout-capture bug rather than the pnpm version that happens - * not to. Why, and why the render step is left alone: README, "Tests, timeouts and CI". + * not to. Why, and why the render step is left alone: INTERNALS.md, "Tests, timeouts and CI". */ describe("the report workflow's two scan steps", () => { it("let the scanner write its own report rather than capturing stdout", () => { @@ -288,7 +289,7 @@ describe("the package's tests are wired into the gate", () => { /** * The third road into this suite, `turbo run test`. Asserts the task is uncacheable, because the * config is one line and reads like a performance oversight to anyone who does not know what the suite - * reads. Measurement and the rejected `inputs` alternative: README, "Tests, timeouts and CI". + * reads. Measurement and the rejected `inputs` alternative: INTERNALS.md, "Tests, timeouts and CI". */ describe("the third road in, turbo", () => { it("keeps its test task out of the turbo cache", () => { @@ -322,7 +323,7 @@ describe("counting candidates independently of the scanner", () => { * Timeout for the two real-tree tests, which do not fit the suite's 10s default. A hang detector and * nothing else: neither test asserts anything about how long a scan takes, so a number tight enough to * be a performance budget would only be a way to fail on a busy runner. The contention measurements - * behind 120s, and why 60s is not enough: README, "Tests, timeouts and CI". + * behind 120s, and why 60s is not enough: INTERNALS.md, "Tests, timeouts and CI". */ const TREE_SCAN_TIMEOUT = 120_000; diff --git a/internal-packages/observability-map/src/mutationCorpus.test.ts b/internal-packages/observability-map/src/mutationCorpus.test.ts index 42bee6f733b..d8a51bcd3db 100644 --- a/internal-packages/observability-map/src/mutationCorpus.test.ts +++ b/internal-packages/observability-map/src/mutationCorpus.test.ts @@ -23,7 +23,7 @@ const ENABLED = process.env.OBS_MAP_MUTATION_CORPUS === "1"; /** * Where a corpus entry goes when the tool does not defend it. `it.fails` keeps the entry running, so * closing the hole later turns this file red until the entry is moved back out deliberately. Both - * gaps are described at length in README, "The mutation harness". + * gaps are described at length in INTERNALS.md, "The mutation harness". */ const KNOWN_GAPS = new Set([ // `canRaise` accepts any call at all, so `try { String(0); }` reads as a clause guarding real work @@ -201,7 +201,8 @@ function mutate( * Deliberately NOT gated behind `OBS_MAP_MUTATION_CORPUS`, unlike everything below it: the corpus * cannot catch its own omission by failing, since omitting a check from the sweep lowers the score * rather than raising it. Belongs in the default suite so adding a check without extending the corpus - * turns `pnpm test` red rather than a job nobody runs locally. See README, "The mutation harness". + * turns `pnpm test` red rather than a job nobody runs locally. See INTERNALS.md, "The mutation + * harness". */ describe("the corpus keeps up with the check registry", () => { it("suppresses every registered check in the exhaustive sweep", () => { @@ -303,7 +304,7 @@ describeCorpus("mutation corpus over the real route tree", { timeout: ENTRY_TIME /** * How much a mutation must reach before its result means anything, since one that silently matched * nothing would otherwise pass by leaving the tree alone. On sites and not only files, and why - * verdict movement cannot be the guard instead: README, "The mutation harness". + * verdict movement cannot be the guard instead: INTERNALS.md, "The mutation harness". */ const MINIMUM_FILES_TOUCHED = 20; const MINIMUM_SITES_TOUCHED = 40; diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index 7c5091409b0..092c3cd8ba9 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -3,7 +3,7 @@ import ts from "typescript"; /** * Source-to-source mutations for the tree-scale corpus in `mutationCorpus.test.ts`. Why they are text * rewrites rather than reprints, what `preserving` and `deleting` mean, and why the additive direction - * is tracked separately in `ADDITIVE_IDS`: README, "The mutation harness". + * is tracked separately in `ADDITIVE_IDS`: INTERNALS.md, "The mutation harness". */ export type MutationKind = "preserving" | "deleting"; @@ -306,7 +306,7 @@ function bindingNameOf(clause: ts.CatchClause): string | null { /** * Splice a statement in at the HEAD of every catch clause that names its binding, which is the whole * point of the helper: 234 of the tree's 260 clauses end in a `return` or a `throw`, so an appended - * shape was dead by ordering before the rule under test looked at it. See README, "The mutation + * shape was dead by ordering before the rule under test looked at it. See INTERNALS.md, "The mutation * harness". */ function prependToEveryCatch( diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 214a11f2fd2..03c0f566852 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -213,7 +213,7 @@ const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) * misses is a change the pull request silently does not report. The terms overlap on purpose, and * what is defended is that their union is complete rather than that each one is load bearing. * `MapReport.suppressions` is the one term deliberately left out, because its totals are summed from - * the very per-entry arrays the loop below compares. See README, "Reporting". + * the very per-entry arrays the loop below compares. See INTERNALS.md, "Reporting". */ export function hasDelta(head: MapReport, base: MapReport | null): boolean { if (!base) return true; diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index e58813953e9..d6d58c390a3 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -92,7 +92,7 @@ const CALLER_ID_FIELD = /** * Callees handed the caller's id that cannot narrow a read with it: the log line and the response * body. A denylist of sinks rather than an allowlist of query callees, which is a measurement and not - * a preference. See README, "What auth-scope reads as scoping". + * a preference. See INTERNALS.md, "What auth-scope reads as scoping". */ const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; @@ -135,7 +135,7 @@ function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { /** * Whether any handler in `fns` assigns the caller's own id to an object-literal property. Three - * conditions, all load bearing: README, "What auth-scope reads as scoping". + * conditions, all load bearing: INTERNALS.md, "What auth-scope reads as scoping". * * Per export rather than per entry point, which is why it is computed here rather than in the main * body walk. Nested functions are walked, since a filter built inside a callback still filters. @@ -272,7 +272,7 @@ function canRaise(node: ts.Node): boolean { /** * What the guarded region does, in the three terms `error-classification` needs. Each term's rule - * and measurement: README, "Catch evidence, per clause". + * and measurement: INTERNALS.md, "Catch evidence, per clause". * * Two residuals in opposite directions, both live. `guardCanRaise` refuses `try { 0; }` and nothing * cleverer, because `canRaise` accepts any call at all, so `try { String(0); }` reads as @@ -577,7 +577,7 @@ function tryBlockMayThrow(block: ts.Block): boolean { /** * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch does * not already rule out. Strictly subtractive against a plain containment read, which is what lets its - * two callers read it for opposite purposes: see README, "Two folds, pointing opposite ways". + * two callers read it for opposite purposes: see INTERNALS.md, "Two folds, pointing opposite ways". * * The `exited` half is pinned by `dead and deferred code prepended to a deciding catch does not blind * it` and the `BRANCH_EXITED` family; the `selectsADistinctPath` half by `an arm whose only exit is @@ -695,8 +695,9 @@ function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): b * Both answers are read off the clause's own guaranteed path: the walk enters a construct exactly * where the entered statements are guaranteed to execute whenever the clause body runs, so no credit * can ever come from code a semantics-preserving edit could have added dead. Which constructs are - * entered and which are refused, and the eleven dead spellings this replaced: README, "The dead-code - * defence". `dead-*` in the mutation corpus is the tree-scale proof, one entry per shape. + * entered and which are refused, and the eleven dead spellings this replaced: INTERNALS.md, + * "The dead-code defence". `dead-*` in the mutation corpus is the tree-scale proof, one entry per + * shape. * * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer * reads as a rethrow, so it fails rather than sitting out. That is the direction to be wrong in. @@ -896,7 +897,8 @@ function isAtMostSingletonArray(expr: ts.Expression): boolean { /** * Whether the function-like `node` is the callback argument of a per-item iteration. Both directions - * of being wrong, and what makes the name list survivable: README, "The iteration-callback boundary". + * of being wrong, and what makes the name list survivable: INTERNALS.md, "The iteration-callback + * boundary". * * The residual a reader here needs: a per-item callback under a callee this list does not know, * `pMap(items, cb)`, is attributed to the route, so a per-element catch that decides can carry it to @@ -1266,7 +1268,7 @@ const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off the * internal diagnostics array the parser hangs on the source file, which a compiler upgrade could * rename out from under us. The host hands the program the `sf` we already have, so nothing is parsed - * twice. Costs and reasoning: README, "Tests, timeouts and CI". + * twice. Costs and reasoning: INTERNALS.md, "Tests, timeouts and CI". */ function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { const host: ts.CompilerHost = { @@ -1437,7 +1439,7 @@ export function scanFile(fileName: string, source: string): EntryPoint | null { // since nesting deeper inside one is still inside it. `calleeNames`, `logCalls` and the statement // count keep descending regardless; a catch does not, and is kept in `callbackCatches` with its // evidence rather than dropped. Only an iteration callback is a boundary, not every function-like - // node. See README, "The iteration-callback boundary". + // node. See INTERNALS.md, "The iteration-callback boundary". const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { if (ts.isFunctionLike(node)) { if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts index c9981c46b55..5ef6582c849 100644 --- a/internal-packages/observability-map/src/sensitivity.ts +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -5,7 +5,8 @@ import { routePathOf } from "./adapters/remix.js"; * Symbols whose presence says the route does something risky: minting or revoking a credential, * escalating to another user, destroying a tenant. Calling a guard is never one of them, because a * mitigation cannot be the hazard, and `webappSymbols.test.ts` fails if a name stops resolving in the - * webapp. Both rules and what they cost: README, "Sensitivity, and the names the tool matches on". + * webapp. Both rules and what they cost: INTERNALS.md, "Sensitivity, and the names the tool + * matches on". */ export const SENSITIVE_SYMBOLS = [ // Escalation: acting as another user. diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts index 9d23e8021cf..ee788a3c5e0 100644 --- a/internal-packages/observability-map/src/suppression.ts +++ b/internal-packages/observability-map/src/suppression.ts @@ -31,7 +31,7 @@ function leafTokens(node: ts.Node): ts.Node[] { * `ignores the directive inside a string literal`. * * The mutation corpus cannot cover any of this, because a suppression can only lower a score. See - * README, "Reading the directive out of the source". + * INTERNALS.md, "Reading the directive out of the source". */ function isClaimedContent(node: ts.Node): boolean { return ( @@ -49,7 +49,8 @@ function isClaimedContent(node: ts.Node): boolean { * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a standalone * `ts.createScanner`, and then filtered against the spans above. Both halves are needed, and the * filter is on the range's start offset rather than on the gap between a token's full start and its - * start: README, "Reading the directive out of the source". Both lexers are called at every token + * start: INTERNALS.md, "Reading the directive out of the source". Both lexers are called at every + * token * boundary, because which one returns a given comment depends on whether it shares a line with the * token before it. */ diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts index 377c1165dec..5868558c3cb 100644 --- a/internal-packages/observability-map/src/triviality.ts +++ b/internal-packages/observability-map/src/triviality.ts @@ -18,7 +18,7 @@ const MAX_STATEMENTS = 3; /** * What the rule reads, so the entry-point-wide answer and a single export's answer are the same rule * over different bodies rather than two rules that can drift. Both limits, the reluctance and the - * measured `hintText` decision: README, "Triviality, in detail". + * measured `hintText` decision: INTERNALS.md, "Triviality, in detail". */ type TrivialityView = { statementCount: number; diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts index 5d1c93ffaec..bee1aae1c44 100644 --- a/internal-packages/observability-map/src/types.ts +++ b/internal-packages/observability-map/src/types.ts @@ -9,7 +9,7 @@ export type CheckResult = { /** * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays - * legible. Every field's exact rule and its measured reasoning: README, "Catch evidence, per + * legible. Every field's exact rule and its measured reasoning: INTERNALS.md, "Catch evidence, per * clause". */ export type CatchEvidence = { @@ -54,7 +54,7 @@ export type LogCall = { /** * Body-scoped evidence for one route module. Which fields are per export and why, and what "the - * body" means: README, "How the scanner reads a route". + * body" means: INTERNALS.md, "How the scanner reads a route". */ export type EntryPoint = { fileName: string; From 03138843e534e2dcbd0559742c06f2ac5461a305 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 15:52:16 +0100 Subject: [PATCH 112/117] fix(observability-map): stop preserving mutations deleting suppression comments --- .../src/mutationTrivia.test.ts | 81 +++++++++++++++++++ .../observability-map/src/mutations.ts | 14 ++++ 2 files changed, 95 insertions(+) create mode 100644 internal-packages/observability-map/src/mutationTrivia.test.ts diff --git a/internal-packages/observability-map/src/mutationTrivia.test.ts b/internal-packages/observability-map/src/mutationTrivia.test.ts new file mode 100644 index 00000000000..7f0d0eb6fda --- /dev/null +++ b/internal-packages/observability-map/src/mutationTrivia.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { MUTATIONS } from "./mutations.js"; +import { parseSuppressions } from "./suppression.js"; + +/** + * A suppression is file-scoped and can only lower a score, so a rewrite that drops one raises the + * file back to its unsuppressed ratio. A `preserving` entry that does that breaks the corpus + * assertion it is measured under, and no route in the real tree carries a directive, so the corpus + * cannot see it. These fixtures put a directive in each position a rewrite joins across. + */ +const FIXTURES: { name: string; fileName: string; source: string }[] = [ + { + name: "between two const declarations in a block", + fileName: "consts.ts", + source: [ + "export async function action() {", + " const a = compute();", + " // obs-map-disable error-classification -- fixture", + " const b = compute();", + " return a + b;", + "}", + "", + ].join("\n"), + }, + { + name: "between two expression statements in a block", + fileName: "statements.ts", + source: [ + "export async function action() {", + " first();", + " // obs-map-disable request-context -- fixture", + " second();", + "}", + "", + ].join("\n"), + }, + { + name: "between two expression statements at file scope", + fileName: "toplevel.ts", + source: ["first();", "// obs-map-disable audit-trail -- fixture", "second();", ""].join("\n"), + }, + { + name: "a block comment between two const declarations", + fileName: "block-comment.ts", + source: [ + "export async function action() {", + " const a = compute();", + " /* obs-map-disable auth-boundary -- fixture */", + " const b = compute();", + " return a + b;", + "}", + "", + ].join("\n"), + }, +]; + +const PRESERVING = MUTATIONS.filter((m) => m.kind === "preserving"); + +describe("preserving mutations keep every suppression directive", () => { + it("has preserving entries to check", () => { + expect(PRESERVING.length).toBeGreaterThan(0); + }); + + for (const mutation of PRESERVING) { + for (const fixture of FIXTURES) { + it(`${mutation.id} keeps the directive ${fixture.name}`, () => { + const before = parseSuppressions(fixture.source, fixture.fileName); + expect(before.byId.size).toBe(1); + + const result = mutation.apply(fixture.fileName, fixture.source); + if (result === null) return; + + // Superset, not equality: `suppress-every-check` adds a directive for every check on + // purpose, and adding one can only lower a score. Losing one is the direction that raises. + const after = parseSuppressions(result.source, fixture.fileName); + const lost = [...before.byId.keys()].filter((id) => !after.byId.has(id)); + expect(lost).toEqual([]); + }); + } + } +}); diff --git a/internal-packages/observability-map/src/mutations.ts b/internal-packages/observability-map/src/mutations.ts index 092c3cd8ba9..f97db9f413d 100644 --- a/internal-packages/observability-map/src/mutations.ts +++ b/internal-packages/observability-map/src/mutations.ts @@ -954,6 +954,7 @@ export const MUTATIONS: Mutation[] = [ if (!isSingleConst(previous) || !isSingleConst(current)) continue; if (source[previous.end - 1] !== ";") continue; const declaration = current.declarationList.declarations[0]!; + if (spansAComment(source, previous.end - 1, declaration.getStart())) continue; edits.push({ start: previous.end - 1, end: declaration.getStart(), text: ", " }); } }); @@ -979,6 +980,7 @@ export const MUTATIONS: Mutation[] = [ // claims not to make. if (ts.isStringLiteral(previous.expression)) continue; if (source[previous.end - 1] !== ";") continue; + if (spansAComment(source, previous.end - 1, current.getStart())) continue; edits.push({ start: previous.end - 1, end: current.getStart(), text: ", " }); } }); @@ -1039,6 +1041,18 @@ export const ADDITIVE_IDS = [ "log-caller-scope-userid", ]; +/** + * Whether a comment sits in the span a statement merge replaces. A suppression is file-scoped and can + * only lower a score, so joining two statements across a directive raises the file back to its + * unsuppressed ratio, which is the one direction a `preserving` entry may not move. The span a merge + * replaces holds a semicolon, trivia and at most a `const` keyword, never a string literal, so a + * textual scan cannot be fooled by `//` inside quotes. Covered by `mutationTrivia.test.ts`. + */ +function spansAComment(source: string, start: number, end: number): boolean { + const between = source.slice(start, end); + return between.includes("//") || between.includes("/*"); +} + function isSingleConst(statement: ts.Statement): statement is ts.VariableStatement { return ( ts.isVariableStatement(statement) && From eb01d0a4906a8cd747ea0a475829802a540bb1c9 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 15:52:39 +0100 Subject: [PATCH 113/117] fix(observability-map): enumerate route modules in name order --- .../observability-map/src/scan.test.ts | 26 ++++++++++++++++++- .../observability-map/src/scan.ts | 7 ++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts index 6688fdb26d8..d65d1398a3f 100644 --- a/internal-packages/observability-map/src/scan.test.ts +++ b/internal-packages/observability-map/src/scan.test.ts @@ -1,7 +1,7 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { ParseFailureError, scanDirectory, scanFile } from "./scan.js"; +import { ParseFailureError, routeModuleFiles, scanDirectory, scanFile } from "./scan.js"; const LOADER = ` import { json } from "@remix-run/server-runtime"; @@ -2957,3 +2957,27 @@ describe("scanFile: callees whose answer the body read", () => { expect(ep!.loaderCheckedCallees).not.toContain("getUser"); }); }); + +/** + * Contract only. Enumeration order is not controllable from a test: ext4 returns name-hash order, so + * every name set constructed here already comes back sorted and the assertion cannot be made to fail + * locally. It still holds the contract for a filesystem that enumerates in creation order. + */ +describe("routeModuleFiles", () => { + it("returns the tree in name order", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-order-")); + try { + for (const name of ["zeta.ts", "alpha.ts", "middle.ts"]) { + writeFileSync(join(dir, name), LOADER); + } + mkdirSync(join(dir, "beta")); + writeFileSync(join(dir, "beta", "route.ts"), LOADER); + + const names = routeModuleFiles(dir).map((f) => f.relativeName); + expect(names).toEqual([...names].sort()); + expect(names).toEqual(["alpha.ts", "beta/route.ts", "middle.ts", "zeta.ts"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts index d6d58c390a3..c80861f775e 100644 --- a/internal-packages/observability-map/src/scan.ts +++ b/internal-packages/observability-map/src/scan.ts @@ -1611,7 +1611,12 @@ export function routeModuleFiles(dir: string): RouteModuleFile[] { if (!entry.isFile() || !isScannableFile(entry.name)) continue; files.push({ absolutePath: join(dir, entry.name), relativeName: entry.name }); } - return files; + // `readdirSync` order is filesystem-defined, and it reaches the report: head and base are scanned + // from two different directories, so an order difference alone would read as a delta and post a + // comment claiming a change no score made. + return files.sort((a, b) => + a.relativeName < b.relativeName ? -1 : a.relativeName > b.relativeName ? 1 : 0 + ); } export function scanDirectory(dir: string): { From ace53423b35270d5b298af6a029d4517ec790505 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 15:52:54 +0100 Subject: [PATCH 114/117] fix(observability-map): name the base score when the head is the unmeasured side --- .../src/report/prComment.test.ts | 19 +++++++++++++++++++ .../observability-map/src/report/prComment.ts | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/internal-packages/observability-map/src/report/prComment.test.ts b/internal-packages/observability-map/src/report/prComment.test.ts index 4d57654f79d..d6d26c2a2cc 100644 --- a/internal-packages/observability-map/src/report/prComment.test.ts +++ b/internal-packages/observability-map/src/report/prComment.test.ts @@ -130,6 +130,25 @@ describe("renderPrComment", () => { expect(row).not.toMatch(/\|\s*100\s*\|/); }); + it("names the base score when it is the head that has none, not the base", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + expect(head.global).toBeNull(); + expect(base.global).not.toBeNull(); + + const out = renderPrComment(head, base); + expect(out).toContain(`(base ${base.global})`); + expect(out).not.toContain("(base not measured)"); + }); + + it("still says the base is the missing one when it really is", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + expect(base.global).toBeNull(); + + expect(renderPrComment(head, base)).toContain("(base not measured)"); + }); + it("renders as not measured in the base column when the head gained real work", () => { const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts index 03c0f566852..beef3818f49 100644 --- a/internal-packages/observability-map/src/report/prComment.ts +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -56,7 +56,10 @@ function scoreLine(head: MapReport, base: MapReport | null): string { : `**${head.global}/100** over ${head.measured} measured of ${head.entries.length} entry points`; if (!base) return headline; - if (base.global === null || head.global === null) return `${headline} (base not measured)`; + // Head first: when this side has no score the headline already says so, and naming the base as the + // missing one sends the reader to the wrong commit. + if (head.global === null) return `${headline} (base ${base.global ?? "not measured"})`; + if (base.global === null) return `${headline} (base not measured)`; const diff = head.global - base.global; const comparison = From 82bb097baaf2816d7931602d4de3957d44a4ad11 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 15:53:00 +0100 Subject: [PATCH 115/117] fix(observability-map): survive a null comment body in the report lookup --- .github/workflows/observability-map.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index a2545ad1a63..702ff5a56a2 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -90,7 +90,7 @@ jobs: ok="" for attempt in 1 2 3; do if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select(.body | startswith(""))][0].id // empty'); then + --jq '[.[] | select((.body // "") | startswith(""))][0].id // empty'); then ok=1 break fi From b7cc5643ebd07afbd001e8b1c55ab44d344b2358 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 17:20:44 +0100 Subject: [PATCH 116/117] fix(observability-map): let the renderer write its own comment file --- .github/workflows/observability-map.yml | 10 ++++- .../observability-map/INTERNALS.md | 6 +++ .../observability-map/src/integration.test.ts | 13 ++++++- .../src/report/prCommentCli.test.ts | 39 ++++++++++++++++++- .../src/report/prCommentCli.ts | 27 +++++++++---- 5 files changed, 83 insertions(+), 12 deletions(-) diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml index 702ff5a56a2..3336492d5a5 100644 --- a/.github/workflows/observability-map.yml +++ b/.github/workflows/observability-map.yml @@ -300,15 +300,21 @@ jobs: COMPARE_URL: ${{ github.server_url }}/${{ github.repository }}/compare/${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} run: | rm -f /tmp/comment.md + # `--out` rather than a stdout redirect, for the reason the scan steps above give, and with a + # worse failure mode than theirs: the marker has to be the comment's first line for the + # lookup to find it, so a line printed ahead of the document makes every push post a new + # comment instead of updating the one already there. Held by + # `it("let the renderer write its own comment rather than capturing stdout")`. render() { pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts \ - --commit-sha="$HEAD_SHA" --commit-url="$COMPARE_URL" "$@" + --commit-sha="$HEAD_SHA" --commit-url="$COMPARE_URL" --out=/tmp/comment.md.partial "$@" } # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte # comment.md for the upsert to skip in silence. emit() { - if render "$@" > /tmp/comment.md.partial; then + rm -f /tmp/comment.md.partial + if render "$@"; then mv /tmp/comment.md.partial /tmp/comment.md return 0 fi diff --git a/internal-packages/observability-map/INTERNALS.md b/internal-packages/observability-map/INTERNALS.md index 270726522d6..c1dd52e2fbb 100644 --- a/internal-packages/observability-map/INTERNALS.md +++ b/internal-packages/observability-map/INTERNALS.md @@ -489,6 +489,12 @@ because `JSON.parse` and degrades the workflow to the stale-report comment permanently. What is asserted is the shape that cannot have the bug rather than the pinned 10.33.2 that happens not to. +The render step writes through `--out` for the same reason, and its failure mode is the worse of the +two. `renderPrComment` puts the marker on the first line and the lookup finds the comment with +`startswith` on it, so a line printed ahead of the document does not degrade the comment, it hides it: +the next push finds no id and posts a second comment, and no later run can reconcile either. The scan +steps degrade to a stale report, which at least stays one comment. + The corpus runs on the package's own paths and on a schedule rather than on every route pull request, because it measures the tool's resistance to laundering, which only an edit to the tool can weaken, and it costs four and a half minutes. The nightly covers tree drift late rather than not at all. diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts index ae9b93a4eeb..de2147505d6 100644 --- a/internal-packages/observability-map/src/integration.test.ts +++ b/internal-packages/observability-map/src/integration.test.ts @@ -177,9 +177,18 @@ describe("the report workflow reconciles a comment the paths no longer reach", ( /** * Asserts the shape that cannot have the stdout-capture bug rather than the pnpm version that happens - * not to. Why, and why the render step is left alone: INTERNALS.md, "Tests, timeouts and CI". + * not to. Why: INTERNALS.md, "Tests, timeouts and CI". */ -describe("the report workflow's two scan steps", () => { +describe("the report workflow's scan and render steps", () => { + it("let the renderer write its own comment rather than capturing stdout", () => { + const render = steps(read(REPORT)).find((step) => step.startsWith("📝 Render"))!; + expect(render).toBeDefined(); + expect(render).toMatch(/--out=\S+\.md\S*/); + // The marker has to be the comment's first line for the lookup to find it, so a redirect that + // could put a package-manager banner ahead of the document costs the upsert, not just tidiness. + expect(render).not.toMatch(/render[^\n]*>\s*\S*\.md/); + }); + it("let the scanner write its own report rather than capturing stdout", () => { const scans = steps(read(REPORT)).filter((step) => step.startsWith("🔎 Scan")); expect(scans).toHaveLength(2); diff --git a/internal-packages/observability-map/src/report/prCommentCli.test.ts b/internal-packages/observability-map/src/report/prCommentCli.test.ts index 6160fa11b9b..58bf07d6b78 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.test.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { main, type Io } from "./prCommentCli.js"; @@ -197,6 +197,43 @@ describe("prCommentCli", () => { expect(r.err).not.toContain(" at "); }); + describe("--out", () => { + it("writes the comment to the file and leaves stdout empty", () => { + const outPath = join(dir, "out-delta.md"); + const r = run(headPath, basePath, `--out=${outPath}`); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + + const written = readFileSync(outPath, "utf8"); + expect(written.split("\n")[0]).toBe(""); + }); + + it("writes an empty file when there is nothing to post, rather than no file", () => { + const outPath = join(dir, "out-nothing.md"); + const r = run(unchangedPath, unchangedPath, `--out=${outPath}`); + expect(r.code).toBe(0); + expect(existsSync(outPath)).toBe(true); + expect(readFileSync(outPath, "utf8")).toBe(""); + }); + + it("writes the resolved and scan-failed comments too", () => { + for (const mode of ["--resolved", "--scan-failed"]) { + const outPath = join(dir, `out${mode}.md`); + expect(run(mode, `--out=${outPath}`).code).toBe(0); + expect(readFileSync(outPath, "utf8").split("\n")[0]).toBe( + "" + ); + } + }); + + it("truncates a file left by an earlier run", () => { + const outPath = join(dir, "out-stale.md"); + writeFileSync(outPath, "stale content from a previous push\n"); + expect(run(unchangedPath, unchangedPath, `--out=${outPath}`).code).toBe(0); + expect(readFileSync(outPath, "utf8")).toBe(""); + }); + }); + it("exits 1 with a one-line message when base.json is malformed", () => { const malformedBasePath = join(dir, "malformed-base.json"); writeFileSync(malformedBasePath, "not json at all"); diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts index c4f8158f31c..e0ad53c9127 100644 --- a/internal-packages/observability-map/src/report/prCommentCli.ts +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { MapReport } from "../score.js"; @@ -72,6 +72,21 @@ export function main(argv: string[], io: Io = processIo): number { const headPath = positional[0]; const basePath = positional[1]; + /** + * The marker has to be the document's first line or the workflow's lookup cannot find the comment it + * left, and every later push posts a new one instead of updating it. `--out` keeps stdout free for + * whatever a tool decides to announce, the same reason `src/cli.ts` has it. An empty write is a real + * outcome and not a failure: it is how "post nothing" reaches the workflow's `-s` check. + */ + const outPath = flag(args, "out"); + const write = (text: string) => { + if (outPath === undefined) { + if (text) io.out(text); + return; + } + writeFileSync(outPath, text); + }; + let commit: CommitContext | undefined; try { commit = commitFrom(args); @@ -81,12 +96,12 @@ export function main(argv: string[], io: Io = processIo): number { } if (scanFailed) { - io.out(`${renderScanFailedComment(commit)}\n`); + write(`${renderScanFailedComment(commit)}\n`); return 0; } if (resolved) { - io.out(`${renderResolvedComment(commit)}\n`); + write(`${renderResolvedComment(commit)}\n`); return 0; } @@ -106,12 +121,10 @@ export function main(argv: string[], io: Io = processIo): number { } if (hasDelta(head, base)) { - io.out(`${renderPrComment(head, base, commit)}\n`); + write(`${renderPrComment(head, base, commit)}\n`); return 0; } - if (existingComment) { - io.out(`${renderResolvedComment(commit)}\n`); - } + write(existingComment ? `${renderResolvedComment(commit)}\n` : ""); return 0; } From 902379b65e27f240c094b6ec7b4b0602140ce1ce Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Mon, 3 Aug 2026 17:20:50 +0100 Subject: [PATCH 117/117] test(observability-map): pin why hasDelta needs no sensitivity term --- .../src/report/prComment.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/internal-packages/observability-map/src/report/prComment.test.ts b/internal-packages/observability-map/src/report/prComment.test.ts index d6d26c2a2cc..18d9466ef16 100644 --- a/internal-packages/observability-map/src/report/prComment.test.ts +++ b/internal-packages/observability-map/src/report/prComment.test.ts @@ -490,6 +490,54 @@ describe("the commit stamp", () => { // B4. The job posts only when the pull request moves the report, so the decision has to be a // tested function of the two reports rather than shell logic in the workflow. +/** + * `hasDelta` compares no `sensitive` field, and does not need one. Sensitivity reaches the rendered + * comment only through `fixFirstSection`, whose primary sort key it is, and a route can only appear + * there with a scored failure. A scored failure needs a try/catch in the body, `isTrivialExport` + * rejects any export that has one, and a sensitive non-trivial export is therefore either accused + * (`fail`) or guarded (`pass`) on `auth-boundary`, never `not-applicable`. So a flip that could move + * the fix list always moves `auth-boundary`, which moves `checkContributions`, which is compared. + * + * Both halves are pinned here because the argument rests on that coupling: make a trivial route + * capable of a scored failure and the first case below starts rendering a difference `hasDelta` cannot + * see. + */ +describe("a sensitivity flip", () => { + const stub = `import { redirect } from "@remix-run/server-runtime"; + export const loader = () => redirect("/");`; + const stubSensitive = `import { redirect } from "@remix-run/server-runtime"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export const loader = () => redirect("/");`; + + it("renders nothing different on a route too trivial to reach the fix list", () => { + const base = buildReport([scanFile("resources.stub.ts", stub)!], []); + const head = buildReport([scanFile("resources.stub.ts", stubSensitive)!], []); + expect(base.entries[0]!.sensitive).toBe(false); + expect(head.entries[0]!.sensitive).toBe(true); + + expect(renderPrComment(head, base)).toBe(renderPrComment(base, base)); + expect(hasDelta(head, base)).toBe(false); + }); + + it("moves auth-boundary, and so is caught, on a route that does real work", () => { + const working = `export async function loader() { + try { compute(); } catch (e) { return null; } + }`; + const workingSensitive = `import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function loader() { + try { compute(); } catch (e) { return null; } + }`; + const base = buildReport([scanFile("resources.work.ts", working)!], []); + const head = buildReport([scanFile("resources.work.ts", workingSensitive)!], []); + + const status = (r: typeof base) => + r.entries[0]!.checks.find((c) => c.id === "auth-boundary")!.status; + expect(status(base)).toBe("not-applicable"); + expect(status(head)).toBe("fail"); + expect(hasDelta(head, base)).toBe(true); + }); +}); + describe("hasDelta", () => { const trivial = `export const loader = () => new Response("ok");`; const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []);