diff --git a/sdk/typescript/_bundled_plugin/references/final-report.md b/sdk/typescript/_bundled_plugin/references/final-report.md index 9e5f555e..755c53f9 100644 --- a/sdk/typescript/_bundled_plugin/references/final-report.md +++ b/sdk/typescript/_bundled_plugin/references/final-report.md @@ -45,6 +45,11 @@ When there are no reportable findings, include a short `No findings` section tha When there are reportable findings, render them as readable markdown findings rather than raw JSON or a dumped schema object. Order findings from highest severity to lowest severity: `critical`, then `high`, then `medium`, then `low`. +`informational` is outside the reportable severity set, so those findings are not +detailed in the report. They are still sealed into `findings.json` and included in +the SARIF and CSV exports, so the report records how many were held back rather +than omitting them silently. + Use a separate finding entry for each independently attackable source/control/sink instance. Do not combine sibling routes, templates, query builders, parser operations, auth/object-access endpoints, or shared-helper callers into one representative finding solely for readability; if grouping helps, add a short grouped summary after the individual finding entries. If validation or attack-path analysis provides a broad family row with multiple independently triggerable sink, parser, helper, API-mode, or protected-action lines, split it into child final findings before writing the report. Multiple affected lines inside one finding are appropriate for one inseparable proof tuple, such as a wrapper plus its shared sink, but not as a substitute for separate findings when sibling operations can be triggered independently. diff --git a/sdk/typescript/_bundled_plugin/scripts/report_projection.py b/sdk/typescript/_bundled_plugin/scripts/report_projection.py index e1e9f45c..5578bae7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/report_projection.py +++ b/sdk/typescript/_bundled_plugin/scripts/report_projection.py @@ -363,6 +363,23 @@ def _code_evidence_lines(evidence: list[dict[str, Any]]) -> list[str]: return lines +def _unreported_findings_note(findings: list[dict[str, Any]]) -> str: + """Describe findings held back from this report by the reportable-severity gate. + + They remain sealed in findings.json and exported to SARIF and CSV, so without + this line the three artifacts of one scan contradict each other and the one a + human reads is the one missing data. + """ + count = len(findings) + subject = "finding is" if count == 1 else "findings are" + pronoun = "It remains" if count == 1 else "They remain" + return ( + f"{count} {subject} outside the reportable severity set " + f"({_severity_mix(findings)}) and not detailed here. " + f"{pronoun} recorded in `findings.json` and in the SARIF and CSV exports." + ) + + def _severity_mix(findings: list[dict[str, Any]]) -> str: counts = Counter(finding["severity"]["level"] for finding in findings) return ( @@ -588,6 +605,11 @@ def build_report_markdown( ), key=_finding_sort_key, ) + unreported = [ + finding + for finding in findings_document["findings"] + if finding["severity"]["level"] not in REPORTABLE_SEVERITIES + ] writeup_paths = [_writeup_report_path(finding) for finding in findings] duplicate_writeup_paths = sorted( path @@ -719,6 +741,8 @@ def build_report_markdown( f"| {finding_link} | {finding['severity']['level']} " f"| {finding['confidence']['level']} | {writeup_link} |" ) + if unreported: + lines.extend(["", _unreported_findings_note(unreported)]) lines.extend( [ "", @@ -746,6 +770,8 @@ def build_report_markdown( "No reportable findings survived the canonical discovery, validation, and reportability gates.", ] ) + if unreported: + lines.extend(["", _unreported_findings_note(unreported)]) if hardening_portfolio_path is not None: lines.extend( [ diff --git a/sdk/typescript/tests-ts/report-projection.test.ts b/sdk/typescript/tests-ts/report-projection.test.ts new file mode 100644 index 00000000..fdfbbf76 --- /dev/null +++ b/sdk/typescript/tests-ts/report-projection.test.ts @@ -0,0 +1,141 @@ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "bun:test"; + +const pluginRoot = join( + dirname(fileURLToPath(import.meta.url)), + "..", + "_bundled_plugin", +); + +function usablePython(): string | null { + for (const candidate of [process.env["PYTHON"], "python3", "python"]) { + if (candidate === undefined) continue; + const probe = spawnSync( + candidate, + [ + "-c", + "import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)", + ], + { encoding: "utf8" }, + ); + if (probe.status === 0) return candidate; + } + return null; +} + +const python = usablePython(); + +/** Render report.md through the bundled projection with each finding forced to `level`. */ +function renderReport(python: string, levels: readonly string[]): string { + const script = ` +import copy, json, sys +sys.path.insert(0, ${JSON.stringify(join(pluginRoot, "scripts"))}) +import report_projection + +base = ${JSON.stringify(join(pluginRoot, "examples", "completed-scan"))} +def load(name): + with open(base + "/" + name, encoding="utf-8") as handle: + return json.load(handle) + +manifest, coverage, findings = load("scan-manifest.json"), load("coverage.json"), load("findings.json") +levels = json.loads(${JSON.stringify(JSON.stringify(levels))}) +template = copy.deepcopy(findings["findings"][0]) +findings["findings"] = [] +for index, level in enumerate(levels): + entry = copy.deepcopy(template) + entry["severity"]["level"] = level + entry["title"] = "Finding %d" % index + entry["findingId"] = "%s-%d" % (entry.get("findingId", "finding"), index) + if index > 0: + entry.pop("writeup", None) + findings["findings"].append(entry) + +sys.stdout.write(report_projection.build_report_markdown(manifest, findings, coverage)) +`; + const result = spawnSync(python, ["-I", "-B", "-c", script], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`report projection failed: ${result.stderr}`); + } + return result.stdout; +} + +describe("report projection severity gate", () => { + // https://github.com/openai/codex-security/issues/48 -- informational findings + // are sealed into findings.json and exported to SARIF and CSV, but excluded + // from report.md, which then claimed the scan produced nothing. + test.skipIf(python === null)( + "records findings held back by the reportable severity gate", + () => { + const text = renderReport(python!, ["informational"]); + expect(text).toContain("### No findings"); + expect(text).toContain( + "1 finding is outside the reportable severity set", + ); + expect(text).toContain("(informational: 1)"); + expect(text).toContain("`findings.json`"); + }, + ); + + test.skipIf(python === null)( + "records held-back findings alongside reportable ones", + () => { + const text = renderReport(python!, [ + "high", + "informational", + "informational", + ]); + expect(text).not.toContain("### No findings"); + expect(text).toContain("Finding 0"); + expect(text).toContain( + "2 findings are outside the reportable severity set", + ); + expect(text).toContain("(informational: 2)"); + }, + ); + + test.skipIf(python === null)( + "stays silent when every finding is reportable", + () => { + const text = renderReport(python!, ["high", "low"]); + expect(text).not.toContain("outside the reportable severity set"); + expect(text).not.toContain("### No findings"); + }, + ); + + test.skipIf(python === null)( + "keeps the rendered report valid for the format validator", + () => { + for (const levels of [["informational"], ["high", "informational"]]) { + const text = renderReport(python!, levels); + const validation = spawnSync( + python!, + [ + "-I", + "-B", + join(pluginRoot, "scripts", "validate_report_format.py"), + "--report-md", + "/dev/stdin", + ], + { encoding: "utf8", input: text }, + ); + expect(validation.status).toBe(0); + } + }, + ); + + test("documents the exclusion in the final-report reference", () => { + const reference = readFileSync( + join(pluginRoot, "references", "final-report.md"), + "utf8", + ); + expect(reference).toContain( + "`informational` is outside the reportable severity set", + ); + }); +});