fix: record findings held back by the reportable severity gate

sdk/typescript/_bundled_plugin/references/final-report.md

@@ -45,6 +45,11 @@ When there are no reportable findings, include a short `No findings` section tha
 When there are reportable findings, render them as readable markdown findings rather than raw JSON or a dumped schema object.
 Order findings from highest severity to lowest severity: `critical`, then `high`, then `medium`, then `low`.
 
+`informational` is outside the reportable severity set, so those findings are not
+detailed in the report. They are still sealed into `findings.json` and included in
+the SARIF and CSV exports, so the report records how many were held back rather
+than omitting them silently.
+
 Use a separate finding entry for each independently attackable source/control/sink instance. Do not combine sibling routes, templates, query builders, parser operations, auth/object-access endpoints, or shared-helper callers into one representative finding solely for readability; if grouping helps, add a short grouped summary after the individual finding entries.
 
 If validation or attack-path analysis provides a broad family row with multiple independently triggerable sink, parser, helper, API-mode, or protected-action lines, split it into child final findings before writing the report. Multiple affected lines inside one finding are appropriate for one inseparable proof tuple, such as a wrapper plus its shared sink, but not as a substitute for separate findings when sibling operations can be triggered independently.

sdk/typescript/_bundled_plugin/scripts/report_projection.py

@@ -363,6 +363,23 @@ def _code_evidence_lines(evidence: list[dict[str, Any]]) -> list[str]:
     return lines
 
 
+def _unreported_findings_note(findings: list[dict[str, Any]]) -> str:
+    """Describe findings held back from this report by the reportable-severity gate.
+
+    They remain sealed in findings.json and exported to SARIF and CSV, so without
+    this line the three artifacts of one scan contradict each other and the one a
+    human reads is the one missing data.
+    """
+    count = len(findings)
+    subject = "finding is" if count == 1 else "findings are"
+    pronoun = "It remains" if count == 1 else "They remain"
+    return (
+        f"{count} {subject} outside the reportable severity set "
+        f"({_severity_mix(findings)}) and not detailed here. "
+        f"{pronoun} recorded in `findings.json` and in the SARIF and CSV exports."
+    )
+
+
 def _severity_mix(findings: list[dict[str, Any]]) -> str:
     counts = Counter(finding["severity"]["level"] for finding in findings)
     return (
@@ -588,6 +605,11 @@ def build_report_markdown(
         ),
         key=_finding_sort_key,
     )
+    unreported = [
+        finding
+        for finding in findings_document["findings"]
+        if finding["severity"]["level"] not in REPORTABLE_SEVERITIES
+    ]
     writeup_paths = [_writeup_report_path(finding) for finding in findings]
     duplicate_writeup_paths = sorted(
         path
@@ -719,6 +741,8 @@ def build_report_markdown(
                     f"| {finding_link} | {finding['severity']['level']} "
                     f"| {finding['confidence']['level']} | {writeup_link} |"
                 )
+        if unreported:
+            lines.extend(["", _unreported_findings_note(unreported)])
         lines.extend(
             [
                 "",
@@ -746,6 +770,8 @@ def build_report_markdown(
                 "No reportable findings survived the canonical discovery, validation, and reportability gates.",
             ]
         )
+        if unreported:
+            lines.extend(["", _unreported_findings_note(unreported)])
     if hardening_portfolio_path is not None:
         lines.extend(
             [

sdk/typescript/tests-ts/report-projection.test.ts

@@ -0,0 +1,141 @@
+import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, test } from "bun:test";
+
+const pluginRoot = join(
+  dirname(fileURLToPath(import.meta.url)),
+  "..",
+  "_bundled_plugin",
+);
+
+function usablePython(): string | null {
+  for (const candidate of [process.env["PYTHON"], "python3", "python"]) {
+    if (candidate === undefined) continue;
+    const probe = spawnSync(
+      candidate,
+      [
+        "-c",
+        "import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)",
+      ],
+      { encoding: "utf8" },
+    );
+    if (probe.status === 0) return candidate;
+  }
+  return null;
+}
+
+const python = usablePython();
+
+/** Render report.md through the bundled projection with each finding forced to `level`. */
+function renderReport(python: string, levels: readonly string[]): string {
+  const script = `
+import copy, json, sys
+sys.path.insert(0, ${JSON.stringify(join(pluginRoot, "scripts"))})
+import report_projection
+
+base = ${JSON.stringify(join(pluginRoot, "examples", "completed-scan"))}
+def load(name):
+    with open(base + "/" + name, encoding="utf-8") as handle:
+        return json.load(handle)
+
+manifest, coverage, findings = load("scan-manifest.json"), load("coverage.json"), load("findings.json")
+levels = json.loads(${JSON.stringify(JSON.stringify(levels))})
+template = copy.deepcopy(findings["findings"][0])
+findings["findings"] = []
+for index, level in enumerate(levels):
+    entry = copy.deepcopy(template)
+    entry["severity"]["level"] = level
+    entry["title"] = "Finding %d" % index
+    entry["findingId"] = "%s-%d" % (entry.get("findingId", "finding"), index)
+    if index > 0:
+        entry.pop("writeup", None)
+    findings["findings"].append(entry)
+
+sys.stdout.write(report_projection.build_report_markdown(manifest, findings, coverage))
+`;
+  const result = spawnSync(python, ["-I", "-B", "-c", script], {
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`report projection failed: ${result.stderr}`);
+  }
+  return result.stdout;
+}
+
+describe("report projection severity gate", () => {
+  // https://github.com/openai/codex-security/issues/48 -- informational findings
+  // are sealed into findings.json and exported to SARIF and CSV, but excluded
+  // from report.md, which then claimed the scan produced nothing.
+  test.skipIf(python === null)(
+    "records findings held back by the reportable severity gate",
+    () => {
+      const text = renderReport(python!, ["informational"]);
+      expect(text).toContain("### No findings");
+      expect(text).toContain(
+        "1 finding is outside the reportable severity set",
+      );
+      expect(text).toContain("(informational: 1)");
+      expect(text).toContain("`findings.json`");
+    },
+  );
+
+  test.skipIf(python === null)(
+    "records held-back findings alongside reportable ones",
+    () => {
+      const text = renderReport(python!, [
+        "high",
+        "informational",
+        "informational",
+      ]);
+      expect(text).not.toContain("### No findings");
+      expect(text).toContain("Finding 0");
+      expect(text).toContain(
+        "2 findings are outside the reportable severity set",
+      );
+      expect(text).toContain("(informational: 2)");
+    },
+  );
+
+  test.skipIf(python === null)(
+    "stays silent when every finding is reportable",
+    () => {
+      const text = renderReport(python!, ["high", "low"]);
+      expect(text).not.toContain("outside the reportable severity set");
+      expect(text).not.toContain("### No findings");
+    },
+  );
+
+  test.skipIf(python === null)(
+    "keeps the rendered report valid for the format validator",
+    () => {
+      for (const levels of [["informational"], ["high", "informational"]]) {
+        const text = renderReport(python!, levels);
+        const validation = spawnSync(
+          python!,
+          [
+            "-I",
+            "-B",
+            join(pluginRoot, "scripts", "validate_report_format.py"),
+            "--report-md",
+            "/dev/stdin",
+          ],
+          { encoding: "utf8", input: text },
+        );
+        expect(validation.status).toBe(0);
+      }
+    },
+  );
+
+  test("documents the exclusion in the final-report reference", () => {
+    const reference = readFileSync(
+      join(pluginRoot, "references", "final-report.md"),
+      "utf8",
+    );
+    expect(reference).toContain(
+      "`informational` is outside the reportable severity set",
+    );
+  });
+});