From ba07754f44f4ea0c557d64883f7108c47557ee82 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 6 Nov 2026 16:06:05 +0000 Subject: [PATCH] Add the seidroid review v2 contract and its tests --- .github/dependabot.yml | 4 + .github/seidroid/README.md | 1 + .github/seidroid/review-v2/.gitignore | 1 + .github/seidroid/review-v2/README.md | 26 ++ .github/seidroid/review-v2/package-lock.json | 57 ++++ .github/seidroid/review-v2/package.json | 19 ++ .github/seidroid/review-v2/src/accepted.ts | 79 ++++++ .github/seidroid/review-v2/src/verdict.ts | 251 ++++++++++++++++++ .../seidroid/review-v2/test/accepted.test.ts | 51 ++++ .../seidroid/review-v2/test/verdict.test.ts | 150 +++++++++++ .github/seidroid/review-v2/tsconfig.json | 17 ++ .github/workflows/seidroid-review-v2-test.yml | 35 +++ 12 files changed, 691 insertions(+) create mode 100644 .github/seidroid/review-v2/.gitignore create mode 100644 .github/seidroid/review-v2/README.md create mode 100644 .github/seidroid/review-v2/package-lock.json create mode 100644 .github/seidroid/review-v2/package.json create mode 100644 .github/seidroid/review-v2/src/accepted.ts create mode 100644 .github/seidroid/review-v2/src/verdict.ts create mode 100644 .github/seidroid/review-v2/test/accepted.test.ts create mode 100644 .github/seidroid/review-v2/test/verdict.test.ts create mode 100644 .github/seidroid/review-v2/tsconfig.json create mode 100644 .github/workflows/seidroid-review-v2-test.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace460..14f9069 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,7 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "npm" + directory: "/.github/seidroid/review-v2" + schedule: + interval: "weekly" diff --git a/.github/seidroid/README.md b/.github/seidroid/README.md index 5d5db30..1dd0818 100644 --- a/.github/seidroid/README.md +++ b/.github/seidroid/README.md @@ -9,6 +9,7 @@ publishes. The reusable *workflows* that run these features live in `.github/wor | Feature | What it is | Trigger | |---|---|---| | [`ai-review/`](ai-review/) | The workflow-driven seidroid[bot] helpers and their base prompts: an automatic three-pass PR review (OpenAI Codex ∥ Cursor → Claude synthesis, posting one PR review + an `AI Review` check) and the conversational `@seidroid` assistant. Prompts: `scout.md`, `review.md`, `assistant.md`. | `pull_request` (review); `@seidroid` mention (assistant) | +| [`review-v2/`](review-v2/) | The next generation of the seidroid review: the same review contract on [pi](https://pi.dev) running on the GitHub runner, with no session server. In development, alongside `seidroid-review.yml`. | `pull_request`; `@seidroid review` | `ai-review` passes the diff to models from inside the Actions runner, so it sees the change but never runs it. A sandbox-backed path that could build and test a PR before reviewing it diff --git a/.github/seidroid/review-v2/.gitignore b/.github/seidroid/review-v2/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/.github/seidroid/review-v2/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/.github/seidroid/review-v2/README.md b/.github/seidroid/review-v2/README.md new file mode 100644 index 0000000..b0206b1 --- /dev/null +++ b/.github/seidroid/review-v2/README.md @@ -0,0 +1,26 @@ +# seidroid review v2 + +The next generation of the seidroid review: the same bot, identity, markers and review +contract as [`seidroid-review.yml`](../../workflows/seidroid-review.yml), on a simpler +engine. The review runs in [pi](https://pi.dev) on the GitHub runner, so there is no +session server, sandbox or teardown, and every part of it is pinned by one uci ref. + +It is in development and runs beside v1 without changing it. + +## Layout + +| Path | What it is | +|---|---| +| `src/verdict.ts` | The review contract: the `submit_review` schema, and how a submission becomes the recorded decision, check conclusion, placeable findings and thread plan. | +| `src/accepted.ts` | The base branch's accepted pre-existing conditions, read from its guidelines. | +| `test/` | Tests, ported from the v1 driver's. | + +## Development + +Node 24 runs the TypeScript directly, so there is no build step. + +```sh +npm ci +npm run typecheck +npm test +``` diff --git a/.github/seidroid/review-v2/package-lock.json b/.github/seidroid/review-v2/package-lock.json new file mode 100644 index 0000000..21c675c --- /dev/null +++ b/.github/seidroid/review-v2/package-lock.json @@ -0,0 +1,57 @@ +{ + "name": "@sei-protocol/seidroid-review-v2", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@sei-protocol/seidroid-review-v2", + "dependencies": { + "typebox": "1.3.27" + }, + "devDependencies": { + "@types/node": "24.19.2", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/typebox": { + "version": "1.3.27", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.27.tgz", + "integrity": "sha512-zu+jc1pcy4UiNThxikUr36f0Rybk9PEeCg/NE6adeWr/SKsdNO4EzZHYRDlv2YCVAfj3Odq3dESSo/jNyoBXzA==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/.github/seidroid/review-v2/package.json b/.github/seidroid/review-v2/package.json new file mode 100644 index 0000000..76b1f2d --- /dev/null +++ b/.github/seidroid/review-v2/package.json @@ -0,0 +1,19 @@ +{ + "name": "@sei-protocol/seidroid-review-v2", + "private": true, + "type": "module", + "engines": { + "node": ">=24" + }, + "scripts": { + "test": "node --test", + "typecheck": "tsc" + }, + "dependencies": { + "typebox": "1.3.27" + }, + "devDependencies": { + "@types/node": "24.19.2", + "typescript": "5.9.3" + } +} diff --git a/.github/seidroid/review-v2/src/accepted.ts b/.github/seidroid/review-v2/src/accepted.ts new file mode 100644 index 0000000..de83b58 --- /dev/null +++ b/.github/seidroid/review-v2/src/accepted.ts @@ -0,0 +1,79 @@ +const ACCEPTED_HEADING = "accepted"; +const MIN_ACCEPTANCE_WORDS = 3; +const MAX_ACCEPTANCES = 100; +const NEGATION_REACH = 3; +const NEGATIONS = new Set(["no", "not", "never", "longer", "without", "stopped", "removed", "dropped", "neither", "nor"]); +const CONJUNCTIONS = [" and ", " but ", " also ", " as well as ", " plus ", "; "]; + +/** + * parseAccepted returns the bullets under a guidelines heading whose first word is + * "Accepted": the pre-existing conditions the base branch accepts. Bullets in fences, + * and bullets naming fewer than three words, are skipped. + */ +export function parseAccepted(text: string): string[] { + const out: string[] = []; + let inSection = false; + let inFence = false; + for (const line of text.replace(/\r\n?/g, "\n").split("\n")) { + const t = line.trim(); + if (t.startsWith("```") || t.startsWith("~~~")) { + inFence = !inFence; + continue; + } + if (inFence) continue; + const title = headingTitle(t); + if (title !== undefined) { + const first = title.toLowerCase().split(/\s+/)[0] ?? ""; + inSection = first.replace(/[:.,]+$/, "") === ACCEPTED_HEADING; + continue; + } + if (!inSection || !/^[-+*](\s|$)/.test(t)) continue; + const entry = t.slice(1).trim(); + if (words(acceptancePhrase(entry)).length < MIN_ACCEPTANCE_WORDS) continue; + out.push(entry); + if (out.length >= MAX_ACCEPTANCES) break; + } + return out; +} + +function headingTitle(line: string): string | undefined { + const m = /^(#{1,6})(?:[ \t]|$)/.exec(line); + if (!m || line[m[1]!.length] === "#") return undefined; + return line.slice(m[1]!.length).trim(); +} + +const words = (s: string): string[] => s.split(/\s+/).filter((w) => w !== ""); + +/** acceptancePhrase is an entry without the rationale after its dash. */ +function acceptancePhrase(entry: string): string { + for (const sep of [" — ", " -- ", " – "]) { + const at = entry.indexOf(sep); + if (at >= 0) return entry.slice(0, at); + } + return entry; +} + +/** + * acceptanceFor returns the accepted entry a finding's body names, or "" when none does. + * A phrase negated just before it, or joined to more after it, does not match. + */ +export function acceptanceFor(body: string, accepted: readonly string[]): string { + const haystack = matchable(body); + for (const entry of accepted) { + const phrase = matchable(acceptancePhrase(entry)); + if (phrase === "") continue; + const at = haystack.indexOf(phrase); + if (at < 0) continue; + const before = haystack.slice(0, at); + const after = haystack.slice(at + phrase.length); + if (!negated(before) && !CONJUNCTIONS.some((c) => (" " + after).includes(c))) return entry; + } + return ""; +} + +const negated = (before: string): boolean => + words(before) + .slice(-NEGATION_REACH) + .some((w) => NEGATIONS.has(w.replace(/^[.,;:()]+|[.,;:()]+$/g, ""))); + +const matchable = (s: string): string => words(s.toLowerCase().replace(/[`"'“”‘’]/g, "")).join(" "); diff --git a/.github/seidroid/review-v2/src/verdict.ts b/.github/seidroid/review-v2/src/verdict.ts new file mode 100644 index 0000000..09903d4 --- /dev/null +++ b/.github/seidroid/review-v2/src/verdict.ts @@ -0,0 +1,251 @@ +import { Type, type Static } from "typebox"; +import { Value } from "typebox/value"; +import { acceptanceFor } from "./accepted.ts"; + +const oneOf = (values: T, description?: string) => + Type.Unsafe({ type: "string", enum: [...values], description }); + +const Ids = (description: string) => Type.Optional(Type.Array(Type.String(), { description })); + +export const InlineComment = Type.Object({ + path: Type.String({ description: "File path relative to the repository root." }), + line: Type.Integer({ minimum: 1, description: "Line in the diff the comment anchors to." }), + side: Type.Optional(oneOf(["RIGHT", "LEFT"] as const, "RIGHT for the new file, LEFT for the old.")), + severity: oneOf(["blocker", "suggestion", "nit"] as const), + body: Type.String({ minLength: 1, description: "What is wrong and why it matters." }), + supersedes_thread_ids: Ids("Ids of your earlier threads this comment restates."), +}); + +export const PreExistingIssue = Type.Object({ + severity: oneOf(["blocker", "suggestion"] as const), + body: Type.String({ minLength: 1, description: "Where it is and what it costs." }), +}); + +export const ReviewSubmission = Type.Object({ + read: Type.Integer({ minimum: 0, description: "Line count of the diff you read; 0 if you never got it." }), + decision: oneOf(["approve", "comment", "request_changes"] as const), + summary: Type.String({ + minLength: 1, + description: "What the change does, then what decides the verdict; two sentences at most.", + }), + inline_comments: Type.Optional(Type.Array(InlineComment)), + blockers: Type.Optional(Type.Array(Type.String(), { description: "Must fix, tied to no single line." })), + non_blockers: Type.Optional(Type.Array(Type.String(), { description: "Worth noting, tied to no single line." })), + pre_existing_issues: Type.Optional(Type.Array(PreExistingIssue)), + resolved_thread_ids: Ids("Ids of your open threads whose finding this diff now addresses."), +}); + +export type ReviewSubmission = Static; +export type Decision = ReviewSubmission["decision"]; +export type Conclusion = "success" | "neutral" | "failure"; + +/** Why a recorded position departs from the decision the review wrote. */ +export type Withheld = "as-said" | "blocker" | "empty-comment" | "unread-diff" | "pre-existing"; + +export interface Verdict { + submission: ReviewSubmission; + /** Accepted pre-existing conditions from the base branch's guidelines. */ + accepted: string[]; + /** The ref accepted was read from; empty when unreported. */ + acceptedFrom: string; + /** Scouts whose readings decided this verdict in place of a review; empty otherwise. */ + settledBy: string; +} + +export interface Finding { + file: string; + line: number; + side: "RIGHT" | "LEFT"; + severity: "blocker" | "suggestion" | "nit"; + detail: string; + supersedes: string[]; +} + +export interface PreExisting { + severity: "blocker" | "suggestion"; + body: string; + /** The acceptance this blocker matched; empty when none. */ + accepted: string; +} + +export type Parsed = { ok: true; submission: ReviewSubmission } | { ok: false; reason: string }; + +/** parseSubmission validates a submission from outside the process, such as an artifact. */ +export function parseSubmission(raw: unknown): Parsed { + if (!Value.Check(ReviewSubmission, raw)) { + const first = Value.Errors(ReviewSubmission, raw)[0]; + return { ok: false, reason: `the submission does not match the schema: ${first?.instancePath || "/"} ${first?.message ?? ""}`.trim() }; + } + if (raw.summary.trim() === "") { + return { ok: false, reason: "the submission decides without a summary" }; + } + return { ok: true, submission: raw }; +} + +export function verdictOf(submission: ReviewSubmission, accepted: string[] = [], acceptedFrom = ""): Verdict { + return { submission, accepted, acceptedFrom, settledBy: "" }; +} + +const MAX_PLACEABLE_FINDINGS = 50; +const MAX_THREAD_IDS = 100; +const MAX_THREAD_ID = 200; +const MAX_REFUSED_THREAD_IDS = 20; +const MAX_COMMENT_PATH = 4096; +const MAX_COMMENT_PATH_SEGMENT = 255; + +const trimmed = (items: readonly string[] | undefined): string[] => + (items ?? []).map((s) => s.trim()).filter((s) => s !== ""); + +export const blockers = (v: Verdict): string[] => trimmed(v.submission.blockers); +export const nonBlockers = (v: Verdict): string[] => trimmed(v.submission.non_blockers); + +const threadIds = (ids: readonly string[] | undefined): string[] => trimmed(ids).slice(0, MAX_THREAD_IDS); + +/** reportedFindings returns every inline comment, placeable or not. */ +export function reportedFindings(v: Verdict): Finding[] { + return (v.submission.inline_comments ?? []).map((c) => ({ + file: c.path.trim(), + line: c.line, + side: c.side === "LEFT" ? "LEFT" : "RIGHT", + severity: c.severity, + detail: c.body.trim(), + supersedes: threadIds(c.supersedes_thread_ids), + })); +} + +/** + * placeableFindings returns the findings that can be posted on a line, deduplicated and + * capped, with nits only when included and supersedes narrowed to this review's own threads. + */ +export function placeableFindings(v: Verdict, includeNits: boolean, own: ReadonlySet): Finding[] { + return placeable(v, includeNits).map((f) => ({ ...f, supersedes: f.supersedes.filter((id) => own.has(id)) })); +} + +function placeable(v: Verdict, includeNits: boolean): Finding[] { + const seen = new Set(); + const out: Finding[] = []; + for (const f of reportedFindings(v)) { + if (f.severity === "nit" && !includeNits) continue; + const key = dedupeKey(f); + if (!postablePath(f.file) || f.line <= 0 || f.detail === "" || seen.has(key)) continue; + seen.add(key); + out.push(f); + if (out.length >= MAX_PLACEABLE_FINDINGS) break; + } + return out; +} + +const dedupeKey = (f: Finding): string => [f.file, f.side, f.line, f.detail].join("\0"); + +/** reportedBySeverity counts distinct reported findings that block and that do not. */ +export function reportedBySeverity(v: Verdict, includeNits: boolean): { blocking: number; other: number } { + const blocks = new Map(); + for (const f of reportedFindings(v)) { + if (f.severity === "nit" && !includeNits) continue; + const key = dedupeKey(f); + blocks.set(key, (blocks.get(key) ?? false) || f.severity === "blocker"); + } + const blocking = [...blocks.values()].filter(Boolean).length; + return { blocking, other: blocks.size - blocking }; +} + +/** postablePath reports whether a path is a plain relative path GitHub can anchor a comment to. */ +export function postablePath(p: string): boolean { + if (p === "" || Buffer.byteLength(p) > MAX_COMMENT_PATH || !p.isWellFormed()) return false; + if (/^[/~-]/.test(p) || /[\x00-\x1f\x7f]/.test(p)) return false; + return p + .split("/") + .every((seg) => seg !== "" && seg !== "." && seg !== ".." && Buffer.byteLength(seg) <= MAX_COMMENT_PATH_SEGMENT); +} + +export function preExisting(v: Verdict): PreExisting[] { + return (v.submission.pre_existing_issues ?? []) + .map((issue) => ({ severity: issue.severity, body: issue.body.trim() })) + .filter((issue) => issue.body !== "") + .map((issue) => ({ + ...issue, + accepted: issue.severity === "blocker" ? acceptanceFor(issue.body, v.accepted) : "", + })); +} + +export interface Standing { + decision: Decision; + conclusion: Conclusion; + why: Withheld; +} + +/** + * standing derives the decision and the check conclusion together from the findings. The + * decision the review wrote can escalate but never clear: approve is recorded only where + * nothing withholds it. + */ +export function standing(v: Verdict): Standing { + const said = v.submission.decision; + if (blockers(v).length > 0 || reportedFindings(v).some((f) => f.severity === "blocker")) { + return { decision: "request_changes", conclusion: "failure", why: said === "request_changes" ? "as-said" : "blocker" }; + } + if (said === "request_changes") return { decision: said, conclusion: "failure", why: "as-said" }; + // A comment with nothing written down is the prompt's read-failure signal. + if (said === "comment" && !wroteAnythingDown(v)) { + return { decision: "comment", conclusion: "neutral", why: "empty-comment" }; + } + // Without a line count a clean review cannot be told from a review of nothing. + if (v.submission.read <= 0) return { decision: "comment", conclusion: "neutral", why: "unread-diff" }; + // A pre-existing blocker is not the author's to fix, so it never fails the check, but + // no approval is recorded while one stands unaccepted. + if (preExisting(v).some((i) => i.severity === "blocker" && i.accepted === "")) { + return { decision: "comment", conclusion: "neutral", why: "pre-existing" }; + } + return { decision: said, conclusion: "success", why: "as-said" }; +} + +const wroteAnythingDown = (v: Verdict): boolean => + reportedFindings(v).length > 0 || nonBlockers(v).length > 0 || preExisting(v).length > 0; + +export interface ThreadPlan { + addressed: string[]; + superseded: string[]; + refused: string[]; + refusedTotal: number; +} + +/** ownThreadIds is the set of well-formed ids of the threads this review left before. */ +export function ownThreadIds(ids: readonly string[]): Set { + return new Set(ids.filter(wellFormedThreadId)); +} + +/** + * buildThreadPlan splits the thread ids a review named into those it addressed, those its + * placed findings supersede, and those refused because they are not this review's own. + */ +export function buildThreadPlan(v: Verdict, includeNits: boolean, own: ReadonlySet): ThreadPlan { + const plan: ThreadPlan = { addressed: [], superseded: [], refused: [], refusedTotal: 0 }; + const seen = new Set(); + const admit = (ids: string[], into?: string[]) => { + for (const id of ids) { + if (seen.has(id) || seen.size >= MAX_THREAD_IDS * MAX_PLACEABLE_FINDINGS) continue; + seen.add(id); + if (own.has(id)) { + into?.push(id); + continue; + } + plan.refusedTotal++; + if (plan.refused.length < MAX_REFUSED_THREAD_IDS) plan.refused.push(safeThreadId(id)); + } + }; + admit(placeable(v, includeNits).flatMap((f) => f.supersedes), plan.superseded); + admit(reportedFindings(v).flatMap((f) => f.supersedes)); + admit(threadIds(v.submission.resolved_thread_ids), plan.addressed); + return plan; +} + +const THREAD_ID_RUNE = /^[A-Za-z0-9_\-=+/]$/; + +export const wellFormedThreadId = (s: string): boolean => + s !== "" && s.length <= MAX_THREAD_ID && [...s].every((r) => THREAD_ID_RUNE.test(r)); + +const safeThreadId = (s: string): string => + [...s] + .map((r) => (THREAD_ID_RUNE.test(r) ? r : "?")) + .join("") + .slice(0, MAX_THREAD_ID); diff --git a/.github/seidroid/review-v2/test/accepted.test.ts b/.github/seidroid/review-v2/test/accepted.test.ts new file mode 100644 index 0000000..198b9ac --- /dev/null +++ b/.github/seidroid/review-v2/test/accepted.test.ts @@ -0,0 +1,51 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { acceptanceFor, parseAccepted } from "../src/accepted.ts"; + +test("parseAccepted", async (t) => { + const cases: [string, string, string[]][] = [ + ["no section", "# Review\n\n- pins uci to the feature branch\n", []], + [ + "the section, then another heading ends it", + "## Accepted pre-existing conditions\n- pins uci to the feature branch\n## Style\n- prefer short names here\n", + ["pins uci to the feature branch"], + ], + [ + "star bullets and a rationale", + "### Accepted\n* pins uci to the feature branch -- until PLT-1300\n", + ["pins uci to the feature branch -- until PLT-1300"], + ], + ["too short to name one condition", "## Accepted\n- pin\n- the pin\n- blocker — anything\n", []], + ["a heading that merely contains the word", "## Not accepted\n- pins uci to the feature branch\n", []], + ["crlf", "## Accepted\r\n- pins uci to the feature branch\r\n", ["pins uci to the feature branch"]], + ["a heading that only begins with the word", "## Acceptedness of pins\n- pins uci to the feature branch\n", []], + ["a heading with the word later", "## Conditions accepted on main\n- pins uci to the feature branch\n", []], + ["seven hashes is no heading", "####### Accepted\n- pins uci to the feature branch\n", []], + [ + "an example in a fence is not an entry", + "## Accepted\n```markdown\n- pins uci to the feature branch\n```\n- reads the token from the env\n", + ["reads the token from the env"], + ], + ]; + for (const [name, text, want] of cases) { + await t.test(name, () => assert.deepEqual(parseAccepted(text), want)); + } +}); + +test("acceptanceFor folds form and stays narrow", () => { + const accepted = ["pins `uci` to the feature branch — deliberate"]; + const cases: [string, boolean][] = [ + ["go.mod PINS uci to the feature branch instead of a tag", true], + ["pins 'uci' to the feature branch", true], + ["pins uci to a tag", false], + ["", false], + ["go.mod no longer pins uci to the feature branch", false], + ["the module is untagged, so go.mod pins uci to the feature branch", true], + ["pins uci to the feature branch and leaks the token", false], + ["pins uci to the feature branch; the token is logged", false], + ["pins uci to the feature branch, which has no tag", true], + ]; + for (const [body, want] of cases) { + assert.equal(acceptanceFor(body, accepted) !== "", want, body); + } +}); diff --git a/.github/seidroid/review-v2/test/verdict.test.ts b/.github/seidroid/review-v2/test/verdict.test.ts new file mode 100644 index 0000000..deebf09 --- /dev/null +++ b/.github/seidroid/review-v2/test/verdict.test.ts @@ -0,0 +1,150 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseAccepted } from "../src/accepted.ts"; +import { + buildThreadPlan, + ownThreadIds, + parseSubmission, + placeableFindings, + postablePath, + preExisting, + reportedBySeverity, + standing, + verdictOf, + type Verdict, +} from "../src/verdict.ts"; + +function verdict(raw: object, accepted: string[] = []): Verdict { + const parsed = parseSubmission(raw); + assert.ok(parsed.ok, parsed.ok ? "" : parsed.reason); + return verdictOf(parsed.submission, accepted); +} + +const finding = (over: object = {}) => ({ path: "a.go", line: 2, severity: "suggestion", body: "x", ...over }); + +test("parseSubmission refuses what the contract does not describe", async (t) => { + const cases: [string, unknown][] = [ + ["not an object", "approve"], + ["no decision", { read: 1, summary: "s" }], + ["an unrecognised decision", { read: 1, decision: "LGTM ship it", summary: "s" }], + ["a near-miss decision", { read: 1, decision: "changes_requested", summary: "s" }], + ["no summary", { read: 1, decision: "approve" }], + ["a blank summary", { read: 1, decision: "approve", summary: " " }], + ["no line count", { decision: "approve", summary: "s" }], + ["a severity outside the set", { read: 1, decision: "comment", summary: "s", inline_comments: [finding({ severity: "critical" })] }], + ["a line of 0", { read: 1, decision: "comment", summary: "s", inline_comments: [finding({ line: 0 })] }], + ]; + for (const [name, raw] of cases) { + await t.test(name, () => { + const parsed = parseSubmission(raw); + assert.equal(parsed.ok, false); + assert.notEqual(parsed.ok ? "" : parsed.reason, ""); + }); + } +}); + +test("standing derives the decision with the conclusion", async (t) => { + const cases: [string, object, string, string, string][] = [ + ["clean approve", { read: 40, decision: "approve", summary: "s" }, "approve", "success", "as-said"], + ["approve over suggestions", { read: 40, decision: "approve", summary: "s", non_blockers: ["consider a table"] }, "approve", "success", "as-said"], + ["request_changes with a blocker", { read: 40, decision: "request_changes", summary: "s", blockers: ["nil deref"] }, "request_changes", "failure", "as-said"], + ["request_changes with nothing written", { read: 40, decision: "request_changes", summary: "s" }, "request_changes", "failure", "as-said"], + ["approve beside a blocker", { read: 40, decision: "approve", summary: "s", blockers: ["nil deref"] }, "request_changes", "failure", "blocker"], + ["approve beside a line-tied blocker", { read: 40, decision: "approve", summary: "s", inline_comments: [finding({ severity: "blocker" })] }, "request_changes", "failure", "blocker"], + ["an unplaceable blocker still blocks", { read: 40, decision: "approve", summary: "s", inline_comments: [finding({ path: "../x", severity: "blocker" })] }, "request_changes", "failure", "blocker"], + ["approve beside an unaccepted pre-existing blocker", { read: 40, decision: "approve", summary: "s", pre_existing_issues: [{ severity: "blocker", body: "b.go:4 leaks a handle" }] }, "comment", "neutral", "pre-existing"], + ["a pre-existing suggestion does not withhold", { read: 40, decision: "approve", summary: "s", pre_existing_issues: [{ severity: "suggestion", body: "README drifts" }] }, "approve", "success", "as-said"], + ["approve that did not read the diff", { read: 0, decision: "approve", summary: "s" }, "comment", "neutral", "unread-diff"], + ["comment and nothing else", { read: 40, decision: "comment", summary: "s" }, "comment", "neutral", "empty-comment"], + ["comment with notes", { read: 40, decision: "comment", summary: "s", non_blockers: ["a note"] }, "comment", "success", "as-said"], + ]; + for (const [name, raw, decision, conclusion, why] of cases) { + await t.test(name, () => assert.deepEqual(standing(verdict(raw)), { decision, conclusion, why })); + } +}); + +test("an accepted pre-existing blocker does not withhold approval", () => { + const raw = { + read: 40, + decision: "approve", + summary: "s", + pre_existing_issues: [ + { severity: "blocker", body: "go.mod pins uci to the feature branch, not a tag" }, + { severity: "suggestion", body: "go.mod pins uci to the feature branch" }, + ], + }; + const accepted = parseAccepted("## Accepted pre-existing conditions\n\n- pins `uci` to the feature branch — until PLT-1300\n"); + assert.equal(standing(verdict(raw)).decision, "comment"); + const v = verdict(raw, accepted); + assert.deepEqual(standing(v), { decision: "approve", conclusion: "success", why: "as-said" }); + assert.deepEqual( + preExisting(v).map((i) => i.accepted), + [accepted[0], ""], + ); +}); + +test("an acceptance lifts only the blocker it names", () => { + const v = verdict( + { + read: 40, + decision: "approve", + summary: "s", + pre_existing_issues: [ + { severity: "blocker", body: "go.mod pins uci to the feature branch" }, + { severity: "blocker", body: "c.go:9 writes the token to the log" }, + ], + }, + ["pins uci to the feature branch"], + ); + assert.equal(standing(v).why, "pre-existing"); +}); + +test("placeableFindings drops, dedupes and gates nits", () => { + const v = verdict({ + read: 40, + decision: "comment", + summary: "s", + inline_comments: [ + finding(), + finding(), + finding({ side: "LEFT" }), + finding({ severity: "nit", body: "nit" }), + finding({ path: "/etc/passwd" }), + finding({ body: " " }), + finding({ body: "y", supersedes_thread_ids: ["PRRT_own", "PRRT_other"] }), + ], + }); + const own = ownThreadIds(["PRRT_own"]); + const keys = (nits: boolean) => placeableFindings(v, nits, own).map((f) => `${f.side}:${f.detail}`); + assert.deepEqual(keys(false), ["RIGHT:x", "LEFT:x", "RIGHT:y"]); + assert.deepEqual(keys(true), ["RIGHT:x", "LEFT:x", "RIGHT:nit", "RIGHT:y"]); + assert.deepEqual(placeableFindings(v, false, own).at(-1)?.supersedes, ["PRRT_own"]); + assert.deepEqual(reportedBySeverity(v, false), { blocking: 0, other: 5 }); +}); + +test("postablePath", () => { + for (const p of ["a.go", "x/y/z.go", "dir.with.dots/f"]) assert.ok(postablePath(p), p); + for (const p of ["", "/abs", "~/x", "-flag", "a//b", "a/./b", "../x", "a/..", "a\nb", "a\x7fb", `${"x".repeat(256)}/f`]) { + assert.ok(!postablePath(p), JSON.stringify(p)); + } +}); + +test("buildThreadPlan admits only this review's own threads", () => { + const v = verdict({ + read: 40, + decision: "comment", + summary: "s", + inline_comments: [ + finding({ supersedes_thread_ids: ["PRRT_a", "PRRT_forged"] }), + finding({ path: "../x", supersedes_thread_ids: ["PRRT_unplaced"] }), + ], + resolved_thread_ids: ["PRRT_b", "PRRT_a", "bad id<>", " "], + }); + const plan = buildThreadPlan(v, false, ownThreadIds(["PRRT_a", "PRRT_b", "PRRT_unplaced", "bad id<>"])); + assert.deepEqual(plan, { + addressed: ["PRRT_b"], + superseded: ["PRRT_a"], + refused: ["PRRT_forged", "bad?id??"], + refusedTotal: 2, + }); +}); diff --git a/.github/seidroid/review-v2/tsconfig.json b/.github/seidroid/review-v2/tsconfig.json new file mode 100644 index 0000000..2571f1e --- /dev/null +++ b/.github/seidroid/review-v2/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "es2024", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "test"] +} diff --git a/.github/workflows/seidroid-review-v2-test.yml b/.github/workflows/seidroid-review-v2-test.yml new file mode 100644 index 0000000..8890cbf --- /dev/null +++ b/.github/workflows/seidroid-review-v2-test.yml @@ -0,0 +1,35 @@ +name: seidroid review v2 + +on: + pull_request: + paths: + - '.github/seidroid/review-v2/**' + - '.github/workflows/seidroid-review-v2-test.yml' + push: + branches: [main] + paths: + - '.github/seidroid/review-v2/**' + - '.github/workflows/seidroid-review-v2-test.yml' + +permissions: + contents: read + +jobs: + test: + name: Test + runs-on: ubuntu-latest + defaults: + run: + working-directory: .github/seidroid/review-v2 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + cache-dependency-path: .github/seidroid/review-v2/package-lock.json + - run: npm ci --ignore-scripts + - run: npm run typecheck + - run: npm test