From e641af8e01a01605b4fa771c04ed58340677210d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 14:03:56 -0800 Subject: [PATCH 01/17] chore(quest): claim quest/m0/routing-evidence/review-tallies.md Quest-Claim: 2770feebecfbbc49 From 487c625c1d631cc6e08e4f3d5e048a46a92a16fd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 14:43:18 -0800 Subject: [PATCH 02/17] feat(review): tally every reviewer's findings across a branch's rounds The PR body's review record now carries a Rounds line per reviewer that counts each finding it raised once, by severity, with how many were fixed, refuted, or left open. The tally lives in the branch's saved review state, survives --new, credits a finding to the reviewer that raised it when a new size pick settles it, and keeps a block for a reviewer the run left out. quest usage reads the tallies, so its accuracy rollup counts every finding caught before merge instead of the round before the latest. Co-Authored-By: Claude Opus 5.5 --- README.md | 27 ++-- quest/m0/routing-evidence/README.md | 3 +- quest/m0/routing-evidence/review-tallies.md | 27 ---- quest/m0/routing-evidence/shadow-review.md | 2 - quest/m0/routing-evidence/writer-ab.md | 2 - src/cli/main.ts | 8 +- src/cli/review-record.ts | 109 +++++++++++++--- src/cli/review.ts | 100 +++++++++++++- src/cli/usage.ts | 44 +++---- test/review.test.ts | 138 +++++++++++++++++++- test/usage.test.ts | 82 ++++++++++-- 11 files changed, 429 insertions(+), 113 deletions(-) delete mode 100644 quest/m0/routing-evidence/review-tallies.md diff --git a/README.md b/README.md index 91c5a2b3..936d8048 100644 --- a/README.md +++ b/README.md @@ -1407,7 +1407,8 @@ reviewer's prompt cache; Claude Code is asked for its 1-hour cache, and Codex keeps its default. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in, so another worktree starts a new one; -`--new` starts over. `--branch` names the branch instead of the checked-out one, which a +`--new` starts new sessions, but continues each reviewer's tally of its findings +on the branch, unless the saved state is one this quest cannot read. `--branch` names the branch instead of the checked-out one, which a detached `HEAD` requires, such as a checkout of a branch another worktree holds, so its rounds continue that branch's session. @@ -1454,15 +1455,20 @@ finding count, then each finding as its number, severity, `path:line`, and title indented body, then `earlier` with the last round's head and its findings, each as its number, disposition, severity, `path:line`, and title over its reason; last come a `record` line and a `next` line. `--json` prints one object with -`rounds`, one per reviewer, `record`, and `next`. +`rounds`, one per reviewer, `tallies`, every reviewer's findings across the +branch's rounds, `record`, and `next`. After each round, `quest review` rewrites the `## Review` section of the open PR from the branch into the base, found among the upstream's PRs from the repository the base's remote pushes to, and prints `record `. The record names the reviewed head and the base's short SHA, then for each reviewer its -harness, model, and effort, the session and round, the score and rationale, -each finding with its location, and the last round's findings with their -dispositions. `quest status` takes the last full SHA the section names as the +harness, model, and effort, the session and round, the score and rationale, its tally, each finding with +its location, and the last round's findings with their dispositions. The +tally, as `- Rounds: 3; findings 4 (high 1, medium 3): fixed 3, refuted 1, open 0`, +counts the reviewer's rounds on the branch and each finding it raised once, +by its latest severity, however many rounds keep it open. A finding counts +for the reviewer that raised it, even when a new size pick's reviewer settles +it, and a reviewer the run left out keeps a block with only its tally. `quest status` takes the last full SHA the section names as the reviewed head, so the head appears in full only once no reviewer reports a finding, and model text appears as inline code, which mentions or links nothing and never holds a full @@ -1500,7 +1506,7 @@ moq-dev/quest, PRs updated and sessions active since 2026-11-06; costs (~$) esti #569 S merged: quest: finish model routing (quest/m0/model-routing/README) implementer: Claude Code, claude-opus-5-5, unknown effort - review: Codex, gpt-6.1-sol, medium effort; round 1, score 10/10, 0 open, last round 0 fixed and 0 refuted + review: Codex, gpt-6.1-sol, medium effort; round 1, score 10/10; rounds 1, findings 0: fixed 0, refuted 0, open 0 land fixes: codex, gpt-6.1-sol, medium effort sessions: implementing claude claude-opus-5-5 low in 50, cache 36.2k written 1.1M read, out 6.1k, ~$0.63 @@ -1519,13 +1525,14 @@ remotes, or, once the checkout is deleted, a Codex log for that exact directory names (never a parent's, which a nested clone may not share); otherwise it stays unattributed. A session is reviewing when the PR's `## Review` record names it or Codex marks it a review, implementing when it began before the PR opened, and iterating otherwise. An opencode session, whose log quest does not read, is -the reviewing session of the newest PR whose `## Review` record names it. Each PR shows its `## Models` size and implementers; each reviewer's latest `quest review` round's -reviewer, round, score, open findings, and the last round's findings fixed or -refuted; the land fixer's fixes; and the later PRs that list it under +the reviewing session of the newest PR whose `## Review` record names it. Each PR shows its `## Models` size and implementers; each reviewer the +`quest review` record names, with its latest round and score and its tally of +findings across every round; the land fixer's fixes; and the later PRs that list it under `## Caused by`. Rollups follow: spend by size, role, harness, model, and effort, and accuracy -by size and implementer, where an `Implementer:` line's own size, as +by size and implementer, which sums the tallies, so it counts every finding +caught before merge; a record written before tallies counts as unreviewed there. An `Implementer:` line's own size, as `(iterate, XS)`, counts over the PR's. Sessions on no PR here, from any repository or agent ccusage reads, total on an `Unattributed` line. `--pr` and `--quest` narrow the report to one PR or one quest's branch, and `--json` diff --git a/quest/m0/routing-evidence/README.md b/quest/m0/routing-evidence/README.md index 7e397aec..30199ed2 100644 --- a/quest/m0/routing-evidence/README.md +++ b/quest/m0/routing-evidence/README.md @@ -30,8 +30,7 @@ Decided 2026-11-11 (user, /quest-plan, from an audit of PRs merged - Experiments? ✅ Shadow reviews and a writer A/B (rejected: shadow reviews only; none). - Records? - - ✅ [Cumulative review tallies](/quest/m0/routing-evidence/review-tallies.md) - and [online reviewers in usage](/quest/m0/routing-evidence/online-reviewers.md) + - ✅ Cumulative review tallies and [online reviewers in usage](/quest/m0/routing-evidence/online-reviewers.md) - ✅ [Attribution through token-max](/quest/m0/routing-evidence/usage-token-max.md), kept after the maintainer first left it out, because the writer A/B compares cost and `quest usage` crashes on this host diff --git a/quest/m0/routing-evidence/review-tallies.md b/quest/m0/routing-evidence/review-tallies.md deleted file mode 100644 index bfa3cd17..00000000 --- a/quest/m0/routing-evidence/review-tallies.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] The review record tallies every round - -## Goal - -A PR body's `## Review` record shows, for each reviewer, the findings it -raised across every round on the branch, by severity, and how many were -fixed, refuted, or left open, beside the latest round it already shows. -`quest usage` reads those tallies, so its accuracy rollup counts every -finding caught before merge, not only the round before the latest. - -## Plan - -Decided 2026-11-11 (user, /quest-plan, -[routing evidence](/quest/m0/routing-evidence/README.md)). Today the record -keeps only the latest round. The review state under the clone's Git -directory holds every round, but only on the host that ran it. - -Agent's choices, for the implementer to revisit: - -- Keep one tally line per reviewer, such as - `- Rounds: 3; findings 4 (high 1, medium 3): fixed 3, refuted 1, open 0`, - computed from the saved session's rounds and their dispositions. A - `--new` session continues the old one's tally, so restarting a reviewer - does not reset its count. -- Drop `quest usage`'s "round before the latest" rule once the tally line is - there. Leave older records, which have no tally line, uncounted, since - nothing is in production. diff --git a/quest/m0/routing-evidence/shadow-review.md b/quest/m0/routing-evidence/shadow-review.md index 5329bd90..6be581d6 100644 --- a/quest/m0/routing-evidence/shadow-review.md +++ b/quest/m0/routing-evidence/shadow-review.md @@ -38,5 +38,3 @@ Agent's choices, for the implementer to revisit: - [Review table](/quest/m0/routing-evidence/review-table.md) - the baseline the shadow steps down from -- [Review tallies](/quest/m0/routing-evidence/review-tallies.md) - counts the - shadow's findings across rounds diff --git a/quest/m0/routing-evidence/writer-ab.md b/quest/m0/routing-evidence/writer-ab.md index 90c09553..6c0fbef3 100644 --- a/quest/m0/routing-evidence/writer-ab.md +++ b/quest/m0/routing-evidence/writer-ab.md @@ -41,7 +41,5 @@ Agent's choices, for the implementer to revisit: - [Review table](/quest/m0/routing-evidence/review-table.md) - the review efforts held fixed while the arms run -- [Review tallies](/quest/m0/routing-evidence/review-tallies.md) - counts the - findings the arms are compared on - [usage through token-max](/quest/m0/routing-evidence/usage-token-max.md) - counts sub-agent writers' cost, without which an arm's cost is undercounted diff --git a/src/cli/main.ts b/src/cli/main.ts index 862c18e2..9c793474 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1205,7 +1205,7 @@ export function main(argv: string[], out: Output): number | Promise { "starts a new session that reviews the whole change and still weighs --responses. A round that would " + "resume on the head the last round reviewed, when that round found nothing, runs no reviewer and " + "repeats that round, marked unchanged, so on the same head only a reviewer with findings to settle runs. " + - "--new starts new sessions. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + + "--new starts new sessions, continuing each reviewer's tally of its findings on the branch. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + "A checked-out branch with an upstream other than the base continues its upstream branch's session, so a " + "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + @@ -1297,17 +1297,17 @@ export function main(argv: string[], out: Output): number | Promise { }, ); pending = run.then( - async (rounds) => { + async ({ rounds, tallies }) => { // The rounds are saved in their sessions either way, so their report prints before a failed record. let record: Recorded | undefined; let failure: unknown; try { - record = await recordRounds(root(), rounds, github); + record = await recordRounds(root(), rounds, tallies, github); } catch (error) { failure = error; } if (options.json) - print(JSON.stringify({ rounds, record: record ?? null, next: nextStep(rounds) })); + print(JSON.stringify({ rounds, tallies, record: record ?? null, next: nextStep(rounds) })); else { for (const line of roundLines(rounds)) print(line); if (record !== undefined) diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index cb474fdd..6a4e3e44 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -5,7 +5,7 @@ import { recorded } from "../github/models"; import { type ClaimDependencies, repositoryPath, upstream } from "./claims"; import { git } from "./git"; import { requireRemote, string } from "./github"; -import { type PullModels, type Round, located, parseBase } from "./review"; +import { type PullModels, type Round, SEVERITIES, type Tally, located, parseBase } from "./review"; const HARNESSES: Record = { claude: "Claude Code", codex: "Codex", opencode: "opencode" }; @@ -25,8 +25,35 @@ function code(text: string): string { const capitalized = (word: string) => word[0]!.toUpperCase() + word.slice(1); -/** The `## Review` section's text for a run's `rounds` of the same head, without its heading. */ -export function reviewRecord(rounds: Round[]): string { +/** A tally's counts, as the record's `Rounds:` line and `quest usage` write them. */ +export interface Counts { + rounds: number; + findings: Tally["findings"]; + fixed: number; + refuted: number; + open: number; +} + +/** The counts as the record's `Rounds:` line ends, after the rounds, naming only the severities found. */ +export function findingsText(counts: Omit): string { + const total = SEVERITIES.reduce((sum, severity) => sum + counts.findings[severity], 0); + const severities = SEVERITIES.filter((severity) => counts.findings[severity] > 0) + .map((severity) => `${severity} ${counts.findings[severity]}`) + .join(", "); + return `findings ${total}${severities && ` (${severities})`}: fixed ${counts.fixed}, refuted ${counts.refuted}, open ${counts.open}`; +} + +/** A tally's counts, where a finding neither fixed nor refuted is open. */ +const counted = (tally: Tally): Counts => ({ + ...tally, + open: SEVERITIES.reduce((sum, severity) => sum + tally.findings[severity], 0) - tally.fixed - tally.refuted, +}); + +/** + * The `## Review` section's text for a run's `rounds` of the same head, without its heading, with every tally + * on the branch: each round's reviewer's beside its round, and a reviewer the run left out on its own. + */ +export function reviewRecord(rounds: Round[], tallies: Tally[]): string { const [shared] = rounds; const clean = rounds.every((round) => round.findings.length === 0); const lines = [ @@ -43,6 +70,9 @@ export function reviewRecord(rounds: Round[]): string { `- Session: ${code(round.session)}, round ${round.round}`, `- Score: ${round.score}/10, ${code(round.rationale)}`, ); + const tally = tallies.find((each) => each.reviewer === round.reviewer); + if (tally === undefined) throw new Error(`no tally for ${round.reviewer}'s round`); + lines.push(`- Rounds: ${tally.rounds}; ${findingsText(counted(tally))}`); if (round.findings.length === 0) lines.push("- Findings: none"); else lines.push("- Findings, open until the next round settles them:"); for (const finding of round.findings) @@ -54,24 +84,28 @@ export function reviewRecord(rounds: Round[]): string { ` - ${capitalized(finding.disposition)}: ${finding.severity}, ${code(located(finding))}: ${code(finding.title)}; ${code(finding.reason)}`, ); } + for (const tally of tallies.filter((each) => !rounds.some((round) => round.reviewer === each.reviewer))) + lines.push( + "", + `- Reviewer: ${HARNESSES[tally.reviewer]}, ${tally.model}, ${tally.effort} effort`, + `- Rounds: ${tally.rounds}; ${findingsText(counted(tally))}`, + ); return lines.join("\n"); } -/** What a PR body's `## Review` record says of one reviewer's latest round, as `reviewRecord` writes it. */ +/** What a PR body's `## Review` record says of one reviewer, as `reviewRecord` writes it. */ export interface ReviewSummary { /** The reviewer's harness as the record names it, such as `Codex`. */ harness: string; model: string; effort: string; - session: string; - round: number; - /** Null on a record written before rounds were scored. */ + /** Its latest session and round, null when the latest run left it out. */ + session: string | null; + round: number | null; + /** Null when the latest run left it out, or on a record written before rounds were scored. */ score: number | null; - /** The latest round's findings, open until the next round settles them. */ - findings: number; - /** The round before's findings this round found fixed, and those whose refutation held. */ - fixed: number; - refuted: number; + /** Its findings across every round on the branch, null on a record written before tallies. */ + tally: Counts | null; } const MARK = "Rewritten by `quest review` after each round."; @@ -99,23 +133,51 @@ export function parseReview(body: string): ReviewSummary[] { return found; }; const reviewer = match(/^- Reviewer: (.+?), (\S+), (\S+) effort\r?$/m); - const session = match(/^- Session: (`+) ?(.*?) ?\1, round (\d+)\r?$/m); + const tally = parseCounts(text); + // A reviewer the latest run left out has only its tally. + const session = + tally !== null && !/^- Session: /m.test(text) + ? null + : match(/^- Session: (`+) ?(.*?) ?\1, round (\d+)\r?$/m); const score = /^- Score: (\d+)\/10,/m.exec(text); - const items = (prefix: RegExp) => text.split(/\r?\n/).filter((line) => prefix.test(line)).length; return { harness: reviewer[1]!, model: reviewer[2]!, effort: reviewer[3]!, - session: session[2]!, - round: Number(session[3]), + session: session && session[2]!, + round: session && Number(session[3]), score: score ? Number(score[1]) : null, - findings: items(/^ {2}- (High|Medium|Low), /), - fixed: items(/^ {2}- Fixed: /), - refuted: items(/^ {2}- Refuted: /), + tally, }; }); } +/** A block's `Rounds:` line, or null without one; malformed or inconsistent counts fail. */ +function parseCounts(text: string): Counts | null { + const line = /^- Rounds: (.*?)\r?$/m.exec(text); + if (!line) return null; + const found = /^(\d+); findings (\d+)(?: \(([^)]*)\))?: fixed (\d+), refuted (\d+), open (\d+)$/.exec(line[1]!); + if (!found) throw new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); + const findings = { high: 0, medium: 0, low: 0 }; + for (const part of found[3]?.split(", ") ?? []) { + const [severity, count] = part.split(" "); + if (!SEVERITIES.includes(severity as keyof typeof findings) || !/^\d+$/.test(count ?? "")) + throw new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); + findings[severity as keyof typeof findings] = Number(count); + } + const counts = { + rounds: Number(found[1]), + findings, + fixed: Number(found[4]), + refuted: Number(found[5]), + open: Number(found[6]), + }; + const total = SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0); + if (total !== Number(found[2]) || counts.fixed + counts.refuted + counts.open !== total) + throw new Error(`the ## Review record's Rounds line does not add up: ${line[1]}`); + return counts; +} + /** `body` with `record` as its `## Review` section, which it gains when it has none; the rest stays as written. */ export function withReview(body: string, record: string): string { const section = reviewSection(body); @@ -174,11 +236,16 @@ export async function pullModels( } /** Record a run's `rounds` in the `## Review` section of their branch's open PR. The edit is the token's user's, which `/quest-merge` counts only when that user wrote the PR. */ -export async function recordRounds(root: string, rounds: Round[], deps: ClaimDependencies): Promise { +export async function recordRounds( + root: string, + rounds: Round[], + tallies: Tally[], + deps: ClaimDependencies, +): Promise { const [shared] = rounds; const pull = await openPull(root, shared!.base.ref, shared!.branch, deps); if ("skipped" in pull) return pull; - const next = withReview(pull.body, reviewRecord(rounds)); + const next = withReview(pull.body, reviewRecord(rounds, tallies)); if (next !== pull.body) await pull.client.request(pull.path, { method: "PATCH", body: { body: next } }); return { url: pull.url }; } diff --git a/src/cli/review.ts b/src/cli/review.ts index fdd20b60..5149131d 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -307,9 +307,29 @@ export const processDependencies = (): ReviewDependencies => ({ env: process.env * A branch's review sessions, one per reviewer, kept in the clone's common Git directory so every worktree * resumes them. They are in the order the last run reported them, which numbers their findings. */ +/** One reviewer's findings across every round on a branch, which a `--new` session continues. */ +export interface Tally { + reviewer: ReviewHarness; + /** Its latest round's model and effort. */ + model: string; + effort: string; + rounds: number; + /** Each finding it raised, once however many rounds keep it open, by its latest severity. */ + findings: Record<(typeof SEVERITIES)[number], number>; + fixed: number; + refuted: number; +} + +/** What a run of `quest review` gives: each reviewer's round, and every tally on the branch, the run's first. */ +export interface Reviewed { + rounds: Round[]; + tallies: Tally[]; +} + interface State { branch: string; sessions: Session[]; + tallies: Tally[]; } /** One reviewer's session on a branch. */ @@ -1405,8 +1425,18 @@ export function parseState(value: unknown): State { typeof state === "object" && state !== null && Array.isArray(state.sessions) && + Array.isArray(state.tallies) && + state.tallies.every( + (tally) => + typeof tally?.reviewer === "string" && + typeof tally.rounds === "number" && + typeof tally.fixed === "number" && + typeof tally.refuted === "number" && + SEVERITIES.every((severity) => typeof tally.findings?.[severity] === "number"), + ) && state.sessions.every( (session) => + state.tallies!.some((tally) => tally.reviewer === session?.reviewer) && typeof session?.reviewer === "string" && typeof session.session === "string" && typeof session.root === "string" && @@ -1524,7 +1554,7 @@ function sandboxFor(root: string, env: NodeJS.ProcessEnv): Sandbox { * and `SECOND_REVIEWER`, or only `--harness`'s, each resuming its session on the branch unless `fresh`. * Every reviewer runs at once, and the first to fail stops the others and fails the whole, saving no round. */ -export async function review(options: ReviewOptions, deps: ReviewDependencies): Promise { +export async function review(options: ReviewOptions, deps: ReviewDependencies): Promise { // Native Windows installs the reviewer CLIs as batch shims, which `run` cannot start without a shell. if (process.platform === "win32") throw new ReviewInputError("quest review does not support native Windows yet; use WSL"); @@ -1558,8 +1588,8 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): const statePath = sessionFile(git(root, ["rev-parse", "--path-format=absolute", "--git-common-dir"]), branch); const dir = dirname(statePath); mkdirSync(dir, { recursive: true }); - const stored = options.fresh ? undefined : load(statePath); - const sessions = stored === undefined ? [] : parseState(stored).sessions; + const saved = savedState(statePath, options.fresh); + const sessions = options.fresh || saved === undefined ? [] : saved.sessions; // The last run numbered each session's findings after those of the sessions before it. const firsts = new Map(); let next = 1; @@ -1643,8 +1673,68 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): // The first failure stopped the others, so it is the one to report. if (controller.signal.aborted) throw controller.signal.reason; const done = results.map((result) => (result as PromiseFulfilledResult).value); - save(statePath, { branch, sessions: [...done.map((each) => each.session), ...kept] } satisfies State); - return done.map((each) => each.round); + const tallies = tallied( + saved?.tallies ?? [], + done.map((each, i) => ({ round: each.round, settles: seats[i]!.session?.reviewer })), + ); + save(statePath, { branch, sessions: [...done.map((each) => each.session), ...kept], tallies } satisfies State); + return { rounds: done.map((each) => each.round), tallies }; +} + +/** + * A branch's saved state, or undefined when it has none. `--new` drops the sessions but continues their tallies, + * so it reads them too, starting them over only when the state is one this quest cannot read, which is what + * `--new` is for. + */ +function savedState(path: string, fresh: boolean | undefined): State | undefined { + try { + const stored = load(path); + return stored === undefined ? undefined : parseState(stored); + } catch (error) { + if (fresh && error instanceof ReviewInputError) return undefined; + throw error; + } +} + +/** + * `tallies` after a run's `rounds`, the run's reviewers first. A round counts its findings for its own reviewer + * and settles the last round's for the reviewer that raised them, `settles`, which is another when the round + * replaced it; one it keeps open moves to its own count, since the round reports it again. + */ +export function tallied(tallies: Tally[], rounds: { round: Round; settles?: ReviewHarness }[]): Tally[] { + const by = new Map(tallies.map((tally) => [tally.reviewer, { ...tally, findings: { ...tally.findings } }])); + for (const { round, settles } of rounds) { + if (round.unchanged) continue; + const settled = round.earlier?.findings ?? []; + if (settled.length > 0) { + const raiser = settles === undefined ? undefined : by.get(settles); + if (raiser === undefined) throw new Error(`no tally holds the findings ${round.reviewer}'s round settles`); + for (const finding of settled) { + if (finding.disposition === "fixed") raiser.fixed++; + else if (finding.disposition === "refuted") raiser.refuted++; + else raiser.findings[finding.severity]--; + } + } + const own = by.get(round.reviewer) ?? { + reviewer: round.reviewer, + model: round.model, + effort: round.effort, + rounds: 0, + findings: { high: 0, medium: 0, low: 0 }, + fixed: 0, + refuted: 0, + }; + own.model = round.model; + own.effort = round.effort; + own.rounds++; + for (const finding of round.findings) own.findings[finding.severity]++; + by.set(round.reviewer, own); + } + const run = rounds.map(({ round }) => round.reviewer); + return [ + ...run.flatMap((reviewer) => by.get(reviewer) ?? []), + ...[...by.values()].filter((tally) => !run.includes(tally.reviewer)), + ]; } /** What every reviewer of a run shares: the branch, its base, its change, and what a session is given. */ diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 9a1db4d2..5c965099 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -12,7 +12,8 @@ import { discoveryEnv, git } from "./git"; import { fixes } from "./land-fixer"; import { MERGER } from "./land-watch-fix"; import { type GitHub, type ObjectValue, type Repository, githubRepository, link, requireRemote } from "./github"; -import { type ReviewSummary, parseReview } from "./review-record"; +import { type Counts, type ReviewSummary, findingsText, parseReview } from "./review-record"; +import { SEVERITIES } from "./review"; /** A session's tokens, and their cost at the model's API list price, which subscriptions do not bill. */ export interface Tokens { @@ -66,7 +67,7 @@ export interface PullRow { handedBack: boolean; size: Size | null; implementers: { harness: string; model: string; effort: string; size: Size | null }[]; - /** Each reviewer's latest round, as the PR's `## Review` record names them. */ + /** Each reviewer the PR's `## Review` record names, with its latest round and its tally. */ review: ReviewSummary[]; /** Who made each mechanical fix the land fixer recorded, or `quest land watch` for a base merge it made alone. */ landFixes: string[]; @@ -93,16 +94,13 @@ export interface AccuracyRow { pulls: number; merged: number; handedBack: number; + /** PRs whose record tallies their reviews; a record written before tallies counts for none of what follows. */ reviewed: number; /** The reviewers' latest rounds on these PRs that have a score, and the scores' sum. */ scored: number; score: number; - /** The sum of each reviewed PR's longest session's rounds. */ - rounds: number; - /** Sums over every reviewer's latest round. */ - findings: number; - fixed: number; - refuted: number; + /** Every reviewer's findings across every round on these PRs, and as `rounds`, each PR's most rounds. */ + tally: Counts; /** The land fixer's model-run fixes, leaving out the watcher's own base merges. */ landFixes: number; /** Later PRs naming these PRs under `## Caused by`. */ @@ -590,10 +588,7 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { reviewed: 0, scored: 0, score: 0, - rounds: 0, - findings: 0, - fixed: 0, - refuted: 0, + tally: { rounds: 0, findings: { high: 0, medium: 0, low: 0 }, fixed: 0, refuted: 0, open: 0 }, landFixes: 0, fixes: 0, }; @@ -604,18 +599,20 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { if (each.handedBack) row.handedBack++; row.fixes += each.fixedBy.length; row.landFixes += each.landFixes.filter((by) => by !== MERGER).length; - if (each.review.length > 0) { + const tallied = each.review.filter((review) => review.tally !== null); + if (tallied.length > 0) { row.reviewed++; - row.rounds += Math.max(...each.review.map((review) => review.round)); + row.tally.rounds += Math.max(...tallied.map((review) => review.tally!.rounds)); } - for (const review of each.review) { - if (review.score !== null) { + for (const { score, tally } of tallied) { + if (score !== null) { row.scored++; - row.score += review.score; + row.score += score; } - row.findings += review.findings; - row.fixed += review.fixed; - row.refuted += review.refuted; + for (const severity of SEVERITIES) row.tally.findings[severity] += tally!.findings[severity]; + row.tally.fixed += tally!.fixed; + row.tally.refuted += tally!.refuted; + row.tally.open += tally!.open; } } } @@ -665,7 +662,8 @@ export async function usage(root: string, options: UsageOptions, deps: UsageDepe const reviewers = new Map(); for (const row of [...allRows].sort((a, b) => a.number - b.number)) for (const review of row.review) - reviewers.set(review.session, { number: row.number, effort: review.effort }); + if (review.session !== null) + reviewers.set(review.session, { number: row.number, effort: review.effort }); const left = attribute(priced, sessionLogs(priced, deps), all, byNumber, reviewers); if (!narrowed) { unattributed = { sessions: left.length, ...zero() }; @@ -718,7 +716,7 @@ export function usageLines(report: Usage): string[] { lines.push(` implementer: ${i.harness}, ${i.model}, ${i.effort} effort${i.size ? ` (${i.size})` : ""}`); for (const r of row.review) lines.push( - ` review: ${r.harness}, ${r.model}, ${r.effort} effort; round ${r.round}, score ${r.score ?? "-"}/10, ${r.findings} open, last round ${r.fixed} fixed and ${r.refuted} refuted`, + ` review: ${r.harness}, ${r.model}, ${r.effort} effort; ${r.round === null ? "not in the latest run" : `round ${r.round}, score ${r.score ?? "-"}/10`}; ${r.tally === null ? "no tally" : `rounds ${r.tally.rounds}, ${findingsText(r.tally)}`}`, ); if (row.landFixes.length) lines.push(` land fixes: ${row.landFixes.join("; ")}`); if (row.fixedBy.length) lines.push(` fixed later by: ${row.fixedBy.map((n) => `#${n}`).join(", ")}`); @@ -755,7 +753,7 @@ export function usageLines(report: Usage): string[] { a.harness, a.model, a.effort, - `${a.pulls} PRs, ${a.merged} merged, ${a.handedBack} handed back, ${a.reviewed} reviewed: score ${mean(a.score, a.scored)}, rounds ${mean(a.rounds, a.reviewed)}, ${a.findings} open, ${a.fixed} fixed, ${a.refuted} refuted; ${a.landFixes} land fixes, ${a.fixes} later fixes`, + `${a.pulls} PRs, ${a.merged} merged, ${a.handedBack} handed back, ${a.reviewed} reviewed: score ${mean(a.score, a.scored)}, rounds ${mean(a.tally.rounds, a.reviewed)}, ${findingsText(a.tally)}; ${a.landFixes} land fixes, ${a.fixes} later fixes`, ]), ), ); diff --git a/test/review.test.ts b/test/review.test.ts index c2358ff3..b08ed7cd 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -6,7 +6,10 @@ import { afterEach, expect, test, vi } from "vitest"; import { type Output, main } from "../src/cli/main"; import { REVIEWERS } from "../src/cli/models"; import { + type Finding, type Round, + type Settled, + type Tally, authored, baseInstructions, nextStep, @@ -15,6 +18,7 @@ import { runningHarness, parseState, sessionFile, + tallied, verdict, } from "../src/cli/review"; import { withReview } from "../src/cli/review-record"; @@ -446,7 +450,9 @@ fakes("a quest branch is reviewed at its quest's size on the base, which its own expect(sized.stdout).toContain("\nreviewer codex gpt-6.1-sol medium\n"); // A checkout on another branch, as the runner's review checkout is, takes the size of the branch it names. git(s.repo, "checkout", "-qb", "kix-review/job1"); - const [runner] = await review( + const { + rounds: [runner], + } = await review( { root: s.repo, base: "origin/main", sizeBranch: "quest/m0/big" }, { env: { ...process.env, CLAUDECODE: "1" } }, ); @@ -1290,7 +1296,13 @@ fakes("--new starts over, and a first round takes no responses", async () => { const fresh = await quest("--root", s.repo, "review", "--new", "--responses", responses); expect(fresh.code).toBe(2); expect(fresh.stderr).toContain("has none to resume"); - expect((await quest("--root", s.repo, "review", "--new")).stdout).toContain("round 1\n"); + const restarted = await quest("--root", s.repo, "review", "--new", "--json"); + expect(JSON.parse(restarted.stdout).rounds[0].round).toBe(1); + // The dropped sessions' findings stay open in their reviewers' tallies, which the new sessions continue. + expect(JSON.parse(restarted.stdout).tallies.map((t: Tally) => [t.reviewer, t.rounds, t.findings.high])).toEqual([ + ["codex", 2, 2], + ["opencode", 2, 2], + ]); }); /** Commit `edit` to origin's `next` branch, leaving the checkout on `feature`; no flags change. */ @@ -1457,12 +1469,14 @@ fakes("each round rewrites the Review section of the branch's open PR, keeping t "- Reviewer: Codex, gpt-6.1-sol, medium effort", "- Session: `thread-1`, round 1", "- Score: 8/10, `Lands once the off-by-one is fixed.`", + "- Rounds: 1; findings 1 (high 1): fixed 0, refuted 0, open 1", "- Findings, open until the next round settles them:", " - High, `src/lib.ts:2`: `Off by one`", "", "- Reviewer: opencode, opencode-go/deepseek-v4.1-flash, high effort", "- Session: `ses_1`, round 1", "- Score: 8/10, `Lands once the off-by-one is fixed.`", + "- Rounds: 1; findings 1 (high 1): fixed 0, refuted 0, open 1", "- Findings, open until the next round settles them:", " - High, `src/lib.ts:2`: `Off by one`", "", @@ -1494,6 +1508,8 @@ fakes("each round rewrites the Review section of the branch's open PR, keeping t expect(record).toContain(`- Head: ${git(s.repo, "rev-parse", "HEAD").slice(0, 12)},`); expect(record).toContain("- Session: `thread-1`, round 2\n"); expect(record).toContain("- Score: 9/10, `Fixes #3.`\n"); + // The refuted finding counts beside the new one, across both rounds. + expect(record).toContain("- Rounds: 2; findings 2 (high 2): fixed 0, refuted 1, open 1\n"); expect(record).toContain( ` - High, \`src/lib.ts\`: \`\`Ping @someone about \`${base.slice(0, 12)}\` in #12\`\`\n`, ); @@ -1846,7 +1862,16 @@ test("saved sessions are read only in the current format, or refused naming --ne last: {}, given: { instructions: "i", rules: null }, }; - const current = { branch: "feature", sessions: [session] }; + const tally = { + reviewer: "codex", + model: "m", + effort: "e", + rounds: 1, + findings: { high: 0, medium: 0, low: 1 }, + fixed: 0, + refuted: 0, + }; + const current = { branch: "feature", sessions: [session], tallies: [tally] }; expect(parseState(current)).toEqual(current); for (const older of [ null, @@ -1858,6 +1883,10 @@ test("saved sessions are read only in the current format, or refused naming --ne { sessions: [{ ...session, given: undefined }] }, { sessions: [{ ...session, session: undefined }] }, { sessions: [{ ...session, root: undefined }] }, + // Saved before tallies, or with a session no tally counts. + { branch: "feature", sessions: [session] }, + { branch: "feature", sessions: [session], tallies: [] }, + { branch: "feature", sessions: [session], tallies: [{ ...tally, findings: { high: 0, low: 1 } }] }, ]) expect(() => parseState(older)).toThrow(/--new/); }); @@ -1947,4 +1976,107 @@ fakes("a new size pick settles the open findings of the reviewer it replaces", a ]); const review = s.calls().find((call) => call.argv.includes("--session-id"))!; expect(review.stdin).toContain("Fixed at src/lib.ts:1."); + // Sol's finding stays Sol's, fixed, while the pick that replaced it counts its own. + expect( + JSON.parse(round.stdout).tallies.map((t: Tally) => [t.reviewer, t.rounds, t.findings.high, t.fixed]), + ).toEqual([ + ["claude", 1, 1, 0], + ["opencode", 2, 0, 0], + ["codex", 1, 1, 1], + ]); +}); + +test("a tally counts each finding once, by who raised it, through every round", () => { + const finding = (severity: Finding["severity"], title: string): Finding => ({ + file: "a.ts", + line: 1, + severity, + title, + body: "b", + }); + const settled = (f: Finding, disposition: Settled["disposition"], number: number): Settled => ({ + ...f, + number, + disposition, + reason: "r", + }); + const base = { + model: "m", + effort: "high", + branch: "feature", + size: "M", + sizeFrom: "default", + session: "s", + base: { ref: "origin/main", sha: "a".repeat(40) }, + head: "b".repeat(40), + usage: [], + score: 8, + rationale: "r", + earlier: null, + unchanged: false, + } satisfies Omit; + const [a, b, c] = [finding("high", "a"), finding("medium", "b"), finding("low", "c")]; + const first = tallied([], [{ round: { ...base, reviewer: "codex", round: 1, findings: [a, b, c] } }]); + // A kept-open finding is reported again, and its new severity replaces its old one. + const second = tallied(first, [ + { + round: { + ...base, + reviewer: "codex", + round: 2, + findings: [{ ...b, severity: "high" }], + earlier: { + head: base.head, + findings: [settled(a, "fixed", 1), settled(b, "open", 2), settled(c, "refuted", 3)], + }, + }, + settles: "codex", + }, + ]); + expect(second).toEqual([ + { + reviewer: "codex", + model: "m", + effort: "high", + rounds: 2, + findings: { high: 2, medium: 0, low: 1 }, + fixed: 1, + refuted: 1, + }, + ]); + expect(first[0]!.rounds).toBe(1); + // A round that repeats the last one counts nothing; a replacing reviewer takes over what it keeps open. + const third = tallied(second, [ + { round: { ...base, reviewer: "codex", round: 2, findings: [], unchanged: true } }, + { + round: { + ...base, + reviewer: "claude", + round: 1, + findings: [{ ...b, severity: "high" }], + earlier: { head: base.head, findings: [settled({ ...b, severity: "high" }, "open", 1)] }, + }, + settles: "codex", + }, + ]); + expect(third.map((t) => [t.reviewer, t.rounds, t.findings, t.fixed, t.refuted])).toEqual([ + ["codex", 2, { high: 1, medium: 0, low: 1 }, 1, 1], + ["claude", 1, { high: 1, medium: 0, low: 0 }, 0, 0], + ]); + expect(() => + tallied( + [], + [ + { + round: { + ...base, + reviewer: "claude", + round: 1, + findings: [], + earlier: { head: base.head, findings: [settled(a, "fixed", 1)] }, + }, + }, + ], + ), + ).toThrow(/no tally/); }); diff --git a/test/usage.test.ts b/test/usage.test.ts index b632fd4e..0a0a4ec3 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -6,7 +6,7 @@ import { describe, expect, test } from "vitest"; import { implementer } from "../src/github/models"; import { fixes } from "../src/cli/land-fixer"; -import type { Round } from "../src/cli/review"; +import type { Round, Tally } from "../src/cli/review"; import { parseReview, reviewRecord } from "../src/cli/review-record"; import { type Priced, parseCcusage, readClaudeLog, readCodexLog, usage, usageLines } from "../src/cli/usage"; import { tempDir } from "./fixture"; @@ -217,8 +217,30 @@ describe("PR records", () => { earlier: null, }; - test("reads back each reviewer's round quest review records, after the land fixer's lines", () => { - const body = `Summary.\n\n## Review\n\nmechanical fix by codex, gpt-6.1-sol, medium effort: \`merged\` (head ${"d".repeat(40)})\n\n${reviewRecord([round, second])}\n\n## Models\n\n- Size: L\n`; + const tally = ( + reviewer: Tally["reviewer"], + rounds: number, + findings: Tally["findings"], + fixed: number, + refuted: number, + ): Tally => ({ + reviewer, + model: "m", + effort: "high", + rounds, + findings, + fixed, + refuted, + }); + const tallies = [ + tally("codex", 3, { high: 1, medium: 1, low: 2 }, 2, 1), + tally("opencode", 1, { high: 0, medium: 0, low: 0 }, 0, 0), + tally("claude", 2, { high: 2, medium: 0, low: 0 }, 1, 0), + ]; + + test("reads back each reviewer's round and tally quest review records, after the land fixer's lines", () => { + const body = `Summary.\n\n## Review\n\nmechanical fix by codex, gpt-6.1-sol, medium effort: \`merged\` (head ${"d".repeat(40)})\n\n${reviewRecord([round, second], tallies)}\n\n## Models\n\n- Size: L\n`; + expect(body).toContain("\n- Rounds: 3; findings 4 (high 1, medium 1, low 2): fixed 2, refuted 1, open 1\n"); expect(parseReview(body)).toEqual([ { harness: "Codex", @@ -227,9 +249,7 @@ describe("PR records", () => { session: THREAD, round: 3, score: 7, - findings: 1, - fixed: 2, - refuted: 1, + tally: { rounds: 3, findings: { high: 1, medium: 1, low: 2 }, fixed: 2, refuted: 1, open: 1 }, }, { harness: "opencode", @@ -238,18 +258,41 @@ describe("PR records", () => { session: "ses_1", round: 1, score: 10, - findings: 0, - fixed: 0, - refuted: 0, + tally: { rounds: 1, findings: { high: 0, medium: 0, low: 0 }, fixed: 0, refuted: 0, open: 0 }, + }, + // A reviewer the run left out, such as the pick a new one replaced, keeps its tally. + { + harness: "Claude Code", + model: "m", + effort: "high", + session: null, + round: null, + score: null, + tally: { rounds: 2, findings: { high: 2, medium: 0, low: 0 }, fixed: 1, refuted: 0, open: 1 }, }, ]); // An open finding keeps the head short; once no reviewer reports one, it is named in full. expect(body).not.toContain("b".repeat(40)); - const clean = reviewRecord([{ ...round, findings: [] }, second]); + const clean = reviewRecord([{ ...round, findings: [] }, second], tallies); expect(clean).toContain(`- Head: ${"b".repeat(40)}\n`); expect(fixes(body)).toEqual(["codex, gpt-6.1-sol, medium effort"]); }); + test("a record written before tallies has none, and a Rounds line that does not add up fails", () => { + const record = reviewRecord([second], [tally("opencode", 1, { high: 0, medium: 0, low: 0 }, 0, 0)]); + const older = `## Review\n\n${record.replace(/^- Rounds: .*\n/m, "")}\n`; + expect(parseReview(older)[0]!.tally).toBeNull(); + for (const line of [ + "1; findings 1 (high 1): fixed 0, refuted 0, open 0", + "1; findings 2 (high 1): fixed 1, refuted 0, open 1", + "1; findings 1 (severe 1): fixed 1, refuted 0, open 0", + "one", + ]) + expect(() => + parseReview(`## Review\n\n${record.replace(/^- Rounds: .*$/m, `- Rounds: ${line}`)}\n`), + ).toThrow(/Rounds line/); + }); + test("a skipped review or a fixer-only section has no round", () => { expect(parseReview("## Review\n\nSkipped: too trivial to need one.\n")).toEqual([]); expect(parseReview("## Review\n\nmechanical fix by quest land watch: `merged main` (head x)\n")).toEqual([]); @@ -374,12 +417,18 @@ describe("usage", () => { "- Reviewer: Codex, gpt-6.1-sol, high effort", `- Session: \`${THREAD}\`, round 2`, "- Score: 9/10, `Fine.`", + "- Rounds: 2; findings 2 (high 1, low 1): fixed 1, refuted 1, open 0", "- Findings: none", "", "- Reviewer: opencode, opencode-go/deepseek-v4.1-flash, high effort", "- Session: `ses_1`, round 1", "- Score: 8/10, `Fine.`", + "- Rounds: 1; findings 0: fixed 0, refuted 0, open 0", "- Findings: none", + "", + // A reviewer the latest run left out, such as one a new size pick replaced, has only its tally. + "- Reviewer: Claude Code, claude-opus-5-5, high effort", + "- Rounds: 1; findings 1 (medium 1): fixed 1, refuted 0, open 0", ].join("\n"); // #9's claim of the same branch named ses_1 before #10 resumed it, so #10, the newer, takes it. const body9 = body10.slice(body10.indexOf("## Review")); @@ -492,11 +541,16 @@ describe("usage", () => { ["L", "reviewing", "codex", 1], ["L", "reviewing", "opencode", 4], ]); - expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.rounds, a.fixes])).toEqual([ - ["S", "gpt-6.1-sol", 1, 1, 17, 2, 1], - ["L", "claude-opus-5-5", 1, 1, 17, 2, 1], + const tally = { rounds: 2, findings: { high: 1, medium: 1, low: 1 }, fixed: 2, refuted: 1, open: 0 }; + expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ + ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], + ["L", "claude-opus-5-5", 1, 1, 17, tally, 1], ]); const text = usageLines(report).join("\n"); + expect(text).toContain( + " review: Claude Code, claude-opus-5-5, high effort; not in the latest run; rounds 1, findings 1 (medium 1): fixed 1, refuted 0, open 0\n", + ); + expect(text).toContain("rounds 2.0, findings 3 (high 1, medium 1, low 1): fixed 2, refuted 1, open 0;"); expect(text).toContain("#10 L merged: PR 10 (quest/m0/a)"); expect(text).toContain("fixed later by: #11"); expect(text).not.toContain("SECRET"); @@ -519,6 +573,6 @@ describe("usage", () => { [9, 0], [7, 0], ]); - expect(quest.pulls[0]!.review.map((r) => r.score)).toEqual([9, 8]); + expect(quest.pulls[0]!.review.map((r) => r.score)).toEqual([9, 8, null]); }); }); From 63267932b6737e2b909c270614c49917885783aa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 15:23:53 -0800 Subject: [PATCH 03/17] fix(review): drop --new's unsettled findings from tallies, count a branch's tally once Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++---- quest/m1/model-check.md | 4 ++-- src/cli/main.ts | 10 +++++----- src/cli/review.ts | 30 +++++++++++++++++++++++------- src/cli/usage.ts | 8 +++++++- test/review.test.ts | 10 ++++++---- test/usage.test.ts | 5 +++-- 7 files changed, 55 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 936d8048..9488e156 100644 --- a/README.md +++ b/README.md @@ -1408,7 +1408,8 @@ keeps its default. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in, so another worktree starts a new one; `--new` starts new sessions, but continues each reviewer's tally of its findings -on the branch, unless the saved state is one this quest cannot read. `--branch` names the branch instead of the checked-out one, which a +on the branch, less those it drops unsettled, unless the saved state is one +this quest cannot read. `--branch` names the branch instead of the checked-out one, which a detached `HEAD` requires, such as a checkout of a branch another worktree holds, so its rounds continue that branch's session. @@ -1467,8 +1468,10 @@ its location, and the last round's findings with their dispositions. The tally, as `- Rounds: 3; findings 4 (high 1, medium 3): fixed 3, refuted 1, open 0`, counts the reviewer's rounds on the branch and each finding it raised once, by its latest severity, however many rounds keep it open. A finding counts -for the reviewer that raised it, even when a new size pick's reviewer settles -it, and a reviewer the run left out keeps a block with only its tally. `quest status` takes the last full SHA the section names as the +for the reviewer that raised it when a new size pick's reviewer fixes or +refutes it, but one that reviewer keeps open becomes its own, since its round +reports it again. `--new` drops the findings it leaves unsettled from the +tallies. A reviewer the run left out keeps a block with only its tally. `quest status` takes the last full SHA the section names as the reviewed head, so the head appears in full only once no reviewer reports a finding, and model text appears as inline code, which mentions or links nothing and never holds a full @@ -1532,7 +1535,9 @@ findings across every round; the land fixer's fixes; and the later PRs that list Rollups follow: spend by size, role, harness, model, and effort, and accuracy by size and implementer, which sums the tallies, so it counts every finding -caught before merge; a record written before tallies counts as unreviewed there. An `Implementer:` line's own size, as +caught before merge. Since a branch's tallies span every PR from it, only its +newest PR in the report counts them, and a record written before tallies +counts as unreviewed. An `Implementer:` line's own size, as `(iterate, XS)`, counts over the PR's. Sessions on no PR here, from any repository or agent ccusage reads, total on an `Unattributed` line. `--pr` and `--quest` narrow the report to one PR or one quest's branch, and `--json` diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index 958fbd61..d1d2e8ac 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -30,8 +30,8 @@ with `quest usage --since `, whose accuracy rollup reads ea `## Models` size and implementers and its `## Review` reviewer. It tallies each implementer by the size on its own `Implementer:` line where it carries one, such as `(iterate, XS)`, not the PR's `## Models` size, so a small iterate on a large PR counts as the -work it was routed for. It counts fixed and refuted findings only from the -round before each PR's latest, the one its record keeps, and leaves design +work it was routed for. It counts each reviewer's findings, fixed, refuted, and open, +across every round on the branch, from the tally its record keeps, and leaves design checkpoints and outages to count by hand: - Tally by size and by implementing harness, model, and effort: review diff --git a/src/cli/main.ts b/src/cli/main.ts index 9c793474..aaa636d1 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1105,9 +1105,9 @@ export function main(argv: string[], out: Output): number | Promise { "before the PR opened, and iterating otherwise. An opencode session, whose log quest does not read, belongs to the newest PR whose " + "## Review record names it, as reviewing. Tokens and cost come from ccusage (session --json), the " + "cost an estimate at each model's API list price. Each PR also shows its ## Models size and implementers, " + - "each reviewer's latest ## Review round, with its round, score, and findings, " + + "each reviewer the ## Review record names, with its latest round and score and its tally of findings across every round, " + "and the later PRs that list it under ## Caused by. Rollups follow: spend by size, role, harness, model, and " + - "effort, and accuracy by size and implementer, where an Implementer line's own size, as (iterate, XS), wins " + + "effort, and accuracy by size and implementer, summing each branch's tallies once, where an Implementer line's own size, as (iterate, XS), wins " + "over the PR's. Without ccusage on PATH, it says how to install it and reports the PRs' records alone. " + "--json prints the same report as one object.\n\n" + "Uses GH_TOKEN, GITHUB_TOKEN, or gh auth token. Exits 0 when done, 2 for invalid arguments, and 1 for " + @@ -1205,7 +1205,7 @@ export function main(argv: string[], out: Output): number | Promise { "starts a new session that reviews the whole change and still weighs --responses. A round that would " + "resume on the head the last round reviewed, when that round found nothing, runs no reviewer and " + "repeats that round, marked unchanged, so on the same head only a reviewer with findings to settle runs. " + - "--new starts new sessions, continuing each reviewer's tally of its findings on the branch. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + + "--new starts new sessions, continuing each reviewer's tally of its findings on the branch, less those it drops unsettled. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + "A checked-out branch with an upstream other than the base continues its upstream branch's session, so a " + "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + @@ -1230,11 +1230,11 @@ export function main(argv: string[], out: Output): number | Promise { "`next checkpoint` when a reviewer's every third round reports other findings, to reconsider the approach " + "before fixing them, `next respond` to fix or refute the findings, or `next done`. A reply without a valid score " + "fails as any malformed reply does. --json prints " + - "one object: `rounds`, one per reviewer, `record`, and `next`.\n\n" + + "one object: `rounds`, one per reviewer, `tallies`, every reviewer's findings across the branch's rounds, `record`, and `next`.\n\n" + "After each round, it rewrites the `## Review` section of the branch's open PR into the base, as the " + "user the GitHub token names, keeping the rest of the body, and prints `record `: the reviewed " + "head, named in full only once no reviewer reports a finding, the base, and each reviewer, its " + - "session and round, score, and each finding with its disposition. With no " + + "session and round, score, its tally of findings across the branch's rounds, and each finding with its disposition. With no " + "such PR, or a remote off GitHub, it prints `record skipped: ` and changes nothing.\n\n" + "Exits 0 once the round finishes, whatever it found, 2 for invalid arguments " + "or a checkout or CLI that cannot be reviewed, including a login the API rejects, 3 when a reviewer's " + diff --git a/src/cli/review.ts b/src/cli/review.ts index 5149131d..fdbfa54d 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -303,10 +303,6 @@ export interface ReviewDependencies { export const processDependencies = (): ReviewDependencies => ({ env: process.env }); -/** - * A branch's review sessions, one per reviewer, kept in the clone's common Git directory so every worktree - * resumes them. They are in the order the last run reported them, which numbers their findings. - */ /** One reviewer's findings across every round on a branch, which a `--new` session continues. */ export interface Tally { reviewer: ReviewHarness; @@ -314,7 +310,10 @@ export interface Tally { model: string; effort: string; rounds: number; - /** Each finding it raised, once however many rounds keep it open, by its latest severity. */ + /** + * Each finding it raised, once however many rounds keep it open, by its latest severity. One a replacing + * reviewer keeps open becomes that reviewer's, since its round reports it again unlinked to the earlier one. + */ findings: Record<(typeof SEVERITIES)[number], number>; fixed: number; refuted: number; @@ -326,6 +325,10 @@ export interface Reviewed { tallies: Tally[]; } +/** + * A branch's review sessions, one per reviewer, kept in the clone's common Git directory so every worktree + * resumes them. They are in the order the last run reported them, which numbers their findings. + */ interface State { branch: string; sessions: Session[]; @@ -1674,7 +1677,7 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): if (controller.signal.aborted) throw controller.signal.reason; const done = results.map((result) => (result as PromiseFulfilledResult).value); const tallies = tallied( - saved?.tallies ?? [], + options.fresh ? dropped(saved) : (saved?.tallies ?? []), done.map((each, i) => ({ round: each.round, settles: seats[i]!.session?.reviewer })), ); save(statePath, { branch, sessions: [...done.map((each) => each.session), ...kept], tallies } satisfies State); @@ -1683,7 +1686,7 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): /** * A branch's saved state, or undefined when it has none. `--new` drops the sessions but continues their tallies, - * so it reads them too, starting them over only when the state is one this quest cannot read, which is what + * less the findings it drops unsettled, so it reads them too, starting them over only when the state is one this quest cannot read, which is what * `--new` is for. */ function savedState(path: string, fresh: boolean | undefined): State | undefined { @@ -1696,6 +1699,19 @@ function savedState(path: string, fresh: boolean | undefined): State | undefined } } +/** + * The saved tallies without the findings `--new` drops unsettled, so a finding no session can settle stays + * open nowhere, and one the new session reports again counts once. + */ +function dropped(saved: State | undefined): Tally[] { + const tallies = (saved?.tallies ?? []).map((tally) => ({ ...tally, findings: { ...tally.findings } })); + for (const session of saved?.sessions ?? []) { + const tally = tallies.find((each) => each.reviewer === session.reviewer)!; + for (const finding of session.rounds.at(-1)!.findings) tally.findings[finding.severity]--; + } + return tallies; +} + /** * `tallies` after a run's `rounds`, the run's reviewers first. A round counts its findings for its own reviewer * and settles the last round's for the reviewer that raised them, `settles`, which is another when the round diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 5c965099..437df8dc 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -564,6 +564,11 @@ function spendRows(pulls: PullRow[]): SpendRow[] { function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const rows = new Map(); + // A branch's saved tallies span every PR from it, so only its newest PR with a tally counts them. + const newest = new Map(); + for (const each of pulls) + if (each.review.some((review) => review.tally !== null)) + newest.set(each.branch, Math.max(newest.get(each.branch) ?? 0, each.number)); for (const each of pulls) { // A session that pushed twice is one implementer of the PR, whether or not each line names its size. const distinct = new Map( @@ -599,7 +604,8 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { if (each.handedBack) row.handedBack++; row.fixes += each.fixedBy.length; row.landFixes += each.landFixes.filter((by) => by !== MERGER).length; - const tallied = each.review.filter((review) => review.tally !== null); + const tallied = + newest.get(each.branch) === each.number ? each.review.filter((review) => review.tally !== null) : []; if (tallied.length > 0) { row.reviewed++; row.tally.rounds += Math.max(...tallied.map((review) => review.tally!.rounds)); diff --git a/test/review.test.ts b/test/review.test.ts index b08ed7cd..0c717a55 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -1298,10 +1298,12 @@ fakes("--new starts over, and a first round takes no responses", async () => { expect(fresh.stderr).toContain("has none to resume"); const restarted = await quest("--root", s.repo, "review", "--new", "--json"); expect(JSON.parse(restarted.stdout).rounds[0].round).toBe(1); - // The dropped sessions' findings stay open in their reviewers' tallies, which the new sessions continue. - expect(JSON.parse(restarted.stdout).tallies.map((t: Tally) => [t.reviewer, t.rounds, t.findings.high])).toEqual([ - ["codex", 2, 2], - ["opencode", 2, 2], + // The new sessions continue their reviewers' tallies, less the findings the dropped sessions left unsettled. + expect( + JSON.parse(restarted.stdout).tallies.map((t: Tally) => [t.reviewer, t.rounds, t.findings.high, t.fixed]), + ).toEqual([ + ["codex", 2, 1, 0], + ["opencode", 2, 1, 0], ]); }); diff --git a/test/usage.test.ts b/test/usage.test.ts index 0a0a4ec3..55cd7953 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -431,7 +431,8 @@ describe("usage", () => { "- Rounds: 1; findings 1 (medium 1): fixed 1, refuted 0, open 0", ].join("\n"); // #9's claim of the same branch named ses_1 before #10 resumed it, so #10, the newer, takes it. - const body9 = body10.slice(body10.indexOf("## Review")); + // Its record and #10's tally the same branch, so the rollup counts the tally once, from #10. + const body9 = `${body10.slice(body10.indexOf("## Review"))}\n\n## Models\n\n- Size: L\n- Implementer: Claude Code, claude-opus-5-5, high effort\n`; const body11 = "Fix.\n\n## Caused by\n\n- [acme/proj#10](https://github.com/acme/proj/pull/10) - missed a case\n"; const pulls = [ @@ -544,7 +545,7 @@ describe("usage", () => { const tally = { rounds: 2, findings: { high: 1, medium: 1, low: 1 }, fixed: 2, refuted: 1, open: 0 }; expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], - ["L", "claude-opus-5-5", 1, 1, 17, tally, 1], + ["L", "claude-opus-5-5", 2, 1, 17, tally, 1], ]); const text = usageLines(report).join("\n"); expect(text).toContain( From 0e96a167438c697fe71dd12eaaaeefa313422888 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 15:37:00 -0800 Subject: [PATCH 04/17] fix(usage): count a branch's tally once per head repository, from a PR with implementers Co-Authored-By: Claude Opus 5.5 --- src/cli/usage.ts | 13 +++++++++---- test/usage.test.ts | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 437df8dc..4ae3ea50 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -61,6 +61,8 @@ export interface PullRow { url: string; title: string; branch: string; + /** The head repository, lowercased as `owner/repo`, or null once a fork's is deleted. */ + repository: string | null; state: "open" | "closed" | "merged"; draft: boolean; /** Its body has a `## Waiting` section, so it was handed back to the maintainer. */ @@ -399,6 +401,7 @@ function pullRows(pulls: Pull[], repository: Repository): PullRow[] { url: each.url, title: each.title, branch: each.branch, + repository: each.repository, state: each.state, draft: each.draft, handedBack: each.draft && handedBack(each.body), @@ -564,11 +567,13 @@ function spendRows(pulls: PullRow[]): SpendRow[] { function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const rows = new Map(); - // A branch's saved tallies span every PR from it, so only its newest PR with a tally counts them. + // A branch's saved tallies span every PR from it, so only its newest PR that has both a tally and an + // implementer to count it for counts them. + const branch = (each: PullRow) => JSON.stringify([each.repository, each.branch]); const newest = new Map(); for (const each of pulls) - if (each.review.some((review) => review.tally !== null)) - newest.set(each.branch, Math.max(newest.get(each.branch) ?? 0, each.number)); + if (each.implementers.length > 0 && each.review.some((review) => review.tally !== null)) + newest.set(branch(each), Math.max(newest.get(branch(each)) ?? 0, each.number)); for (const each of pulls) { // A session that pushed twice is one implementer of the PR, whether or not each line names its size. const distinct = new Map( @@ -605,7 +610,7 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { row.fixes += each.fixedBy.length; row.landFixes += each.landFixes.filter((by) => by !== MERGER).length; const tallied = - newest.get(each.branch) === each.number ? each.review.filter((review) => review.tally !== null) : []; + newest.get(branch(each)) === each.number ? each.review.filter((review) => review.tally !== null) : []; if (tallied.length > 0) { row.reviewed++; row.tally.rounds += Math.max(...tallied.map((review) => review.tally!.rounds)); diff --git a/test/usage.test.ts b/test/usage.test.ts index 55cd7953..810d25c9 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -446,7 +446,8 @@ describe("usage", () => { }), // A fork's PR whose head shares #10's branch name, opened earlier and still open: no session here is // shown to be its. - pr(7, "quest/m0/a", "", "2026-11-03T00:00:00Z", { + // Its tally is the fork's own, so it counts beside #10's. + pr(7, "quest/m0/a", body9, "2026-11-03T00:00:00Z", { state: "open", merged_at: null, closed_at: null, @@ -454,6 +455,14 @@ describe("usage", () => { head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, }), pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), + // The fork's newer PR has no implementer to count its tally for, so #7 still counts the fork's. + pr(13, "quest/m0/a", body10.slice(body10.indexOf("## Review")), "2026-11-04T00:00:00Z", { + state: "open", + merged_at: null, + closed_at: null, + updated_at: "2026-11-06T00:00:00Z", + head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, + }), ]; const requests: string[] = []; const fetch = async (input: string | URL | Request) => { @@ -543,9 +552,10 @@ describe("usage", () => { ["L", "reviewing", "opencode", 4], ]); const tally = { rounds: 2, findings: { high: 1, medium: 1, low: 1 }, fixed: 2, refuted: 1, open: 0 }; + const double = { rounds: 4, findings: { high: 2, medium: 2, low: 2 }, fixed: 4, refuted: 2, open: 0 }; expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], - ["L", "claude-opus-5-5", 2, 1, 17, tally, 1], + ["L", "claude-opus-5-5", 3, 2, 34, double, 1], ]); const text = usageLines(report).join("\n"); expect(text).toContain( From fcd3fed8139890f15bada78c7a6442d20107c0f5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 15:59:53 -0800 Subject: [PATCH 05/17] test(usage): fetch the no-implementer fork PR so its dedupe case is exercised Co-Authored-By: Claude Opus 5.5 --- test/usage.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/usage.test.ts b/test/usage.test.ts index 810d25c9..dc958ee8 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -433,6 +433,10 @@ describe("usage", () => { // #9's claim of the same branch named ses_1 before #10 resumed it, so #10, the newer, takes it. // Its record and #10's tally the same branch, so the rollup counts the tally once, from #10. const body9 = `${body10.slice(body10.indexOf("## Review"))}\n\n## Models\n\n- Size: L\n- Implementer: Claude Code, claude-opus-5-5, high effort\n`; + const fork13 = body10 + .slice(body10.indexOf("## Review")) + .replaceAll(THREAD, "thread-fork") + .replaceAll("ses_1", "ses_fork"); const body11 = "Fix.\n\n## Caused by\n\n- [acme/proj#10](https://github.com/acme/proj/pull/10) - missed a case\n"; const pulls = [ @@ -454,15 +458,16 @@ describe("usage", () => { updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, }), - pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), // The fork's newer PR has no implementer to count its tally for, so #7 still counts the fork's. - pr(13, "quest/m0/a", body10.slice(body10.indexOf("## Review")), "2026-11-04T00:00:00Z", { + // Its own sessions, so it takes none of #10's. + pr(13, "quest/m0/a", fork13, "2026-11-04T00:00:00Z", { state: "open", merged_at: null, closed_at: null, updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, }), + pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), ]; const requests: string[] = []; const fetch = async (input: string | URL | Request) => { @@ -518,7 +523,7 @@ describe("usage", () => { test("ties each session to its PR and role, and rolls up spend and accuracy", async () => { const { root, deps, priced } = setup(); const report = await usage(root, { since: "2026-11-05" }, deps(priced)); - expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7]); + expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7, 13]); const ten = report.pulls.find((p) => p.number === 10)!; expect(ten.sessions.map((s) => [s.harness, s.session, s.role, s.effort])).toEqual([ ["claude", CLAUDE, "implementing", "high"], @@ -583,6 +588,7 @@ describe("usage", () => { [10, 0], [9, 0], [7, 0], + [13, 0], ]); expect(quest.pulls[0]!.review.map((r) => r.score)).toEqual([9, 8, null]); }); From 00fec4b6c73e521f95d120d04505d62afc36b44f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 18:12:12 -0800 Subject: [PATCH 06/17] feat(review): settle low findings with --settle so tallies count them A round whose findings are all low ends without another round, and the author used to settle them in a hand-written line after the record, which left them open in the tally. `quest review --settle --responses` records each disposition in the tally and the PR's record without running a reviewer. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 +++++ assets/skills/review.md | 2 +- src/cli/main.ts | 31 +++++++++---- src/cli/review-record.ts | 8 +++- src/cli/review.ts | 93 ++++++++++++++++++++++++++++++++++++++- test/review.test.ts | 94 ++++++++++++++++++++++++++++++++++++++++ test/usage.test.ts | 1 + 7 files changed, 227 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index db3320a5..beaa89fd 100644 --- a/README.md +++ b/README.md @@ -1365,6 +1365,7 @@ and counts as logged out when `opencode models` does not list the model. ```sh quest review # round 1 against origin's default branch quest review --responses .tmp/responses.md # round 2: the new head, with fixes and refutations +quest review --settle --responses .tmp/responses.md # settle a round's low findings without a reviewer quest review --base origin/release --new # a new session against another base quest review --size S --exclude-model gpt-6.1-sol # an S PR that Sol wrote, with no record in its body quest review --branch quest/m1/foo --responses .tmp/responses.md # a detached checkout of that branch's head @@ -1413,6 +1414,15 @@ this quest cannot read. `--branch` names the branch instead of the checked-out o detached `HEAD` requires, such as a checkout of a branch another worktree holds, so its rounds continue that branch's session. +A round whose findings are all low severity needs no further round. After +fixing or refuting them, `--settle` runs no reviewer: it reads `--responses` as +one `. fixed: ` or `. refuted: ` line per finding, by its +number, adds each disposition to its reviewer's tally, and drops the findings +from the session, so its next round checks what changed since their head with +nothing to settle. It prints and records the last run's rounds at the +committed head, which the record names in full, marking the findings the +author settled. + The reviewer is read-only: Codex in its `read-only` sandbox, ignoring execpolicy rules, and Claude Code with only read tools and the Bash commands it judges read-only, in `dontAsk` mode. Its instructions say the base's files and diff --git a/assets/skills/review.md b/assets/skills/review.md index a5a8b6e0..55c6c66d 100644 --- a/assets/skills/review.md +++ b/assets/skills/review.md @@ -35,7 +35,7 @@ Refute a finding only with evidence the reviewer cannot rebut: a `path:line`, a Commit the fixes, rerun the checks, and run the next round with `--responses`, one response per finding by its number, written in the checkout's `.tmp/`. Repeat until a round of the head reports no finding you have not refuted; never block on the score. A finding the reviewer keeps against such a refutation is a decision for the user; a background agent lists it under `## Waiting`. -A round whose findings are all low severity needs no further round: fix or refute each, then add a line after the record naming the new head's full SHA and each finding's disposition. +A round whose findings are all low severity needs no further round: fix or refute each, commit, rerun the checks, then record their dispositions with `quest review --settle --responses `, one `. fixed: ` or `. refuted: ` line per finding. Checkpoint the design: a round is one `quest review` round, numbered within each reviewer's session on the branch, and when a round whose number is a multiple of three reports findings, stop before fixing them and reconsider the approach; a clean round needs no checkpoint. Findings that keep coming usually mean the approach is wrong, not the last fix: brainstorm other designs that would avoid the whole class of findings, favoring simpler ones, and compare them with the current one. diff --git a/src/cli/main.ts b/src/cli/main.ts index 3da58382..f322097a 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -53,6 +53,7 @@ import { ReviewLimitError, processDependencies as reviewDependencies, review, + settleLow, nextStep, roundLines, } from "./review"; @@ -1214,6 +1215,11 @@ export function main(argv: string[], out: Output): number | Promise { "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + "checkout of a branch another worktree holds.\n\n" + + "--settle runs no reviewer: it settles the last round's findings, which must all be low severity, from " + + "--responses, one `. fixed: ` or `. refuted: ` line per finding by its number, adding " + + "them to each reviewer's tally and dropping them from its session, so its next round checks what changed " + + "since their head with nothing to settle. It prints and records the last run's rounds at the committed " + + "head, naming that head in full and each settled finding's disposition.\n\n" + "The reviewer is read-only: Codex in its read-only sandbox without execpolicy rules, user config, or its " + "skills catalog, Claude Code with " + "no settings files, only read tools, and the Bash commands it judges read-only, and opencode with no " + @@ -1230,7 +1236,7 @@ export function main(argv: string[], out: Output): number | Promise { "part read from the prompt cache, and output), a score from 0 to 10 for how ready the head is to land with " + "its one-sentence rationale, the finding count, each finding by its number, then the last round's findings with " + "their numbers and dispositions; then the `record` line below and a last `next` line naming the author's next step: " + - "`next finish` when every finding is low severity, to fix or refute them without another round, " + + "`next finish` when every finding is low severity, to fix or refute them and settle them with --settle, " + "`next checkpoint` when a reviewer's every third round reports other findings, to reconsider the approach " + "before fixing them, `next respond` to fix or refute the findings, or `next done`. A reply without a valid score " + "fails as any malformed reply does. --json prints " + @@ -1270,6 +1276,12 @@ export function main(argv: string[], out: Output): number | Promise { ) .option("--responses ", "The author's responses to the last round's findings; - reads stdin") .option("--new", "Start a new review session instead of resuming the branch's") + .addOption( + new Option( + "--settle", + "Settle the last round's low findings from --responses without running a reviewer", + ).conflicts(["harness", "model", "effort", "excludeModel", "size", "new"]), + ) .option( "--branch ", "Branch whose review session to continue, instead of the checked-out one or its upstream", @@ -1286,6 +1298,7 @@ export function main(argv: string[], out: Output): number | Promise { size?: Size; responses?: string; new?: boolean; + settle?: boolean; json?: boolean; }) => { const responses = @@ -1293,13 +1306,15 @@ export function main(argv: string[], out: Output): number | Promise { ? undefined : readFileSync(options.responses === "-" ? 0 : options.responses, "utf8"); const github = { github: () => gitHubDependencies(warn), warn }; - const run = review( - { ...options, root: root(), exclude: options.excludeModel, responses, fresh: options.new }, - { - ...reviewDependencies(), - models: (dir, base, branch) => pullModels(dir, base, branch, github), - }, - ); + const run = options.settle + ? settleLow({ ...options, root: root(), responses }) + : review( + { ...options, root: root(), exclude: options.excludeModel, responses, fresh: options.new }, + { + ...reviewDependencies(), + models: (dir, base, branch) => pullModels(dir, base, branch, github), + }, + ); pending = run.then( async ({ rounds, tallies }) => { // The rounds are saved in their sessions either way, so their report prints before a failed record. diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index 6a4e3e44..dd3c6441 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -56,11 +56,12 @@ const counted = (tally: Tally): Counts => ({ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { const [shared] = rounds; const clean = rounds.every((round) => round.findings.length === 0); + const author = rounds.find((round) => round.settled)?.earlier; const lines = [ MARK, "", // `quest status` counts a full SHA here as a review of that head, which an open finding has not passed. - `- Head: ${clean ? shared!.head : `${shared!.head.slice(0, 12)}, named in full once no reviewer reports a finding`}`, + `- Head: ${clean ? shared!.head : `${shared!.head.slice(0, 12)}, named in full once no reviewer reports a finding`}${author ? `, where the author settled the low findings reviewed at ${author.head.slice(0, 12)} with no further round` : ""}`, `- Base: ${shared!.base.ref} at ${shared!.base.sha.slice(0, 12)}`, ]; for (const round of rounds) { @@ -78,7 +79,10 @@ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { for (const finding of round.findings) lines.push(` - ${capitalized(finding.severity)}, ${code(located(finding))}: ${code(finding.title)}`); const settled = round.earlier?.findings ?? []; - if (settled.length > 0) lines.push(`- The last round's findings, at ${round.earlier!.head.slice(0, 12)}:`); + if (settled.length > 0) + lines.push( + `- The last round's findings, at ${round.earlier!.head.slice(0, 12)}${round.settled ? ", settled by the author" : ""}:`, + ); for (const finding of settled) lines.push( ` - ${capitalized(finding.disposition)}: ${finding.severity}, ${code(located(finding))}: ${code(finding.title)}; ${code(finding.reason)}`, diff --git a/src/cli/review.ts b/src/cli/review.ts index fdbfa54d..93a959bb 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -106,6 +106,8 @@ export interface Round { earlier: { head: string; findings: Settled[] } | null; /** No reviewer ran: the head is the last round's, which found nothing, so this repeats that round. */ unchanged: boolean; + /** No reviewer ran: the author settled the last round's low findings, `earlier`, at `head`, with `--settle`. */ + settled: boolean; } /** A reply's verdict on the reviewed head; `earlier` settles the last round's findings by their number. */ @@ -1684,6 +1686,89 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): return { rounds: done.map((each) => each.round), tallies }; } +/** + * Settle the last round's findings, all low severity, from the author's `responses` without running a reviewer: + * one `. fixed: ` or `. refuted: ` line per finding, by its number. Each reviewer's tally + * counts them, and its session forgets them, so its next round checks what changed since their head with nothing + * to settle. Gives the last run's rounds at the checked-out head, those the author settled marked `settled`. + */ +export async function settleLow( + options: Pick & BaseOptions, +): Promise { + const root = git(options.root, ["rev-parse", "--show-toplevel"]); + if (git(root, ["status", "--porcelain"]) !== "") + throw new ReviewInputError( + "the checkout has uncommitted changes; commit them, since the record names the committed head", + ); + const baseRef = options.fetched ? options.base : await fetchBase(root, options.base); + const branch = sessionBranch(root, options.branch, parseBase(root, baseRef).branch); + const statePath = sessionFile(git(root, ["rev-parse", "--path-format=absolute", "--git-common-dir"]), branch); + const saved = savedState(statePath, false); + // Numbered as the last run printed them, across the sessions in order. + const open = (saved?.sessions ?? []).flatMap((session) => + session.rounds.at(-1)!.findings.map((finding) => ({ session, finding })), + ); + if (open.length === 0) throw new ReviewInputError("the branch's last round left no findings to settle"); + if (open.some(({ finding }) => finding.severity !== "low")) + throw new ReviewInputError( + "--settle settles only findings that are all low severity; run the next round with --responses", + ); + const reviewed = open[0]!.session.rounds.at(-1)!.head; + if (open.some(({ session }) => session.rounds.at(-1)!.head !== reviewed)) + throw new Error("the findings to settle are from rounds of different heads"); + if (options.responses === undefined) + throw new ResponsesError( + "--settle needs --responses, one `. fixed: ` or `. refuted: ` line per finding", + ); + const answers = new Map(); + for (const line of options.responses.split("\n").map((each) => each.trim())) { + if (line === "") continue; + const match = /^(\d+)\. (fixed|refuted): (.+)$/i.exec(line); + if (!match) + throw new ResponsesError( + `a --settle response is not \`. fixed: \` or \`. refuted: \`: ${line}`, + ); + const number = Number(match[1]); + if (number < 1 || number > open.length || answers.has(number)) + throw new ResponsesError( + `--settle settles each of the last round's ${open.length} finding(s) once: ${line}`, + ); + answers.set(number, { disposition: match[2]!.toLowerCase() as "fixed" | "refuted", reason: match[3]! }); + } + if (answers.size !== open.length) + throw new ResponsesError(`--settle settles each of the last round's ${open.length} finding(s) once`); + + const head = git(root, ["rev-parse", "HEAD"]); + const tallies = saved!.tallies.map((tally) => ({ ...tally, findings: { ...tally.findings } })); + const rounds: Round[] = []; + const sessions = saved!.sessions.map((session): Session => { + const own = open.flatMap(({ session: raiser, finding }, i) => + raiser === session ? [{ ...finding, number: i + 1, ...answers.get(i + 1)! }] : [], + ); + if (own.length > 0) { + const tally = tallies.find((each) => each.reviewer === session.reviewer)!; + for (const finding of own) tally[finding.disposition]++; + const last: Round = { + ...session.last, + findings: [], + earlier: { head: reviewed, findings: own }, + settled: true, + }; + rounds.push({ ...last, head }); + return { + ...session, + rounds: [...session.rounds.slice(0, -1), { ...session.rounds.at(-1)!, findings: [] }], + last, + }; + } + // A reviewer of the same run found nothing; it stands beside the settled ones in the record. + if (session.rounds.at(-1)!.head === reviewed) rounds.push({ ...session.last, head }); + return session; + }); + save(statePath, { ...saved!, sessions, tallies } satisfies State); + return { rounds, tallies }; +} + /** * A branch's saved state, or undefined when it has none. `--new` drops the sessions but continues their tallies, * less the findings it drops unsettled, so it reads them too, starting them over only when the state is one this quest cannot read, which is what @@ -1902,6 +1987,7 @@ async function reviewWith(shared: Shared, seat: Seat, responses: string | undefi })), }, unchanged: false, + settled: false, }; return { round, session: { reviewer: harness, session, root, rounds, last: round, given } }; } @@ -1916,7 +2002,7 @@ export function nextStep(rounds: Round[]): string { const findings = rounds.flatMap((round) => round.findings); if (findings.length === 0) return "next done: no findings remain on this head"; if (findings.every((finding) => finding.severity === "low")) - return "next finish: fix or refute each finding, commit, rerun the checks, then add a line after the PR body's review record naming the new head's full SHA and each finding's disposition; run no further round"; + return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --settle --responses with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; if (rounds.some((round) => round.findings.length > 0 && round.round % 3 === 0)) return "next checkpoint: before fixing these findings, reconsider the approach and let the user decide, as /quest-review describes"; return "next respond: fix each finding or refute it with evidence the reviewer cannot rebut, commit, rerun the checks, then run the next round with --responses, one response per finding by its number"; @@ -1943,6 +2029,11 @@ export function roundLines(rounds: Round[]): string[] { ...(round.unchanged ? [`unchanged round ${round.round} reviewed this head and found nothing; no reviewer ran`] : []), + ...(round.settled + ? [ + `settled the author settled round ${round.round}'s low findings at ${round.earlier!.head}; no reviewer ran`, + ] + : []), ...round.usage.map((u) => `usage ${u.turn} input ${u.input} cached ${u.cached} output ${u.output}`), `score ${round.score} ${round.rationale.replace(/\s+/g, " ").trim()}`, `findings ${round.findings.length}`, diff --git a/test/review.test.ts b/test/review.test.ts index ca434a17..456c58d7 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2016,6 +2016,7 @@ test("a tally counts each finding once, by who raised it, through every round", rationale: "r", earlier: null, unchanged: false, + settled: false, } satisfies Omit; const [a, b, c] = [finding("high", "a"), finding("medium", "b"), finding("low", "c")]; const first = tallied([], [{ round: { ...base, reviewer: "codex", round: 1, findings: [a, b, c] } }]); @@ -2082,3 +2083,96 @@ test("a tally counts each finding once, by who raised it, through every round", ), ).toThrow(/no tally/); }); + +fakes("--settle records low findings' dispositions without running a reviewer", async () => { + const t = setup(); + expect((await quest("--root", t.repo, "review")).code).toBe(0); + write(t.repo, ".tmp/responses.md", "1. fixed: src/lib.ts:1.\n2. fixed: src/lib.ts:1.\n"); + const high = await quest( + "--root", + t.repo, + "review", + "--settle", + "--responses", + join(t.repo, ".tmp", "responses.md"), + ); + expect(high.code).toBe(2); + expect(high.stderr).toContain("all low severity"); + + const s = setup(); + const pull = { + number: 7, + body: "Fixes averaging.\n\n## Models\n\n- Size: S\n- Implementer: Claude Code, claude-opus-5-5, high effort", + }; + const writes = onGitHub(s.repo, [pull]); + try { + vi.stubEnv( + "FAKE_REPLY", + JSON.stringify({ score: 9, rationale: "Nits.", findings: [{ ...FINDING, severity: "low" }] }), + ); + vi.stubEnv("FAKE_OPENCODE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); + const first = await quest("--root", s.repo, "review"); + expect(first.code, first.stderr).toBe(0); + expect(first.stdout).toMatch(/\nnext finish: .*--settle/); + const reviewed = git(s.repo, "rev-parse", "HEAD"); + write(s.repo, "src/lib.ts", "export const x = 4;\n"); + git(s.repo, "commit", "-qam", "fix"); + const head = git(s.repo, "rev-parse", "HEAD"); + const calls = s.calls().length; + const responses = join(s.repo, ".tmp", "responses.md"); + for (const bad of ["1 fixed src/lib.ts:1\n", "", "1. fixed: a\n1. refuted: b\n", "2. fixed: a\n"]) { + write(s.repo, ".tmp/responses.md", bad); + const refused = await quest("--root", s.repo, "review", "--settle", "--responses", responses); + expect(refused.code, bad).toBe(2); + } + write(s.repo, ".tmp/responses.md", "1. Fixed: src/lib.ts:1 guards it.\n"); + const settled = await quest("--root", s.repo, "review", "--settle", "--responses", responses); + expect(settled.code, settled.stderr).toBe(0); + expect(s.calls()).toHaveLength(calls); + expect(settled.stdout).toContain(`head ${head}\n`); + expect(settled.stdout).toContain( + `settled the author settled round 1's low findings at ${reviewed}; no reviewer ran\n`, + ); + expect(settled.stdout).toContain( + `earlier 1 ${reviewed}\n\n1. fixed low src/lib.ts:2 Off by one\n src/lib.ts:1 guards it.\n`, + ); + expect(settled.stdout).toMatch(/\nnext done: no findings remain on this head\n$/); + const record = pull.body.split("## Review\n\n")[1]!; + expect(record).toContain( + `- Head: ${head}, where the author settled the low findings reviewed at ${reviewed.slice(0, 12)} with no further round\n`, + ); + expect(record).toContain( + "- Rounds: 1; findings 1 (low 1): fixed 1, refuted 0, open 0\n- Findings: none\n" + + `- The last round's findings, at ${reviewed.slice(0, 12)}, settled by the author:\n` + + " - Fixed: low, `src/lib.ts:2`: `Off by one`; `src/lib.ts:1 guards it.`\n", + ); + // The clean reviewer stands beside it, its tally unchanged. + expect(record).toContain( + "- Session: `ses_1`, round 1\n- Score: 10/10, `Lands as is.`\n- Rounds: 1; findings 0: fixed 0, refuted 0, open 0\n", + ); + expect(writes).toHaveLength(2); + + // Nothing is left to settle, and the next round checks the fix with no responses, counting the finding once. + expect((await quest("--root", s.repo, "review", "--settle", "--responses", responses)).code).toBe(2); + vi.stubEnv("FAKE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); + const next = await quest("--root", s.repo, "review", "--json"); + expect(next.code, next.stderr).toBe(0); + const resumed = s.calls().at(-1)!; + expect(resumed.argv.slice(0, 2)).toEqual(["exec", "resume"]); + expect(resumed.stdin).toContain(`git diff ${reviewed} ${head}`); + expect( + JSON.parse(next.stdout).tallies.map((t: Tally) => [ + t.reviewer, + t.rounds, + t.findings.low, + t.fixed, + t.refuted, + ]), + ).toEqual([ + ["codex", 2, 1, 1, 0], + ["opencode", 2, 0, 0, 0], + ]); + } finally { + vi.unstubAllGlobals(); + } +}); diff --git a/test/usage.test.ts b/test/usage.test.ts index e590ab15..40f405f6 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -204,6 +204,7 @@ describe("PR records", () => { ], }, unchanged: false, + settled: false, }; const second: Round = { From a6e2b9060663a7f37cd1f429b56d7908ee9a94cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 18:13:54 -0800 Subject: [PATCH 07/17] fix(review): --settle reports no reviewer usage Co-Authored-By: Claude Opus 5.5 --- src/cli/review.ts | 4 ++-- test/review.test.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli/review.ts b/src/cli/review.ts index 93a959bb..776b4cd0 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -1754,7 +1754,7 @@ export async function settleLow( earlier: { head: reviewed, findings: own }, settled: true, }; - rounds.push({ ...last, head }); + rounds.push({ ...last, head, usage: [] }); return { ...session, rounds: [...session.rounds.slice(0, -1), { ...session.rounds.at(-1)!, findings: [] }], @@ -1762,7 +1762,7 @@ export async function settleLow( }; } // A reviewer of the same run found nothing; it stands beside the settled ones in the record. - if (session.rounds.at(-1)!.head === reviewed) rounds.push({ ...session.last, head }); + if (session.rounds.at(-1)!.head === reviewed) rounds.push({ ...session.last, head, usage: [] }); return session; }); save(statePath, { ...saved!, sessions, tallies } satisfies State); diff --git a/test/review.test.ts b/test/review.test.ts index 456c58d7..53bcc4a6 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2129,6 +2129,7 @@ fakes("--settle records low findings' dispositions without running a reviewer", const settled = await quest("--root", s.repo, "review", "--settle", "--responses", responses); expect(settled.code, settled.stderr).toBe(0); expect(s.calls()).toHaveLength(calls); + expect(settled.stdout).not.toContain("\nusage "); expect(settled.stdout).toContain(`head ${head}\n`); expect(settled.stdout).toContain( `settled the author settled round 1's low findings at ${reviewed}; no reviewer ran\n`, From 0834454fa89b8199629472c84e6d801be610f7f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:13:34 -0800 Subject: [PATCH 08/17] fix(review): save a settlement once recorded, on the last round's base Co-Authored-By: Claude Opus 5.5 --- README.md | 15 ++++++++------- src/cli/main.ts | 19 +++++++++++++------ src/cli/review.ts | 31 +++++++++++++++++++++---------- test/review.test.ts | 26 +++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index beaa89fd..6a1748be 100644 --- a/README.md +++ b/README.md @@ -1415,13 +1415,14 @@ detached `HEAD` requires, such as a checkout of a branch another worktree holds, so its rounds continue that branch's session. A round whose findings are all low severity needs no further round. After -fixing or refuting them, `--settle` runs no reviewer: it reads `--responses` as -one `. fixed: ` or `. refuted: ` line per finding, by its -number, adds each disposition to its reviewer's tally, and drops the findings -from the session, so its next round checks what changed since their head with -nothing to settle. It prints and records the last run's rounds at the -committed head, which the record names in full, marking the findings the -author settled. +fixing or refuting them, `--settle` runs no reviewer: against the base the last +round used, it reads `--responses` as one `. fixed: ` or +`. refuted: ` line per finding, by its number, adds each disposition +to its reviewer's tally, and drops the findings from the session, so its next +round checks what changed since their head with nothing to settle. It prints +and records the last run's rounds at the committed head, which the record names +in full, marking the findings the author settled. It saves the settlement only +once the record is written or skipped, so a failed record can be retried. The reviewer is read-only: Codex in its `read-only` sandbox, ignoring execpolicy rules, and Claude Code with only read tools and the Bash commands it diff --git a/src/cli/main.ts b/src/cli/main.ts index f322097a..c78baf5c 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -50,6 +50,7 @@ import { REVIEW_HARNESSES, type ReviewHarness, ReviewInputError, + type Reviewed, ReviewLimitError, processDependencies as reviewDependencies, review, @@ -1215,11 +1216,12 @@ export function main(argv: string[], out: Output): number | Promise { "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + "checkout of a branch another worktree holds.\n\n" + - "--settle runs no reviewer: it settles the last round's findings, which must all be low severity, from " + + "--settle runs no reviewer: against the base the last round used, it settles that round's findings, which must all be low severity, from " + "--responses, one `. fixed: ` or `. refuted: ` line per finding by its number, adding " + "them to each reviewer's tally and dropping them from its session, so its next round checks what changed " + "since their head with nothing to settle. It prints and records the last run's rounds at the committed " + - "head, naming that head in full and each settled finding's disposition.\n\n" + + "head, naming that head in full and each settled finding's disposition, and saves the settlement only once " + + "the record is written or skipped, so a failed record can be retried.\n\n" + "The reviewer is read-only: Codex in its read-only sandbox without execpolicy rules, user config, or its " + "skills catalog, Claude Code with " + "no settings files, only read tools, and the Bash commands it judges read-only, and opencode with no " + @@ -1316,8 +1318,10 @@ export function main(argv: string[], out: Output): number | Promise { }, ); pending = run.then( - async ({ rounds, tallies }) => { - // The rounds are saved in their sessions either way, so their report prints before a failed record. + async (result: Reviewed & { save?: () => void }) => { + const { rounds, tallies } = result; + // A review's rounds are saved in their sessions either way, so their report prints before a failed + // record; a settlement is saved only once recorded, so it can be retried. let record: Recorded | undefined; let failure: unknown; try { @@ -1333,9 +1337,12 @@ export function main(argv: string[], out: Output): number | Promise { print("url" in record ? `record ${record.url}` : `record skipped: ${record.skipped}`); print(nextStep(rounds)); } - if (failure === undefined) return 0; + if (failure === undefined) { + result.save?.(); + return 0; + } warn( - `the round finished, but recording it in the PR body failed: ${failure instanceof Error ? failure.message : String(failure)}`, + `${options.settle ? "the settlement is not saved, since" : "the round finished, but"} recording it in the PR body failed: ${failure instanceof Error ? failure.message : String(failure)}`, ); return 1; }, diff --git a/src/cli/review.ts b/src/cli/review.ts index 776b4cd0..beaa4b45 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -1686,6 +1686,11 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): return { rounds: done.map((each) => each.round), tallies }; } +/** A settlement of low findings, saved only by `save`, once its record is written, so a failed record can be retried. */ +export interface Settlement extends Reviewed { + save(): void; +} + /** * Settle the last round's findings, all low severity, from the author's `responses` without running a reviewer: * one `. fixed: ` or `. refuted: ` line per finding, by its number. Each reviewer's tally @@ -1694,7 +1699,7 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): */ export async function settleLow( options: Pick & BaseOptions, -): Promise { +): Promise { const root = git(options.root, ["rev-parse", "--show-toplevel"]); if (git(root, ["status", "--porcelain"]) !== "") throw new ReviewInputError( @@ -1716,6 +1721,9 @@ export async function settleLow( const reviewed = open[0]!.session.rounds.at(-1)!.head; if (open.some(({ session }) => session.rounds.at(-1)!.head !== reviewed)) throw new Error("the findings to settle are from rounds of different heads"); + // The base keys the session, so a different one may have found another branch's. + const last = open[0]!.session.last.base.ref; + if (last !== baseRef) throw new ReviewInputError(`the last round reviewed against ${last}; pass --base ${last}`); if (options.responses === undefined) throw new ResponsesError( "--settle needs --responses, one `. fixed: ` or `. refuted: ` line per finding", @@ -1748,25 +1756,28 @@ export async function settleLow( if (own.length > 0) { const tally = tallies.find((each) => each.reviewer === session.reviewer)!; for (const finding of own) tally[finding.disposition]++; - const last: Round = { + const settled: Round = { ...session.last, findings: [], earlier: { head: reviewed, findings: own }, settled: true, }; - rounds.push({ ...last, head, usage: [] }); + rounds.push({ ...settled, head, usage: [] }); return { ...session, rounds: [...session.rounds.slice(0, -1), { ...session.rounds.at(-1)!, findings: [] }], - last, + last: settled, }; } - // A reviewer of the same run found nothing; it stands beside the settled ones in the record. - if (session.rounds.at(-1)!.head === reviewed) rounds.push({ ...session.last, head, usage: [] }); + // A reviewer of the same run found nothing; it stands beside the settled ones in the record, without + // the findings an earlier settlement of its own showed. + if (session.rounds.at(-1)!.head === reviewed) { + const { last: clean } = session; + rounds.push({ ...clean, head, usage: [], settled: false, earlier: clean.settled ? null : clean.earlier }); + } return session; }); - save(statePath, { ...saved!, sessions, tallies } satisfies State); - return { rounds, tallies }; + return { rounds, tallies, save: () => save(statePath, { ...saved!, sessions, tallies } satisfies State) }; } /** @@ -2002,7 +2013,7 @@ export function nextStep(rounds: Round[]): string { const findings = rounds.flatMap((round) => round.findings); if (findings.length === 0) return "next done: no findings remain on this head"; if (findings.every((finding) => finding.severity === "low")) - return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --settle --responses with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; + return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --settle --responses , with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; if (rounds.some((round) => round.findings.length > 0 && round.round % 3 === 0)) return "next checkpoint: before fixing these findings, reconsider the approach and let the user decide, as /quest-review describes"; return "next respond: fix each finding or refute it with evidence the reviewer cannot rebut, commit, rerun the checks, then run the next round with --responses, one response per finding by its number"; @@ -2031,7 +2042,7 @@ export function roundLines(rounds: Round[]): string[] { : []), ...(round.settled ? [ - `settled the author settled round ${round.round}'s low findings at ${round.earlier!.head}; no reviewer ran`, + `settled round ${round.round}'s low findings at ${round.earlier!.head}, by the author; no reviewer ran`, ] : []), ...round.usage.map((u) => `usage ${u.turn} input ${u.input} cached ${u.cached} output ${u.output}`), diff --git a/test/review.test.ts b/test/review.test.ts index 53bcc4a6..e4c1dd68 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2126,13 +2126,37 @@ fakes("--settle records low findings' dispositions without running a reviewer", expect(refused.code, bad).toBe(2); } write(s.repo, ".tmp/responses.md", "1. Fixed: src/lib.ts:1 guards it.\n"); + // The base keys the session, so another one is refused. + git(s.repo, "branch", "release", "origin/main"); + git(s.repo, "push", "-q", "origin", "release"); + const elsewhere = await quest( + "--root", + s.repo, + "review", + "--settle", + "--base", + "origin/release", + "--responses", + responses, + ); + expect(elsewhere.code).toBe(2); + expect(elsewhere.stderr).toContain("pass --base origin/main"); + // A record GitHub refuses saves nothing, so the settlement can be retried. + const fetch = globalThis.fetch; + vi.stubGlobal("fetch", async (input: URL, init?: RequestInit) => + init?.method === "PATCH" ? new Response("{}", { status: 503 }) : fetch(input, init), + ); + const refused = await quest("--root", s.repo, "review", "--settle", "--responses", responses); + expect(refused.code).toBe(1); + expect(refused.stderr).toContain("the settlement is not saved, since recording it in the PR body failed"); + vi.stubGlobal("fetch", fetch); const settled = await quest("--root", s.repo, "review", "--settle", "--responses", responses); expect(settled.code, settled.stderr).toBe(0); expect(s.calls()).toHaveLength(calls); expect(settled.stdout).not.toContain("\nusage "); expect(settled.stdout).toContain(`head ${head}\n`); expect(settled.stdout).toContain( - `settled the author settled round 1's low findings at ${reviewed}; no reviewer ran\n`, + `settled round 1's low findings at ${reviewed}, by the author; no reviewer ran\n`, ); expect(settled.stdout).toContain( `earlier 1 ${reviewed}\n\n1. fixed low src/lib.ts:2 Off by one\n src/lib.ts:1 guards it.\n`, From 64e548b5d5461d82e66be1b6820fc0b8ac031446 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 19:26:42 -0800 Subject: [PATCH 09/17] fix(review): settle only a session whose reviewed head HEAD contains, and report a failed save Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/cli/main.ts | 13 ++++++++++--- src/cli/review.ts | 12 ++++++++++-- test/review.test.ts | 15 +++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6a1748be..6da707a7 100644 --- a/README.md +++ b/README.md @@ -1416,7 +1416,7 @@ holds, so its rounds continue that branch's session. A round whose findings are all low severity needs no further round. After fixing or refuting them, `--settle` runs no reviewer: against the base the last -round used, it reads `--responses` as one `. fixed: ` or +round used, on a head that contains the one it reviewed, it reads `--responses` as one `. fixed: ` or `. refuted: ` line per finding, by its number, adds each disposition to its reviewer's tally, and drops the findings from the session, so its next round checks what changed since their head with nothing to settle. It prints diff --git a/src/cli/main.ts b/src/cli/main.ts index c78baf5c..f8bf946c 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1216,7 +1216,7 @@ export function main(argv: string[], out: Output): number | Promise { "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + "checkout of a branch another worktree holds.\n\n" + - "--settle runs no reviewer: against the base the last round used, it settles that round's findings, which must all be low severity, from " + + "--settle runs no reviewer: against the base the last round used and on a head that contains the one it reviewed, it settles that round's findings, which must all be low severity, from " + "--responses, one `. fixed: ` or `. refuted: ` line per finding by its number, adding " + "them to each reviewer's tally and dropping them from its session, so its next round checks what changed " + "since their head with nothing to settle. It prints and records the last run's rounds at the committed " + @@ -1338,8 +1338,15 @@ export function main(argv: string[], out: Output): number | Promise { print(nextStep(rounds)); } if (failure === undefined) { - result.save?.(); - return 0; + try { + result.save?.(); + return 0; + } catch (error) { + warn( + `the settlement is recorded in the PR body, but saving it failed: ${error instanceof Error ? error.message : String(error)}`, + ); + return 1; + } } warn( `${options.settle ? "the settlement is not saved, since" : "the round finished, but"} recording it in the PR body failed: ${failure instanceof Error ? failure.message : String(failure)}`, diff --git a/src/cli/review.ts b/src/cli/review.ts index beaa4b45..9d6ad5e4 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -1713,7 +1713,10 @@ export async function settleLow( const open = (saved?.sessions ?? []).flatMap((session) => session.rounds.at(-1)!.findings.map((finding) => ({ session, finding })), ); - if (open.length === 0) throw new ReviewInputError("the branch's last round left no findings to settle"); + if (open.length === 0) + throw new ReviewInputError( + `the last round on ${branch} left no findings to settle; pass the --base and --branch that round used`, + ); if (open.some(({ finding }) => finding.severity !== "low")) throw new ReviewInputError( "--settle settles only findings that are all low severity; run the next round with --responses", @@ -1724,6 +1727,12 @@ export async function settleLow( // The base keys the session, so a different one may have found another branch's. const last = open[0]!.session.last.base.ref; if (last !== baseRef) throw new ReviewInputError(`the last round reviewed against ${last}; pass --base ${last}`); + const head = git(root, ["rev-parse", "HEAD"]); + try { + git(root, ["merge-base", "--is-ancestor", reviewed, head]); + } catch { + throw new ReviewInputError(`HEAD does not contain ${reviewed}, which the last round on ${branch} reviewed`); + } if (options.responses === undefined) throw new ResponsesError( "--settle needs --responses, one `. fixed: ` or `. refuted: ` line per finding", @@ -1746,7 +1755,6 @@ export async function settleLow( if (answers.size !== open.length) throw new ResponsesError(`--settle settles each of the last round's ${open.length} finding(s) once`); - const head = git(root, ["rev-parse", "HEAD"]); const tallies = saved!.tallies.map((tally) => ({ ...tally, findings: { ...tally.findings } })); const rounds: Round[] = []; const sessions = saved!.sessions.map((session): Session => { diff --git a/test/review.test.ts b/test/review.test.ts index e4c1dd68..73f3f43c 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2141,6 +2141,21 @@ fakes("--settle records low findings' dispositions without running a reviewer", ); expect(elsewhere.code).toBe(2); expect(elsewhere.stderr).toContain("pass --base origin/main"); + // Nor is a session whose reviewed head this checkout does not contain, as another branch's under the same key. + git(s.repo, "checkout", "-q", "-b", "other", "origin/main"); + const other = await quest( + "--root", + s.repo, + "review", + "--settle", + "--branch", + "feature", + "--responses", + responses, + ); + expect(other.code).toBe(2); + expect(other.stderr).toContain(`HEAD does not contain ${reviewed}`); + git(s.repo, "checkout", "-q", "-"); // A record GitHub refuses saves nothing, so the settlement can be retried. const fetch = globalThis.fetch; vi.stubGlobal("fetch", async (input: URL, init?: RequestInit) => From 9b9ff6eb300e6292c3bc04d356d902f91f0ba6be Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 20:03:08 -0800 Subject: [PATCH 10/17] Name --base in the settle line, matching every round Co-Authored-By: Claude Opus 5.5 --- assets/skills/review.md | 2 +- src/cli/review.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/skills/review.md b/assets/skills/review.md index 55c6c66d..2eb32eff 100644 --- a/assets/skills/review.md +++ b/assets/skills/review.md @@ -35,7 +35,7 @@ Refute a finding only with evidence the reviewer cannot rebut: a `path:line`, a Commit the fixes, rerun the checks, and run the next round with `--responses`, one response per finding by its number, written in the checkout's `.tmp/`. Repeat until a round of the head reports no finding you have not refuted; never block on the score. A finding the reviewer keeps against such a refutation is a decision for the user; a background agent lists it under `## Waiting`. -A round whose findings are all low severity needs no further round: fix or refute each, commit, rerun the checks, then record their dispositions with `quest review --settle --responses `, one `. fixed: ` or `. refuted: ` line per finding. +A round whose findings are all low severity needs no further round: fix or refute each, commit, rerun the checks, then record their dispositions with `quest review --base / --settle --responses `, one `. fixed: ` or `. refuted: ` line per finding. Checkpoint the design: a round is one `quest review` round, numbered within each reviewer's session on the branch, and when a round whose number is a multiple of three reports findings, stop before fixing them and reconsider the approach; a clean round needs no checkpoint. Findings that keep coming usually mean the approach is wrong, not the last fix: brainstorm other designs that would avoid the whole class of findings, favoring simpler ones, and compare them with the current one. diff --git a/src/cli/review.ts b/src/cli/review.ts index 9d6ad5e4..7aa815b7 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -2021,7 +2021,7 @@ export function nextStep(rounds: Round[]): string { const findings = rounds.flatMap((round) => round.findings); if (findings.length === 0) return "next done: no findings remain on this head"; if (findings.every((finding) => finding.severity === "low")) - return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --settle --responses , with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; + return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --base / --settle --responses , with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; if (rounds.some((round) => round.findings.length > 0 && round.round % 3 === 0)) return "next checkpoint: before fixing these findings, reconsider the approach and let the user decide, as /quest-review describes"; return "next respond: fix each finding or refute it with evidence the reviewer cannot rebut, commit, rerun the checks, then run the next round with --responses, one response per finding by its number"; From 2b0ff4fe1ed5f394142c29e53f29fd56fb034583 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 20:39:47 -0800 Subject: [PATCH 11/17] Review again after a settle, keep deleted forks apart, fix settle labels Co-Authored-By: Claude Opus 5.5 --- src/cli/main.ts | 2 +- src/cli/review.ts | 20 +++++++++++++++++--- src/cli/usage.ts | 4 ++-- test/review.test.ts | 9 ++++++++- test/usage.test.ts | 19 ++++++++++++++++--- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/cli/main.ts b/src/cli/main.ts index f8bf946c..1453c62a 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1343,7 +1343,7 @@ export function main(argv: string[], out: Output): number | Promise { return 0; } catch (error) { warn( - `the settlement is recorded in the PR body, but saving it failed: ${error instanceof Error ? error.message : String(error)}`, + `the settlement is ${record !== undefined && "url" in record ? "recorded in the PR body" : "printed"}, but saving it failed: ${error instanceof Error ? error.message : String(error)}`, ); return 1; } diff --git a/src/cli/review.ts b/src/cli/review.ts index 7aa815b7..fcc80e37 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -1781,7 +1781,15 @@ export async function settleLow( // the findings an earlier settlement of its own showed. if (session.rounds.at(-1)!.head === reviewed) { const { last: clean } = session; - rounds.push({ ...clean, head, usage: [], settled: false, earlier: clean.settled ? null : clean.earlier }); + rounds.push({ + ...clean, + head, + usage: [], + // It reviewed the settled head, not this one. + unchanged: false, + settled: false, + earlier: clean.settled ? null : clean.earlier, + }); } return session; }); @@ -1917,8 +1925,14 @@ async function reviewWith(shared: Shared, seat: Seat, responses: string | undefi covers(root, previous.mergeBase, change.mergeBase); // The last round already passed this head, and has nothing left to settle; it repeats against the run's base - // and size. - if (resume && previous!.head === change.head && previous!.mergeBase === change.mergeBase && earlier.length === 0) { + // and size. A settled round cleared findings it raised at that head, so a head back at it is reviewed again. + if ( + resume && + !last!.settled && + previous!.head === change.head && + previous!.mergeBase === change.mergeBase && + earlier.length === 0 + ) { const { size, from: sizeFrom } = shared.sized; return { round: { ...last!, base: shared.base, size, sizeFrom, usage: [], unchanged: true }, session: stored! }; } diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 336aa0ed..8b72a816 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -588,8 +588,8 @@ function spendRows(pulls: PullRow[]): SpendRow[] { function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const rows = new Map(); // A branch's saved tallies span every PR from it, so only its newest PR that has both a tally and an - // implementer to count it for counts them. - const branch = (each: PullRow) => JSON.stringify([each.repository, each.branch]); + // implementer to count it for counts them. A deleted fork has no repository, so its PR stands alone. + const branch = (each: PullRow) => JSON.stringify([each.repository ?? each.number, each.branch]); const newest = new Map(); for (const each of pulls) if (each.implementers.length > 0 && each.review.some((review) => review.tally !== null)) diff --git a/test/review.test.ts b/test/review.test.ts index 73f3f43c..1e576eb5 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2195,6 +2195,12 @@ fakes("--settle records low findings' dispositions without running a reviewer", // Nothing is left to settle, and the next round checks the fix with no responses, counting the finding once. expect((await quest("--root", s.repo, "review", "--settle", "--responses", responses)).code).toBe(2); vi.stubEnv("FAKE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); + // A head back at the reviewed one is reviewed again, since the settlement cleared the findings raised there. + git(s.repo, "checkout", "-q", "--detach", reviewed); + const back = await quest("--root", s.repo, "review", "--branch", "feature", "--json"); + expect(back.code, back.stderr).toBe(0); + expect(JSON.parse(back.stdout).rounds.map((r: { unchanged: boolean }) => r.unchanged)).toEqual([false, true]); + git(s.repo, "checkout", "-q", "-"); const next = await quest("--root", s.repo, "review", "--json"); expect(next.code, next.stderr).toBe(0); const resumed = s.calls().at(-1)!; @@ -2209,7 +2215,8 @@ fakes("--settle records low findings' dispositions without running a reviewer", t.refuted, ]), ).toEqual([ - ["codex", 2, 1, 1, 0], + // Codex reviewed the head back at the reviewed one, too. + ["codex", 3, 1, 1, 0], ["opencode", 2, 0, 0, 0], ]); } finally { diff --git a/test/usage.test.ts b/test/usage.test.ts index 40f405f6..034000d1 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -468,6 +468,16 @@ describe("usage", () => { updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, }), + // Two deleted forks' PRs from a branch of the same name: neither names its repository, so each counts. + ...[5, 6].map((number) => + pr(number, "quest/m0/a", body9, "2026-11-02T00:00:00Z", { + state: "open", + merged_at: null, + closed_at: null, + updated_at: "2026-11-06T00:00:00Z", + head: { ref: "quest/m0/a", repo: null }, + }), + ), pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), ]; const requests: string[] = []; @@ -590,7 +600,7 @@ describe("usage", () => { test("ties each session to its PR and role, and rolls up spend and accuracy", async () => { const { root, deps, priced } = setup(); const report = await usage(root, { since: "2026-11-05" }, deps(priced)); - expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7, 13]); + expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7, 13, 5, 6]); const ten = report.pulls.find((p) => p.number === 10)!; expect(ten.sessions.map((s) => [s.harness, s.session, s.role, s.effort])).toEqual([ ["claude", CLAUDE, "implementing", "high"], @@ -624,10 +634,11 @@ describe("usage", () => { ["L", "reviewing", "opencode", 4], ]); const tally = { rounds: 2, findings: { high: 1, medium: 1, low: 1 }, fixed: 2, refuted: 1, open: 0 }; - const double = { rounds: 4, findings: { high: 2, medium: 2, low: 2 }, fixed: 4, refuted: 2, open: 0 }; + // #10's, #7's, and each deleted fork's. + const four = { rounds: 8, findings: { high: 4, medium: 4, low: 4 }, fixed: 8, refuted: 4, open: 0 }; expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], - ["L", "claude-opus-5-5", 3, 2, 34, double, 1], + ["L", "claude-opus-5-5", 5, 4, 68, four, 1], ]); expect(ten.online).toEqual([ { @@ -687,6 +698,8 @@ describe("usage", () => { [9, 0], [7, 0], [13, 0], + [5, 0], + [6, 0], ]); expect(quest.pulls[0]!.review.map((r) => r.score)).toEqual([9, 8, null]); }); From b510daa08a7f8fd0281314718d389ea3efc635e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:06:34 -0800 Subject: [PATCH 12/17] Review the whole change on a head without the settled one Co-Authored-By: Claude Opus 5.5 --- src/cli/review-record.ts | 8 +++++--- src/cli/review.ts | 28 +++++++++++++--------------- test/review.test.ts | 5 ++++- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index dd3c6441..e453b84a 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -50,13 +50,15 @@ const counted = (tally: Tally): Counts => ({ }); /** - * The `## Review` section's text for a run's `rounds` of the same head, without its heading, with every tally + * The `## Review` section's text for a run's `rounds` of the same head (or a settlement's), without its heading, with every tally * on the branch: each round's reviewer's beside its round, and a reviewer the run left out on its own. */ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { - const [shared] = rounds; + // A settlement's head is the settled rounds', beside clean rounds of the head they reviewed. + const settled = rounds.find((round) => round.settled); + const shared = settled ?? rounds[0]; const clean = rounds.every((round) => round.findings.length === 0); - const author = rounds.find((round) => round.settled)?.earlier; + const author = settled?.earlier; const lines = [ MARK, "", diff --git a/src/cli/review.ts b/src/cli/review.ts index fcc80e37..0296a4e3 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -1764,28 +1764,29 @@ export async function settleLow( if (own.length > 0) { const tally = tallies.find((each) => each.reviewer === session.reviewer)!; for (const finding of own) tally[finding.disposition]++; + // Saved at the settlement head, so a later round on a head without it reviews the whole change again. const settled: Round = { ...session.last, + head, findings: [], earlier: { head: reviewed, findings: own }, settled: true, }; - rounds.push({ ...settled, head, usage: [] }); + rounds.push({ ...settled, usage: [] }); return { ...session, rounds: [...session.rounds.slice(0, -1), { ...session.rounds.at(-1)!, findings: [] }], last: settled, }; } - // A reviewer of the same run found nothing; it stands beside the settled ones in the record, without - // the findings an earlier settlement of its own showed. + // A reviewer of the same run found nothing; it stands beside the settled ones in the record, at the head + // it reviewed, without the findings an earlier settlement of its own showed. if (session.rounds.at(-1)!.head === reviewed) { const { last: clean } = session; rounds.push({ ...clean, - head, + head: reviewed, usage: [], - // It reviewed the settled head, not this one. unchanged: false, settled: false, earlier: clean.settled ? null : clean.earlier, @@ -1922,17 +1923,13 @@ async function reviewWith(shared: Shared, seat: Seat, responses: string | undefi last.effort === picked.effort && last.base.ref === shared.base.ref && (!reviewer.bound || stored!.root === root) && - covers(root, previous.mergeBase, change.mergeBase); + covers(root, previous.mergeBase, change.mergeBase) && + // A settlement cleared the findings the last round raised, which only a head containing its fix answers. + (!last.settled || covers(root, last.head, change.head)); // The last round already passed this head, and has nothing left to settle; it repeats against the run's base - // and size. A settled round cleared findings it raised at that head, so a head back at it is reviewed again. - if ( - resume && - !last!.settled && - previous!.head === change.head && - previous!.mergeBase === change.mergeBase && - earlier.length === 0 - ) { + // and size. + if (resume && previous!.head === change.head && previous!.mergeBase === change.mergeBase && earlier.length === 0) { const { size, from: sizeFrom } = shared.sized; return { round: { ...last!, base: shared.base, size, sizeFrom, usage: [], unchanged: true }, session: stored! }; } @@ -2046,7 +2043,8 @@ export function nextStep(rounds: Round[]): string { * line, and its findings, numbered across the reviewers in order, each with its body indented. */ export function roundLines(rounds: Round[]): string[] { - const [shared] = rounds; + // A settlement's head is the settled rounds', beside clean rounds of the head they reviewed. + const shared = rounds.find((round) => round.settled) ?? rounds[0]; const lines = [ `size ${shared!.size} ${shared!.sizeFrom}`, `base ${shared!.base.sha} ${shared!.base.ref}`, diff --git a/test/review.test.ts b/test/review.test.ts index 1e576eb5..6eef9ba7 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2195,10 +2195,13 @@ fakes("--settle records low findings' dispositions without running a reviewer", // Nothing is left to settle, and the next round checks the fix with no responses, counting the finding once. expect((await quest("--root", s.repo, "review", "--settle", "--responses", responses)).code).toBe(2); vi.stubEnv("FAKE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); - // A head back at the reviewed one is reviewed again, since the settlement cleared the findings raised there. + // A head without the settled one starts a new session that reviews the whole change, since the settlement + // cleared the findings raised there; the clean reviewer's round of that head stands. git(s.repo, "checkout", "-q", "--detach", reviewed); + const before = s.calls().length; const back = await quest("--root", s.repo, "review", "--branch", "feature", "--json"); expect(back.code, back.stderr).toBe(0); + expect(s.calls()[before]!.argv.slice(0, 2)).not.toEqual(["exec", "resume"]); expect(JSON.parse(back.stdout).rounds.map((r: { unchanged: boolean }) => r.unchanged)).toEqual([false, true]); git(s.repo, "checkout", "-q", "-"); const next = await quest("--root", s.repo, "review", "--json"); From 22d6288bb0442cee26fc54e65ac2840606a5f36e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:14:19 -0800 Subject: [PATCH 13/17] Remove quest review --settle The maintainer dropped --settle (2026-11-11) for the merged-means-settled rule. Co-Authored-By: Claude Opus 5.5 --- README.md | 11 --- assets/skills/review.md | 9 +-- src/cli/main.ts | 53 ++++---------- src/cli/review-record.ts | 14 ++-- src/cli/review.ts | 128 +--------------------------------- test/review.test.ts | 144 --------------------------------------- test/usage.test.ts | 1 - 7 files changed, 22 insertions(+), 338 deletions(-) diff --git a/README.md b/README.md index 6da707a7..db3320a5 100644 --- a/README.md +++ b/README.md @@ -1365,7 +1365,6 @@ and counts as logged out when `opencode models` does not list the model. ```sh quest review # round 1 against origin's default branch quest review --responses .tmp/responses.md # round 2: the new head, with fixes and refutations -quest review --settle --responses .tmp/responses.md # settle a round's low findings without a reviewer quest review --base origin/release --new # a new session against another base quest review --size S --exclude-model gpt-6.1-sol # an S PR that Sol wrote, with no record in its body quest review --branch quest/m1/foo --responses .tmp/responses.md # a detached checkout of that branch's head @@ -1414,16 +1413,6 @@ this quest cannot read. `--branch` names the branch instead of the checked-out o detached `HEAD` requires, such as a checkout of a branch another worktree holds, so its rounds continue that branch's session. -A round whose findings are all low severity needs no further round. After -fixing or refuting them, `--settle` runs no reviewer: against the base the last -round used, on a head that contains the one it reviewed, it reads `--responses` as one `. fixed: ` or -`. refuted: ` line per finding, by its number, adds each disposition -to its reviewer's tally, and drops the findings from the session, so its next -round checks what changed since their head with nothing to settle. It prints -and records the last run's rounds at the committed head, which the record names -in full, marking the findings the author settled. It saves the settlement only -once the record is written or skipped, so a failed record can be retried. - The reviewer is read-only: Codex in its `read-only` sandbox, ignoring execpolicy rules, and Claude Code with only read tools and the Bash commands it judges read-only, in `dontAsk` mode. Its instructions say the base's files and diff --git a/assets/skills/review.md b/assets/skills/review.md index 2eb32eff..7a70ed32 100644 --- a/assets/skills/review.md +++ b/assets/skills/review.md @@ -34,12 +34,12 @@ Never defer a relevant fix to a follow-up just to narrow the PR; add only out-of Refute a finding only with evidence the reviewer cannot rebut: a `path:line`, a command's output, or a recorded maintainer decision. Commit the fixes, rerun the checks, and run the next round with `--responses`, one response per finding by its number, written in the checkout's `.tmp/`. Repeat until a round of the head reports no finding you have not refuted; never block on the score. -A finding the reviewer keeps against such a refutation is a decision for the user; a background agent lists it under `## Waiting`. -A round whose findings are all low severity needs no further round: fix or refute each, commit, rerun the checks, then record their dispositions with `quest review --base / --settle --responses `, one `. fixed: ` or `. refuted: ` line per finding. +A finding the reviewer keeps against such a refutation is a decision for the user. +A round whose findings are all low severity needs no further round: fix or refute each, then add a line after the record naming the new head's full SHA and each finding's disposition. Checkpoint the design: a round is one `quest review` round, numbered within each reviewer's session on the branch, and when a round whose number is a multiple of three reports findings, stop before fixing them and reconsider the approach; a clean round needs no checkpoint. Findings that keep coming usually mean the approach is wrong, not the last fix: brainstorm other designs that would avoid the whole class of findings, favoring simpler ones, and compare them with the current one. -Report the alternatives and what remains, recommending to continue, rethink the approach, merge as is, or drop the PR, and let the user decide; a background agent leaves this under `## Waiting` rather than deciding it. +Report the alternatives and what remains, recommending to continue, rethink the approach, merge as is, or drop the PR, and let the user decide. Record each design change, including a rethink the user picks, in a PR comment that starts with `Design change:`, and start the next review with `quest review --new`, which restarts the count. A reviewer that is out of quota or logged out is an outage, not a skip: stop reviewing, keep the PR a draft, hand it back, and report the outage. @@ -48,7 +48,6 @@ Once the reset passes or the login is restored, the item has cleared: whoever fi Push, then flip the PR to "Ready for Review" with `quest ready `, which also drops its `## Waiting` section. For your own draft, mark it ready for review yourself. -Do not wait for online reviewers. Summarize the review, with its last round's score and rationale, and summarize CI from `quest status `. Name any decisions the review raised without answering them, then interactively prompt the user with your recommended action: @@ -59,7 +58,5 @@ Name any decisions the review raised without answering them, then interactively While its required checks are still running, report that instead of asking about it; `/quest-run` asks once they finish. -A background agent that cannot prompt lists these in its report instead. - Hand the PR back with `quest release ` before you ask, report, or go on to `/quest-merge`. To go straight on to `/quest-iterate` in this session, pass the token to its `quest iterate` instead, which keeps the hold. diff --git a/src/cli/main.ts b/src/cli/main.ts index 1453c62a..3da58382 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -50,11 +50,9 @@ import { REVIEW_HARNESSES, type ReviewHarness, ReviewInputError, - type Reviewed, ReviewLimitError, processDependencies as reviewDependencies, review, - settleLow, nextStep, roundLines, } from "./review"; @@ -1216,12 +1214,6 @@ export function main(argv: string[], out: Output): number | Promise { "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + "checkout of a branch another worktree holds.\n\n" + - "--settle runs no reviewer: against the base the last round used and on a head that contains the one it reviewed, it settles that round's findings, which must all be low severity, from " + - "--responses, one `. fixed: ` or `. refuted: ` line per finding by its number, adding " + - "them to each reviewer's tally and dropping them from its session, so its next round checks what changed " + - "since their head with nothing to settle. It prints and records the last run's rounds at the committed " + - "head, naming that head in full and each settled finding's disposition, and saves the settlement only once " + - "the record is written or skipped, so a failed record can be retried.\n\n" + "The reviewer is read-only: Codex in its read-only sandbox without execpolicy rules, user config, or its " + "skills catalog, Claude Code with " + "no settings files, only read tools, and the Bash commands it judges read-only, and opencode with no " + @@ -1238,7 +1230,7 @@ export function main(argv: string[], out: Output): number | Promise { "part read from the prompt cache, and output), a score from 0 to 10 for how ready the head is to land with " + "its one-sentence rationale, the finding count, each finding by its number, then the last round's findings with " + "their numbers and dispositions; then the `record` line below and a last `next` line naming the author's next step: " + - "`next finish` when every finding is low severity, to fix or refute them and settle them with --settle, " + + "`next finish` when every finding is low severity, to fix or refute them without another round, " + "`next checkpoint` when a reviewer's every third round reports other findings, to reconsider the approach " + "before fixing them, `next respond` to fix or refute the findings, or `next done`. A reply without a valid score " + "fails as any malformed reply does. --json prints " + @@ -1278,12 +1270,6 @@ export function main(argv: string[], out: Output): number | Promise { ) .option("--responses ", "The author's responses to the last round's findings; - reads stdin") .option("--new", "Start a new review session instead of resuming the branch's") - .addOption( - new Option( - "--settle", - "Settle the last round's low findings from --responses without running a reviewer", - ).conflicts(["harness", "model", "effort", "excludeModel", "size", "new"]), - ) .option( "--branch ", "Branch whose review session to continue, instead of the checked-out one or its upstream", @@ -1300,7 +1286,6 @@ export function main(argv: string[], out: Output): number | Promise { size?: Size; responses?: string; new?: boolean; - settle?: boolean; json?: boolean; }) => { const responses = @@ -1308,20 +1293,16 @@ export function main(argv: string[], out: Output): number | Promise { ? undefined : readFileSync(options.responses === "-" ? 0 : options.responses, "utf8"); const github = { github: () => gitHubDependencies(warn), warn }; - const run = options.settle - ? settleLow({ ...options, root: root(), responses }) - : review( - { ...options, root: root(), exclude: options.excludeModel, responses, fresh: options.new }, - { - ...reviewDependencies(), - models: (dir, base, branch) => pullModels(dir, base, branch, github), - }, - ); + const run = review( + { ...options, root: root(), exclude: options.excludeModel, responses, fresh: options.new }, + { + ...reviewDependencies(), + models: (dir, base, branch) => pullModels(dir, base, branch, github), + }, + ); pending = run.then( - async (result: Reviewed & { save?: () => void }) => { - const { rounds, tallies } = result; - // A review's rounds are saved in their sessions either way, so their report prints before a failed - // record; a settlement is saved only once recorded, so it can be retried. + async ({ rounds, tallies }) => { + // The rounds are saved in their sessions either way, so their report prints before a failed record. let record: Recorded | undefined; let failure: unknown; try { @@ -1337,19 +1318,9 @@ export function main(argv: string[], out: Output): number | Promise { print("url" in record ? `record ${record.url}` : `record skipped: ${record.skipped}`); print(nextStep(rounds)); } - if (failure === undefined) { - try { - result.save?.(); - return 0; - } catch (error) { - warn( - `the settlement is ${record !== undefined && "url" in record ? "recorded in the PR body" : "printed"}, but saving it failed: ${error instanceof Error ? error.message : String(error)}`, - ); - return 1; - } - } + if (failure === undefined) return 0; warn( - `${options.settle ? "the settlement is not saved, since" : "the round finished, but"} recording it in the PR body failed: ${failure instanceof Error ? failure.message : String(failure)}`, + `the round finished, but recording it in the PR body failed: ${failure instanceof Error ? failure.message : String(failure)}`, ); return 1; }, diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index e453b84a..6a4e3e44 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -50,20 +50,17 @@ const counted = (tally: Tally): Counts => ({ }); /** - * The `## Review` section's text for a run's `rounds` of the same head (or a settlement's), without its heading, with every tally + * The `## Review` section's text for a run's `rounds` of the same head, without its heading, with every tally * on the branch: each round's reviewer's beside its round, and a reviewer the run left out on its own. */ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { - // A settlement's head is the settled rounds', beside clean rounds of the head they reviewed. - const settled = rounds.find((round) => round.settled); - const shared = settled ?? rounds[0]; + const [shared] = rounds; const clean = rounds.every((round) => round.findings.length === 0); - const author = settled?.earlier; const lines = [ MARK, "", // `quest status` counts a full SHA here as a review of that head, which an open finding has not passed. - `- Head: ${clean ? shared!.head : `${shared!.head.slice(0, 12)}, named in full once no reviewer reports a finding`}${author ? `, where the author settled the low findings reviewed at ${author.head.slice(0, 12)} with no further round` : ""}`, + `- Head: ${clean ? shared!.head : `${shared!.head.slice(0, 12)}, named in full once no reviewer reports a finding`}`, `- Base: ${shared!.base.ref} at ${shared!.base.sha.slice(0, 12)}`, ]; for (const round of rounds) { @@ -81,10 +78,7 @@ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { for (const finding of round.findings) lines.push(` - ${capitalized(finding.severity)}, ${code(located(finding))}: ${code(finding.title)}`); const settled = round.earlier?.findings ?? []; - if (settled.length > 0) - lines.push( - `- The last round's findings, at ${round.earlier!.head.slice(0, 12)}${round.settled ? ", settled by the author" : ""}:`, - ); + if (settled.length > 0) lines.push(`- The last round's findings, at ${round.earlier!.head.slice(0, 12)}:`); for (const finding of settled) lines.push( ` - ${capitalized(finding.disposition)}: ${finding.severity}, ${code(located(finding))}: ${code(finding.title)}; ${code(finding.reason)}`, diff --git a/src/cli/review.ts b/src/cli/review.ts index 0296a4e3..fdbfa54d 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -106,8 +106,6 @@ export interface Round { earlier: { head: string; findings: Settled[] } | null; /** No reviewer ran: the head is the last round's, which found nothing, so this repeats that round. */ unchanged: boolean; - /** No reviewer ran: the author settled the last round's low findings, `earlier`, at `head`, with `--settle`. */ - settled: boolean; } /** A reply's verdict on the reviewed head; `earlier` settles the last round's findings by their number. */ @@ -1686,117 +1684,6 @@ export async function review(options: ReviewOptions, deps: ReviewDependencies): return { rounds: done.map((each) => each.round), tallies }; } -/** A settlement of low findings, saved only by `save`, once its record is written, so a failed record can be retried. */ -export interface Settlement extends Reviewed { - save(): void; -} - -/** - * Settle the last round's findings, all low severity, from the author's `responses` without running a reviewer: - * one `. fixed: ` or `. refuted: ` line per finding, by its number. Each reviewer's tally - * counts them, and its session forgets them, so its next round checks what changed since their head with nothing - * to settle. Gives the last run's rounds at the checked-out head, those the author settled marked `settled`. - */ -export async function settleLow( - options: Pick & BaseOptions, -): Promise { - const root = git(options.root, ["rev-parse", "--show-toplevel"]); - if (git(root, ["status", "--porcelain"]) !== "") - throw new ReviewInputError( - "the checkout has uncommitted changes; commit them, since the record names the committed head", - ); - const baseRef = options.fetched ? options.base : await fetchBase(root, options.base); - const branch = sessionBranch(root, options.branch, parseBase(root, baseRef).branch); - const statePath = sessionFile(git(root, ["rev-parse", "--path-format=absolute", "--git-common-dir"]), branch); - const saved = savedState(statePath, false); - // Numbered as the last run printed them, across the sessions in order. - const open = (saved?.sessions ?? []).flatMap((session) => - session.rounds.at(-1)!.findings.map((finding) => ({ session, finding })), - ); - if (open.length === 0) - throw new ReviewInputError( - `the last round on ${branch} left no findings to settle; pass the --base and --branch that round used`, - ); - if (open.some(({ finding }) => finding.severity !== "low")) - throw new ReviewInputError( - "--settle settles only findings that are all low severity; run the next round with --responses", - ); - const reviewed = open[0]!.session.rounds.at(-1)!.head; - if (open.some(({ session }) => session.rounds.at(-1)!.head !== reviewed)) - throw new Error("the findings to settle are from rounds of different heads"); - // The base keys the session, so a different one may have found another branch's. - const last = open[0]!.session.last.base.ref; - if (last !== baseRef) throw new ReviewInputError(`the last round reviewed against ${last}; pass --base ${last}`); - const head = git(root, ["rev-parse", "HEAD"]); - try { - git(root, ["merge-base", "--is-ancestor", reviewed, head]); - } catch { - throw new ReviewInputError(`HEAD does not contain ${reviewed}, which the last round on ${branch} reviewed`); - } - if (options.responses === undefined) - throw new ResponsesError( - "--settle needs --responses, one `. fixed: ` or `. refuted: ` line per finding", - ); - const answers = new Map(); - for (const line of options.responses.split("\n").map((each) => each.trim())) { - if (line === "") continue; - const match = /^(\d+)\. (fixed|refuted): (.+)$/i.exec(line); - if (!match) - throw new ResponsesError( - `a --settle response is not \`. fixed: \` or \`. refuted: \`: ${line}`, - ); - const number = Number(match[1]); - if (number < 1 || number > open.length || answers.has(number)) - throw new ResponsesError( - `--settle settles each of the last round's ${open.length} finding(s) once: ${line}`, - ); - answers.set(number, { disposition: match[2]!.toLowerCase() as "fixed" | "refuted", reason: match[3]! }); - } - if (answers.size !== open.length) - throw new ResponsesError(`--settle settles each of the last round's ${open.length} finding(s) once`); - - const tallies = saved!.tallies.map((tally) => ({ ...tally, findings: { ...tally.findings } })); - const rounds: Round[] = []; - const sessions = saved!.sessions.map((session): Session => { - const own = open.flatMap(({ session: raiser, finding }, i) => - raiser === session ? [{ ...finding, number: i + 1, ...answers.get(i + 1)! }] : [], - ); - if (own.length > 0) { - const tally = tallies.find((each) => each.reviewer === session.reviewer)!; - for (const finding of own) tally[finding.disposition]++; - // Saved at the settlement head, so a later round on a head without it reviews the whole change again. - const settled: Round = { - ...session.last, - head, - findings: [], - earlier: { head: reviewed, findings: own }, - settled: true, - }; - rounds.push({ ...settled, usage: [] }); - return { - ...session, - rounds: [...session.rounds.slice(0, -1), { ...session.rounds.at(-1)!, findings: [] }], - last: settled, - }; - } - // A reviewer of the same run found nothing; it stands beside the settled ones in the record, at the head - // it reviewed, without the findings an earlier settlement of its own showed. - if (session.rounds.at(-1)!.head === reviewed) { - const { last: clean } = session; - rounds.push({ - ...clean, - head: reviewed, - usage: [], - unchanged: false, - settled: false, - earlier: clean.settled ? null : clean.earlier, - }); - } - return session; - }); - return { rounds, tallies, save: () => save(statePath, { ...saved!, sessions, tallies } satisfies State) }; -} - /** * A branch's saved state, or undefined when it has none. `--new` drops the sessions but continues their tallies, * less the findings it drops unsettled, so it reads them too, starting them over only when the state is one this quest cannot read, which is what @@ -1923,9 +1810,7 @@ async function reviewWith(shared: Shared, seat: Seat, responses: string | undefi last.effort === picked.effort && last.base.ref === shared.base.ref && (!reviewer.bound || stored!.root === root) && - covers(root, previous.mergeBase, change.mergeBase) && - // A settlement cleared the findings the last round raised, which only a head containing its fix answers. - (!last.settled || covers(root, last.head, change.head)); + covers(root, previous.mergeBase, change.mergeBase); // The last round already passed this head, and has nothing left to settle; it repeats against the run's base // and size. @@ -2017,7 +1902,6 @@ async function reviewWith(shared: Shared, seat: Seat, responses: string | undefi })), }, unchanged: false, - settled: false, }; return { round, session: { reviewer: harness, session, root, rounds, last: round, given } }; } @@ -2032,7 +1916,7 @@ export function nextStep(rounds: Round[]): string { const findings = rounds.flatMap((round) => round.findings); if (findings.length === 0) return "next done: no findings remain on this head"; if (findings.every((finding) => finding.severity === "low")) - return "next finish: fix or refute each finding, commit, rerun the checks, then run quest review --base / --settle --responses , with one `. fixed: ` or `. refuted: ` line per finding; run no further round"; + return "next finish: fix or refute each finding, commit, rerun the checks, then add a line after the PR body's review record naming the new head's full SHA and each finding's disposition; run no further round"; if (rounds.some((round) => round.findings.length > 0 && round.round % 3 === 0)) return "next checkpoint: before fixing these findings, reconsider the approach and let the user decide, as /quest-review describes"; return "next respond: fix each finding or refute it with evidence the reviewer cannot rebut, commit, rerun the checks, then run the next round with --responses, one response per finding by its number"; @@ -2043,8 +1927,7 @@ export function nextStep(rounds: Round[]): string { * line, and its findings, numbered across the reviewers in order, each with its body indented. */ export function roundLines(rounds: Round[]): string[] { - // A settlement's head is the settled rounds', beside clean rounds of the head they reviewed. - const shared = rounds.find((round) => round.settled) ?? rounds[0]; + const [shared] = rounds; const lines = [ `size ${shared!.size} ${shared!.sizeFrom}`, `base ${shared!.base.sha} ${shared!.base.ref}`, @@ -2060,11 +1943,6 @@ export function roundLines(rounds: Round[]): string[] { ...(round.unchanged ? [`unchanged round ${round.round} reviewed this head and found nothing; no reviewer ran`] : []), - ...(round.settled - ? [ - `settled round ${round.round}'s low findings at ${round.earlier!.head}, by the author; no reviewer ran`, - ] - : []), ...round.usage.map((u) => `usage ${u.turn} input ${u.input} cached ${u.cached} output ${u.output}`), `score ${round.score} ${round.rationale.replace(/\s+/g, " ").trim()}`, `findings ${round.findings.length}`, diff --git a/test/review.test.ts b/test/review.test.ts index 6eef9ba7..ca434a17 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -2016,7 +2016,6 @@ test("a tally counts each finding once, by who raised it, through every round", rationale: "r", earlier: null, unchanged: false, - settled: false, } satisfies Omit; const [a, b, c] = [finding("high", "a"), finding("medium", "b"), finding("low", "c")]; const first = tallied([], [{ round: { ...base, reviewer: "codex", round: 1, findings: [a, b, c] } }]); @@ -2083,146 +2082,3 @@ test("a tally counts each finding once, by who raised it, through every round", ), ).toThrow(/no tally/); }); - -fakes("--settle records low findings' dispositions without running a reviewer", async () => { - const t = setup(); - expect((await quest("--root", t.repo, "review")).code).toBe(0); - write(t.repo, ".tmp/responses.md", "1. fixed: src/lib.ts:1.\n2. fixed: src/lib.ts:1.\n"); - const high = await quest( - "--root", - t.repo, - "review", - "--settle", - "--responses", - join(t.repo, ".tmp", "responses.md"), - ); - expect(high.code).toBe(2); - expect(high.stderr).toContain("all low severity"); - - const s = setup(); - const pull = { - number: 7, - body: "Fixes averaging.\n\n## Models\n\n- Size: S\n- Implementer: Claude Code, claude-opus-5-5, high effort", - }; - const writes = onGitHub(s.repo, [pull]); - try { - vi.stubEnv( - "FAKE_REPLY", - JSON.stringify({ score: 9, rationale: "Nits.", findings: [{ ...FINDING, severity: "low" }] }), - ); - vi.stubEnv("FAKE_OPENCODE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); - const first = await quest("--root", s.repo, "review"); - expect(first.code, first.stderr).toBe(0); - expect(first.stdout).toMatch(/\nnext finish: .*--settle/); - const reviewed = git(s.repo, "rev-parse", "HEAD"); - write(s.repo, "src/lib.ts", "export const x = 4;\n"); - git(s.repo, "commit", "-qam", "fix"); - const head = git(s.repo, "rev-parse", "HEAD"); - const calls = s.calls().length; - const responses = join(s.repo, ".tmp", "responses.md"); - for (const bad of ["1 fixed src/lib.ts:1\n", "", "1. fixed: a\n1. refuted: b\n", "2. fixed: a\n"]) { - write(s.repo, ".tmp/responses.md", bad); - const refused = await quest("--root", s.repo, "review", "--settle", "--responses", responses); - expect(refused.code, bad).toBe(2); - } - write(s.repo, ".tmp/responses.md", "1. Fixed: src/lib.ts:1 guards it.\n"); - // The base keys the session, so another one is refused. - git(s.repo, "branch", "release", "origin/main"); - git(s.repo, "push", "-q", "origin", "release"); - const elsewhere = await quest( - "--root", - s.repo, - "review", - "--settle", - "--base", - "origin/release", - "--responses", - responses, - ); - expect(elsewhere.code).toBe(2); - expect(elsewhere.stderr).toContain("pass --base origin/main"); - // Nor is a session whose reviewed head this checkout does not contain, as another branch's under the same key. - git(s.repo, "checkout", "-q", "-b", "other", "origin/main"); - const other = await quest( - "--root", - s.repo, - "review", - "--settle", - "--branch", - "feature", - "--responses", - responses, - ); - expect(other.code).toBe(2); - expect(other.stderr).toContain(`HEAD does not contain ${reviewed}`); - git(s.repo, "checkout", "-q", "-"); - // A record GitHub refuses saves nothing, so the settlement can be retried. - const fetch = globalThis.fetch; - vi.stubGlobal("fetch", async (input: URL, init?: RequestInit) => - init?.method === "PATCH" ? new Response("{}", { status: 503 }) : fetch(input, init), - ); - const refused = await quest("--root", s.repo, "review", "--settle", "--responses", responses); - expect(refused.code).toBe(1); - expect(refused.stderr).toContain("the settlement is not saved, since recording it in the PR body failed"); - vi.stubGlobal("fetch", fetch); - const settled = await quest("--root", s.repo, "review", "--settle", "--responses", responses); - expect(settled.code, settled.stderr).toBe(0); - expect(s.calls()).toHaveLength(calls); - expect(settled.stdout).not.toContain("\nusage "); - expect(settled.stdout).toContain(`head ${head}\n`); - expect(settled.stdout).toContain( - `settled round 1's low findings at ${reviewed}, by the author; no reviewer ran\n`, - ); - expect(settled.stdout).toContain( - `earlier 1 ${reviewed}\n\n1. fixed low src/lib.ts:2 Off by one\n src/lib.ts:1 guards it.\n`, - ); - expect(settled.stdout).toMatch(/\nnext done: no findings remain on this head\n$/); - const record = pull.body.split("## Review\n\n")[1]!; - expect(record).toContain( - `- Head: ${head}, where the author settled the low findings reviewed at ${reviewed.slice(0, 12)} with no further round\n`, - ); - expect(record).toContain( - "- Rounds: 1; findings 1 (low 1): fixed 1, refuted 0, open 0\n- Findings: none\n" + - `- The last round's findings, at ${reviewed.slice(0, 12)}, settled by the author:\n` + - " - Fixed: low, `src/lib.ts:2`: `Off by one`; `src/lib.ts:1 guards it.`\n", - ); - // The clean reviewer stands beside it, its tally unchanged. - expect(record).toContain( - "- Session: `ses_1`, round 1\n- Score: 10/10, `Lands as is.`\n- Rounds: 1; findings 0: fixed 0, refuted 0, open 0\n", - ); - expect(writes).toHaveLength(2); - - // Nothing is left to settle, and the next round checks the fix with no responses, counting the finding once. - expect((await quest("--root", s.repo, "review", "--settle", "--responses", responses)).code).toBe(2); - vi.stubEnv("FAKE_REPLY", JSON.stringify({ score: 10, rationale: "Lands as is.", findings: [] })); - // A head without the settled one starts a new session that reviews the whole change, since the settlement - // cleared the findings raised there; the clean reviewer's round of that head stands. - git(s.repo, "checkout", "-q", "--detach", reviewed); - const before = s.calls().length; - const back = await quest("--root", s.repo, "review", "--branch", "feature", "--json"); - expect(back.code, back.stderr).toBe(0); - expect(s.calls()[before]!.argv.slice(0, 2)).not.toEqual(["exec", "resume"]); - expect(JSON.parse(back.stdout).rounds.map((r: { unchanged: boolean }) => r.unchanged)).toEqual([false, true]); - git(s.repo, "checkout", "-q", "-"); - const next = await quest("--root", s.repo, "review", "--json"); - expect(next.code, next.stderr).toBe(0); - const resumed = s.calls().at(-1)!; - expect(resumed.argv.slice(0, 2)).toEqual(["exec", "resume"]); - expect(resumed.stdin).toContain(`git diff ${reviewed} ${head}`); - expect( - JSON.parse(next.stdout).tallies.map((t: Tally) => [ - t.reviewer, - t.rounds, - t.findings.low, - t.fixed, - t.refuted, - ]), - ).toEqual([ - // Codex reviewed the head back at the reviewed one, too. - ["codex", 3, 1, 1, 0], - ["opencode", 2, 0, 0, 0], - ]); - } finally { - vi.unstubAllGlobals(); - } -}); diff --git a/test/usage.test.ts b/test/usage.test.ts index 034000d1..7eea14ed 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -204,7 +204,6 @@ describe("PR records", () => { ], }, unchanged: false, - settled: false, }; const second: Round = { From 82b0f7aaae653b6437e15090a2d5988424318c1f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:27:45 -0800 Subject: [PATCH 14/17] feat(usage): count a merged PR's last all-low round as settled Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- quest/m1/model-check.md | 2 +- src/cli/main.ts | 1 + src/cli/review-record.ts | 12 +++++- src/cli/usage.ts | 28 +++++++++++++- test/usage.test.ts | 79 +++++++++++++++++++++++++++++++--------- 6 files changed, 104 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index cfeb4746..9ec1ccd0 100644 --- a/README.md +++ b/README.md @@ -1531,7 +1531,9 @@ it or Codex marks it a review, implementing when it began before the PR opened, and iterating otherwise. An opencode session, whose log quest does not read, is the reviewing session of the newest PR whose `## Review` record names it. Each PR shows its `## Models` size and implementers; each reviewer the `quest review` record names, with its latest round and score and its tally of -findings across every round; the land fixer's fixes; the later PRs that list it under +findings across every round, where a merged PR whose last round left only low +findings open counts them settled, since `/quest-review` fixes or refutes those +before merge without another round; the land fixer's fixes; the later PRs that list it under `## Caused by`; and each online reviewer's record, read from GitHub. An online reviewer is any bot account whose review has a body or opens a diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index 02490928..4d35e6df 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -30,7 +30,7 @@ with `quest usage --since `, whose accuracy rollup reads ea `## Models` size and implementers and its `## Review` reviewer. It tallies each implementer by the size on its own `Implementer:` line where it carries one, such as `(iterate, XS)`, not the PR's `## Models` size, so a small iterate on a large PR counts as the -work it was routed for. It counts each reviewer's findings, fixed, refuted, and open, +work it was routed for. It counts each reviewer's findings, fixed, refuted, settled (a merged PR's last all-low round), and open, across every round on the branch, from the tally its record keeps, and leaves design checkpoints and outages to count by hand: diff --git a/src/cli/main.ts b/src/cli/main.ts index 3da58382..330f16d4 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1106,6 +1106,7 @@ export function main(argv: string[], out: Output): number | Promise { "## Review record names it, as reviewing. Tokens and cost come from ccusage (session --json), the " + "cost an estimate at each model's API list price. Each PR also shows its ## Models size and implementers, " + "each reviewer the ## Review record names, with its latest round and score and its tally of findings across every round, " + + "where a merged PR whose last round left only low findings open counts them settled, since the author fixes or refutes those before merge, " + "the later PRs that list it under ## Caused by, and each online reviewer's record from GitHub: whether it " + "reviewed the last head, and its findings accepted, declined, unclassified, or unanswered. An online reviewer " + "is a bot whose review has a body or opens a thread, or a review whose body opens with Automated review by ; a finding is a " + diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index 6a4e3e44..9728e97b 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -31,6 +31,8 @@ export interface Counts { findings: Tally["findings"]; fixed: number; refuted: number; + /** Open findings of a merged PR's last all-low round, which the review rule fixes or refutes before merge. */ + settled: number; open: number; } @@ -40,12 +42,13 @@ export function findingsText(counts: Omit): string { const severities = SEVERITIES.filter((severity) => counts.findings[severity] > 0) .map((severity) => `${severity} ${counts.findings[severity]}`) .join(", "); - return `findings ${total}${severities && ` (${severities})`}: fixed ${counts.fixed}, refuted ${counts.refuted}, open ${counts.open}`; + return `findings ${total}${severities && ` (${severities})`}: fixed ${counts.fixed}, refuted ${counts.refuted}${counts.settled > 0 ? `, settled ${counts.settled}` : ""}, open ${counts.open}`; } /** A tally's counts, where a finding neither fixed nor refuted is open. */ const counted = (tally: Tally): Counts => ({ ...tally, + settled: 0, open: SEVERITIES.reduce((sum, severity) => sum + tally.findings[severity], 0) - tally.fixed - tally.refuted, }); @@ -106,6 +109,8 @@ export interface ReviewSummary { score: number | null; /** Its findings across every round on the branch, null on a record written before tallies. */ tally: Counts | null; + /** The severity of each finding its latest round left open. */ + open: Tally["findings"]; } const MARK = "Rewritten by `quest review` after each round."; @@ -140,6 +145,9 @@ export function parseReview(body: string): ReviewSummary[] { ? null : match(/^- Session: (`+) ?(.*?) ?\1, round (\d+)\r?$/m); const score = /^- Score: (\d+)\/10,/m.exec(text); + const open = { high: 0, medium: 0, low: 0 }; + for (const [, severity] of text.matchAll(/^ - (High|Medium|Low), /gm)) + open[severity!.toLowerCase() as keyof typeof open]++; return { harness: reviewer[1]!, model: reviewer[2]!, @@ -148,6 +156,7 @@ export function parseReview(body: string): ReviewSummary[] { round: session && Number(session[3]), score: score ? Number(score[1]) : null, tally, + open, }; }); } @@ -170,6 +179,7 @@ function parseCounts(text: string): Counts | null { findings, fixed: Number(found[4]), refuted: Number(found[5]), + settled: 0, open: Number(found[6]), }; const total = SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0); diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 8b72a816..f5688e8d 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -407,6 +407,22 @@ function causes(body: string, repository: Repository): number[] { const sized = (value: string | null | undefined): Size | null => (SIZES as readonly string[]).includes(value ?? "") ? (value as Size) : null; +/** + * A merged PR's reviews with its last round's open findings settled when all of them are low, since the review + * rule fixes or refutes those before merge without another round; nothing records which, so they count as settled. + */ +function settledOnMerge(reviews: ReviewSummary[], merged: boolean): ReviewSummary[] { + const settles = + merged && + reviews.every( + ({ tally, open }) => tally === null || (open.high === 0 && open.medium === 0 && tally.open === open.low), + ); + if (!settles) return reviews; + return reviews.map((review) => + review.tally === null ? review : { ...review, tally: { ...review.tally, settled: review.tally.open, open: 0 } }, + ); +} + /** Each PR's record, without its sessions yet. */ function pullRows(pulls: Pull[], repository: Repository): PullRow[] { const fixedBy = new Map(); @@ -429,7 +445,7 @@ function pullRows(pulls: Pull[], repository: Repository): PullRow[] { const fields = implementer(item); return fields ? [{ ...fields, size: sized(fields.size) }] : []; }), - review: parseReview(each.body), + review: settledOnMerge(parseReview(each.body), each.state === "merged"), landFixes: fixes(each.body), fixedBy: (fixedBy.get(each.number) ?? []).sort((a, b) => a - b), online: [], @@ -618,7 +634,14 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { reviewed: 0, scored: 0, score: 0, - tally: { rounds: 0, findings: { high: 0, medium: 0, low: 0 }, fixed: 0, refuted: 0, open: 0 }, + tally: { + rounds: 0, + findings: { high: 0, medium: 0, low: 0 }, + fixed: 0, + refuted: 0, + settled: 0, + open: 0, + }, landFixes: 0, fixes: 0, }; @@ -643,6 +666,7 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { for (const severity of SEVERITIES) row.tally.findings[severity] += tally!.findings[severity]; row.tally.fixed += tally!.fixed; row.tally.refuted += tally!.refuted; + row.tally.settled += tally!.settled; row.tally.open += tally!.open; } } diff --git a/test/usage.test.ts b/test/usage.test.ts index 7eea14ed..cf59f453 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -249,7 +249,15 @@ describe("PR records", () => { session: THREAD, round: 3, score: 7, - tally: { rounds: 3, findings: { high: 1, medium: 1, low: 2 }, fixed: 2, refuted: 1, open: 1 }, + tally: { + rounds: 3, + findings: { high: 1, medium: 1, low: 2 }, + fixed: 2, + refuted: 1, + settled: 0, + open: 1, + }, + open: { high: 0, medium: 1, low: 0 }, }, { harness: "opencode", @@ -258,7 +266,15 @@ describe("PR records", () => { session: "ses_1", round: 1, score: 10, - tally: { rounds: 1, findings: { high: 0, medium: 0, low: 0 }, fixed: 0, refuted: 0, open: 0 }, + tally: { + rounds: 1, + findings: { high: 0, medium: 0, low: 0 }, + fixed: 0, + refuted: 0, + settled: 0, + open: 0, + }, + open: { high: 0, medium: 0, low: 0 }, }, // A reviewer the run left out, such as the pick a new one replaced, keeps its tally. { @@ -268,7 +284,15 @@ describe("PR records", () => { session: null, round: null, score: null, - tally: { rounds: 2, findings: { high: 2, medium: 0, low: 0 }, fixed: 1, refuted: 0, open: 1 }, + tally: { + rounds: 2, + findings: { high: 2, medium: 0, low: 0 }, + fixed: 1, + refuted: 0, + settled: 0, + open: 1, + }, + open: { high: 0, medium: 0, low: 0 }, }, ]); // An open finding keeps the head short; once no reviewer reports one, it is named in full. @@ -467,15 +491,23 @@ describe("usage", () => { updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: { full_name: "fork/proj" } }, }), - // Two deleted forks' PRs from a branch of the same name: neither names its repository, so each counts. - ...[5, 6].map((number) => - pr(number, "quest/m0/a", body9, "2026-11-02T00:00:00Z", { - state: "open", - merged_at: null, - closed_at: null, - updated_at: "2026-11-06T00:00:00Z", - head: { ref: "quest/m0/a", repo: null }, - }), + // Two deleted forks' merged PRs from a branch of the same name: neither names its repository, so each + // counts. Each merged with one finding open: #5's last round's all low, so merging settled it, and + // #6's medium, so it stays open. + ...[ + [5, "low 1): fixed 1, refuted 0, open 1", "Low"], + [6, "medium 1): fixed 1, refuted 0, open 1", "Medium"], + ].map(([number, counts, severity]) => + pr( + number as number, + "quest/m0/a", + body9.replace( + "low 1): fixed 1, refuted 1, open 0\n- Findings: none", + `${counts}\n- Findings, open until the next round settles them:\n - ${severity}, \`a:1\`: \`x\``, + ), + "2026-11-02T00:00:00Z", + { updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: null } }, + ), ), pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), ]; @@ -632,9 +664,21 @@ describe("usage", () => { ["L", "reviewing", "codex", 1], ["L", "reviewing", "opencode", 4], ]); - const tally = { rounds: 2, findings: { high: 1, medium: 1, low: 1 }, fixed: 2, refuted: 1, open: 0 }; + const tally = { + rounds: 2, + findings: { high: 1, medium: 1, low: 1 }, + fixed: 2, + refuted: 1, + settled: 0, + open: 0, + }; // #10's, #7's, and each deleted fork's. - const four = { rounds: 8, findings: { high: 4, medium: 4, low: 4 }, fixed: 8, refuted: 4, open: 0 }; + const four = { rounds: 8, findings: { high: 4, medium: 5, low: 3 }, fixed: 8, refuted: 2, settled: 1, open: 1 }; + const codex = (n: number) => report.pulls.find((p) => p.number === n)!.review[0]!.tally; + expect([codex(5), codex(6)].map((t) => [t!.settled, t!.open])).toEqual([ + [1, 0], + [0, 1], + ]); expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], ["L", "claude-opus-5-5", 5, 4, 68, four, 1], @@ -650,11 +694,11 @@ describe("usage", () => { unanswered: 0, }, ]); - // #11 and #10 merged; #9 closed and #7 is open, so neither counts. + // #11, #10, #5, and #6 merged; #9 closed and #7 is open, so neither counts. expect(report.online).toEqual([ { reviewer: "rabbit[bot]", - merged: 2, + merged: 4, reviewed: 1, finalHead: 1, findings: 1, @@ -670,11 +714,12 @@ describe("usage", () => { ); expect(text).toContain("rounds 2.0, findings 3 (high 1, medium 1, low 1): fixed 2, refuted 1, open 0;"); expect(text).toContain("#10 L merged: PR 10 (quest/m0/a)"); + expect(text).toContain("findings 2 (high 1, low 1): fixed 1, refuted 0, settled 1, open 0\n"); expect(text).toContain( "online review: rabbit[bot], at the final head; 1 finding: 1 accepted, 0 declined, 0 unclassified, 0 unanswered", ); expect(text).toContain( - "rabbit[bot] reviewed 1/2 (50%), final head 1 (50%), 1.0 findings per PR reviewed, 100% accepted;", + "rabbit[bot] reviewed 1/4 (25%), final head 1 (25%), 1.0 findings per PR reviewed, 100% accepted;", ); expect(text).toContain("fixed later by: #11"); expect(text).not.toContain("SECRET"); From f76457e3bf88d8dbf5c6f4b9bb278ae4e589e0c3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:31:02 -0800 Subject: [PATCH 15/17] quest: drop charts' links to the finished review-tallies quest Co-Authored-By: Claude Opus 5.5 --- quest/m0/routing-evidence/charts.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/quest/m0/routing-evidence/charts.md b/quest/m0/routing-evidence/charts.md index 6ccfed08..2ed1283d 100644 --- a/quest/m0/routing-evidence/charts.md +++ b/quest/m0/routing-evidence/charts.md @@ -69,16 +69,13 @@ Agent's choices, for the implementer to revisit: readout: which `quest usage --json` fields feed each axis, how cells with too few PRs are marked rather than dropped, and that each chart names its date range and PR counts. -- `quest usage --json` reports each reviewer's latest round, and - [review tallies](/quest/m0/routing-evidence/review-tallies.md) add totals - across rounds. If the tally line does not also record the first round's - findings, which the readout compares, add them there. +- `quest usage --json` reports each reviewer's latest round and its tally + across rounds. The tally does not record the first round's findings, which + the readout compares; add them to it. - Make the first charts from the data on main when this lands, link them in this quest's PR, and say which cells are still too thin. ## Required -- [Review tallies](/quest/m0/routing-evidence/review-tallies.md) - the - findings each reviewer raised across rounds - [usage through token-max](/quest/m0/routing-evidence/usage-token-max.md) - the cost axis, which today misses sub-agent transcripts, 72% of host spend From 61c71df6e2a753b238abefb17c2cca67a58e0695 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:20:46 -0800 Subject: [PATCH 16/17] fix(review): read a settled count back, keep the head short for a left-out open tally Co-Authored-By: Claude Opus 5.5 --- quest/m0/routing-evidence/charts.md | 6 +++--- src/cli/main.ts | 2 +- src/cli/review-record.ts | 15 ++++++++++----- src/cli/usage.ts | 6 +++--- test/usage.test.ts | 13 +++++++++++-- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/quest/m0/routing-evidence/charts.md b/quest/m0/routing-evidence/charts.md index 2ed1283d..580c174f 100644 --- a/quest/m0/routing-evidence/charts.md +++ b/quest/m0/routing-evidence/charts.md @@ -69,9 +69,9 @@ Agent's choices, for the implementer to revisit: readout: which `quest usage --json` fields feed each axis, how cells with too few PRs are marked rather than dropped, and that each chart names its date range and PR counts. -- `quest usage --json` reports each reviewer's latest round and its tally - across rounds. The tally does not record the first round's findings, which - the readout compares; add them to it. +- `quest usage --json` reports each reviewer's latest round and its tally, + which sums findings across rounds. If the readout compares first-round + findings on their own, record them beside the tally. - Make the first charts from the data on main when this lands, link them in this quest's PR, and say which cells are still too thin. diff --git a/src/cli/main.ts b/src/cli/main.ts index 330f16d4..1a485b95 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1210,7 +1210,7 @@ export function main(argv: string[], out: Output): number | Promise { "starts a new session that reviews the whole change and still weighs --responses. A round that would " + "resume on the head the last round reviewed, when that round found nothing, runs no reviewer and " + "repeats that round, marked unchanged, so on the same head only a reviewer with findings to settle runs. " + - "--new starts new sessions, continuing each reviewer's tally of its findings on the branch, less those it drops unsettled. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + + "--new starts new sessions, continuing each reviewer's tally of its findings on the branch, less those it drops unsettled, unless the saved state is one this quest cannot read. Each reviewer's session is kept per branch in the clone's common Git directory, so any worktree of the clone resumes it, except opencode's, which stays in the checkout it started in. " + "A checked-out branch with an upstream other than the base continues its upstream branch's session, so a " + "temporary branch tracking the PR's head shares the head's rounds. " + "--branch names the branch instead of the checked-out one, as a detached HEAD requires, such as a " + diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index 9728e97b..d25d8253 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -58,7 +58,9 @@ const counted = (tally: Tally): Counts => ({ */ export function reviewRecord(rounds: Round[], tallies: Tally[]): string { const [shared] = rounds; - const clean = rounds.every((round) => round.findings.length === 0); + // A reviewer the run left out may still hold open findings, which keep the head from passing too. + const clean = + rounds.every((round) => round.findings.length === 0) && tallies.every((tally) => counted(tally).open === 0); const lines = [ MARK, "", @@ -165,7 +167,10 @@ export function parseReview(body: string): ReviewSummary[] { function parseCounts(text: string): Counts | null { const line = /^- Rounds: (.*?)\r?$/m.exec(text); if (!line) return null; - const found = /^(\d+); findings (\d+)(?: \(([^)]*)\))?: fixed (\d+), refuted (\d+), open (\d+)$/.exec(line[1]!); + const found = + /^(\d+); findings (\d+)(?: \(([^)]*)\))?: fixed (\d+), refuted (\d+)(?:, settled (\d+))?, open (\d+)$/.exec( + line[1]!, + ); if (!found) throw new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); const findings = { high: 0, medium: 0, low: 0 }; for (const part of found[3]?.split(", ") ?? []) { @@ -179,11 +184,11 @@ function parseCounts(text: string): Counts | null { findings, fixed: Number(found[4]), refuted: Number(found[5]), - settled: 0, - open: Number(found[6]), + settled: Number(found[6] ?? 0), + open: Number(found[7]), }; const total = SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0); - if (total !== Number(found[2]) || counts.fixed + counts.refuted + counts.open !== total) + if (total !== Number(found[2]) || counts.fixed + counts.refuted + counts.settled + counts.open !== total) throw new Error(`the ## Review record's Rounds line does not add up: ${line[1]}`); return counts; } diff --git a/src/cli/usage.ts b/src/cli/usage.ts index f5688e8d..81fcc270 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -101,10 +101,10 @@ export interface AccuracyRow { handedBack: number; /** PRs whose record tallies their reviews; a record written before tallies counts for none of what follows. */ reviewed: number; - /** The reviewers' latest rounds on these PRs that have a score, and the scores' sum. */ + /** The reviewers' latest rounds on the reviewed PRs that have a score, and the scores' sum. */ scored: number; score: number; - /** Every reviewer's findings across every round on these PRs, and as `rounds`, each PR's most rounds. */ + /** Every reviewer's findings across every round on the reviewed PRs, and as `rounds`, each PR's most rounds. */ tally: Counts; /** The land fixer's model-run fixes, leaving out the watcher's own base merges. */ landFixes: number; @@ -415,7 +415,7 @@ function settledOnMerge(reviews: ReviewSummary[], merged: boolean): ReviewSummar const settles = merged && reviews.every( - ({ tally, open }) => tally === null || (open.high === 0 && open.medium === 0 && tally.open === open.low), + ({ tally, open }) => open.high === 0 && open.medium === 0 && (tally === null || tally.open === open.low), ); if (!settles) return reviews; return reviews.map((review) => diff --git a/test/usage.test.ts b/test/usage.test.ts index cf59f453..6720600f 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -297,8 +297,11 @@ describe("PR records", () => { ]); // An open finding keeps the head short; once no reviewer reports one, it is named in full. expect(body).not.toContain("b".repeat(40)); - const clean = reviewRecord([{ ...round, findings: [] }, second], tallies); - expect(clean).toContain(`- Head: ${"b".repeat(40)}\n`); + // Claude's left-out tally still holds an open finding, which keeps it short too. + const clean = [{ ...round, findings: [] }, second]; + expect(reviewRecord(clean, tallies)).not.toContain("b".repeat(40)); + const settledAll = [tally("codex", 3, { high: 1, medium: 1, low: 2 }, 3, 1), tallies[1]!]; + expect(reviewRecord(clean, settledAll)).toContain(`- Head: ${"b".repeat(40)}\n`); expect(fixes(body)).toEqual(["codex, gpt-6.1-sol, medium effort"]); }); @@ -306,9 +309,15 @@ describe("PR records", () => { const record = reviewRecord([second], [tally("opencode", 1, { high: 0, medium: 0, low: 0 }, 0, 0)]); const older = `## Review\n\n${record.replace(/^- Rounds: .*\n/m, "")}\n`; expect(parseReview(older)[0]!.tally).toBeNull(); + // The settled count `quest usage` prints reads back. + const settled = "1; findings 2 (low 2): fixed 1, refuted 0, settled 1, open 0"; + expect( + parseReview(`## Review\n\n${record.replace(/^- Rounds: .*$/m, `- Rounds: ${settled}`)}\n`)[0]!.tally, + ).toEqual({ rounds: 1, findings: { high: 0, medium: 0, low: 2 }, fixed: 1, refuted: 0, settled: 1, open: 0 }); for (const line of [ "1; findings 1 (high 1): fixed 0, refuted 0, open 0", "1; findings 2 (high 1): fixed 1, refuted 0, open 1", + "1; findings 2 (low 2): fixed 1, refuted 0, settled 1, open 1", "1; findings 1 (severe 1): fixed 1, refuted 0, open 0", "one", ]) From 6817927968039bb6909ac763f08fb1d038a356c6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:30:07 -0800 Subject: [PATCH 17/17] fix(usage): count a branch's latest tally by its rounds, keep a recorded settled count Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +++-- src/cli/usage.ts | 28 ++++++++++++++++++++-------- test/usage.test.ts | 28 ++++++++++++++++++++++++---- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9ec1ccd0..6ee5b989 100644 --- a/README.md +++ b/README.md @@ -1554,8 +1554,9 @@ unanswered otherwise. Rollups follow: spend by size, role, harness, model, and effort; accuracy by size and implementer, which sums the tallies, so it counts every finding -caught before merge. Since a branch's tallies span every PR from it, only its -newest PR in the report counts them, and a record written before tallies +caught before merge. Since a branch's tallies span every PR from it, only the +PR whose record holds its latest tally, by the most rounds and then the newest, +counts them, and a record written before tallies counts as unreviewed. An `Implementer:` line's own size, as `(iterate, XS)`, counts over the PR's; and, over the merged PRs, each online reviewer's coverage, final heads reviewed, findings per PR reviewed, and diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 81fcc270..79e2e367 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -419,7 +419,9 @@ function settledOnMerge(reviews: ReviewSummary[], merged: boolean): ReviewSummar ); if (!settles) return reviews; return reviews.map((review) => - review.tally === null ? review : { ...review, tally: { ...review.tally, settled: review.tally.open, open: 0 } }, + review.tally === null + ? review + : { ...review, tally: { ...review.tally, settled: review.tally.settled + review.tally.open, open: 0 } }, ); } @@ -603,13 +605,23 @@ function spendRows(pulls: PullRow[]): SpendRow[] { function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const rows = new Map(); - // A branch's saved tallies span every PR from it, so only its newest PR that has both a tally and an - // implementer to count it for counts them. A deleted fork has no repository, so its PR stands alone. + // A branch's saved tallies span every PR from it, so only one PR that has both a tally and an implementer + // to count it for counts them: the one whose record holds the latest tally, as the most rounds show, since + // each round rewrites only its own PR's record; the newest breaks a tie. A deleted fork has no repository, + // so its PR stands alone. const branch = (each: PullRow) => JSON.stringify([each.repository ?? each.number, each.branch]); - const newest = new Map(); - for (const each of pulls) - if (each.implementers.length > 0 && each.review.some((review) => review.tally !== null)) - newest.set(branch(each), Math.max(newest.get(branch(each)) ?? 0, each.number)); + const rounds = (each: PullRow) => each.review.reduce((sum, review) => sum + (review.tally?.rounds ?? 0), 0); + const latest = new Map(); + for (const each of pulls) { + if (each.implementers.length === 0 || each.review.every((review) => review.tally === null)) continue; + const other = latest.get(branch(each)); + if ( + other === undefined || + rounds(each) > rounds(other) || + (rounds(each) === rounds(other) && each.number > other.number) + ) + latest.set(branch(each), each); + } for (const each of pulls) { // A session that pushed twice is one implementer of the PR, whether or not each line names its size. const distinct = new Map( @@ -653,7 +665,7 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { row.fixes += each.fixedBy.length; row.landFixes += each.landFixes.filter((by) => by !== MERGER).length; const tallied = - newest.get(branch(each)) === each.number ? each.review.filter((review) => review.tally !== null) : []; + latest.get(branch(each)) === each ? each.review.filter((review) => review.tally !== null) : []; if (tallied.length > 0) { row.reviewed++; row.tally.rounds += Math.max(...tallied.map((review) => review.tally!.rounds)); diff --git a/test/usage.test.ts b/test/usage.test.ts index 6720600f..7a9781c8 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -518,6 +518,19 @@ describe("usage", () => { { updated_at: "2026-11-06T00:00:00Z", head: { ref: "quest/m0/a", repo: null } }, ), ), + // Two open PRs from another branch, whose older record holds the later tally (3 Sol rounds against + // 2), as when a round rewrote only its own PR's record: the older counts, not the stale newer one. + ...[ + [3, body9.replace("- Rounds: 2;", "- Rounds: 3;")], + [4, body9], + ].map(([number, body]) => + pr(number as number, "quest/m0/b", body as string, "2026-11-02T00:00:00Z", { + state: "open", + merged_at: null, + closed_at: null, + updated_at: "2026-11-06T00:00:00Z", + }), + ), pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), ]; const requests: string[] = []; @@ -640,7 +653,7 @@ describe("usage", () => { test("ties each session to its PR and role, and rolls up spend and accuracy", async () => { const { root, deps, priced } = setup(); const report = await usage(root, { since: "2026-11-05" }, deps(priced)); - expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7, 13, 5, 6]); + expect(report.pulls.map((p) => p.number)).toEqual([11, 10, 9, 7, 13, 5, 6, 3, 4]); const ten = report.pulls.find((p) => p.number === 10)!; expect(ten.sessions.map((s) => [s.harness, s.session, s.role, s.effort])).toEqual([ ["claude", CLAUDE, "implementing", "high"], @@ -681,8 +694,15 @@ describe("usage", () => { settled: 0, open: 0, }; - // #10's, #7's, and each deleted fork's. - const four = { rounds: 8, findings: { high: 4, medium: 5, low: 3 }, fixed: 8, refuted: 2, settled: 1, open: 1 }; + // #10's, #7's, each deleted fork's, and #3's. + const five = { + rounds: 11, + findings: { high: 5, medium: 6, low: 4 }, + fixed: 10, + refuted: 3, + settled: 1, + open: 1, + }; const codex = (n: number) => report.pulls.find((p) => p.number === n)!.review[0]!.tally; expect([codex(5), codex(6)].map((t) => [t!.settled, t!.open])).toEqual([ [1, 0], @@ -690,7 +710,7 @@ describe("usage", () => { ]); expect(report.accuracy.map((a) => [a.size, a.model, a.pulls, a.reviewed, a.score, a.tally, a.fixes])).toEqual([ ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], - ["L", "claude-opus-5-5", 5, 4, 68, four, 1], + ["L", "claude-opus-5-5", 7, 5, 85, five, 1], ]); expect(ten.online).toEqual([ {