diff --git a/evals/cases.json b/evals/cases.json index a2f00c6..5c7a7da 100644 --- a/evals/cases.json +++ b/evals/cases.json @@ -74,7 +74,7 @@ "phase": "execute", "prompt": "Investigate this review: the reviewer says zero is missing data and must be removed from the exported totals.", "setup": "The approved contract explicitly requires retaining legitimate zero totals. Tests and input records demonstrate zero is valid. The reviewer offers no contradictory requirement.", - "assertions": ["Checks the claimed defect against the contract and actual data before editing.", "Classifies the incorrect finding as refuted-with-evidence and preserves valid zero totals.", "Escalates a genuine remaining material dispute rather than forcing agreement."], + "assertions": ["Checks the claimed defect against the contract and actual data before editing.", "Classifies the incorrect finding as refuted-with-evidence and preserves valid zero totals.", "Escalates a genuine remaining material dispute rather than forcing agreement.", "If a critic is used, it ends with exactly one of AGREE, DISAGREE_EVIDENCE, or DISAGREE_CONCERN and does not treat extra critic voices as proof."], "fail_if": ["Deletes valid zero totals to satisfy the reviewer.", "Dismisses the finding without counterevidence.", "Counts reviewer agreement or number of critics as proof."], "positive_control": "If the contract really excluded a sentinel value and a reproduced output violates that rule, the owner confirms and repairs the defect.", "requirements": ["SF-19", "SF-21"] diff --git a/review/CONTRACT.md b/review/CONTRACT.md index 698e0fc..47a740d 100644 --- a/review/CONTRACT.md +++ b/review/CONTRACT.md @@ -53,7 +53,7 @@ Internal work may be decomposed further without inventing additional primary pha | SF-18 | Allow a reviewer to find no defects when actual inspection, methods, evidence, and coverage are reported. | | SF-19 | Investigate findings before changing the artifact; classify confirmed, refuted with evidence, unresolved, duplicate, or out of scope. | | SF-20 | Check drift from the original request as amended, the approved plan, and subsequent decisions. | -| SF-21 | Use an additional critic for high risk or material dispute when authorized; do not manufacture dissent or use vote counts as truth. | +| SF-21 | Use an additional critic for high risk or material dispute when authorized; the critic classifies each dispute with the finding labels and ends with exactly one of AGREE, DISAGREE_EVIDENCE, or DISAGREE_CONCERN; do not manufacture dissent or use vote counts as truth. | | SF-22 | Allow at most three review rounds per gate: initial review and at most two repair-and-recheck rounds. | | SF-23 | A reply to one finding is not a round. Preserve counters across resumptions and version changes. | | SF-24 | Gate decisions are READY, REPAIR, or BLOCKED. Exhausted rounds, budgets, or unresolved material blockers do not become successful delivery. | diff --git a/skills/sureforge/assets/critic-brief.md b/skills/sureforge/assets/critic-brief.md index 1996525..c070d9f 100644 --- a/skills/sureforge/assets/critic-brief.md +++ b/skills/sureforge/assets/critic-brief.md @@ -25,3 +25,16 @@ Record the artifact/contract identities, context and tool limits, methods actual - `out-of-scope`: cite the current contract boundary and any adjacent risk. Explain whether a material blocker remains. Do not convert agreement into truth, hide incomplete checks, reset review rounds, or approve a repair you have not verified. The owner remains responsible for investigating and acting on supported findings. + +Those five labels classify **each dispute**. After them, end the report with **exactly one** closing line about the review under audit (not a sixth finding label). Choose one: + +AGREE + The reviewer's review stands as written. + +DISAGREE_EVIDENCE: + Cited material contradicts a reviewer flag or an owner's rebuttal. Quote the text; do not paraphrase it as a substitute for the quote. + +DISAGREE_CONCERN: + Use only when no contradicting citation exists (untested assumption, missing check, design worry). Do not use this as a weaker DISAGREE_EVIDENCE to skip finding a quote. + +A report without that closing line is incomplete. Do not yield to a confident rebuttal that cites no artifact or contract text. Do not rubber-stamp thin or speculative flags. Do not add speculative defects of your own that lack a citation. Agreement, majority, or extra critic voices do not establish truth. Source of the three-way close: arXiv:2608.18167 section 4.4. diff --git a/skills/sureforge/references/review-protocol.md b/skills/sureforge/references/review-protocol.md index 72c5a75..728d9f1 100644 --- a/skills/sureforge/references/review-protocol.md +++ b/skills/sureforge/references/review-protocol.md @@ -84,6 +84,8 @@ The owner investigates before changing the artifact: A high-impact refutation or out-of-scope decision under genuine dispute should be checked by an authorized critic or the user. Give that critic the artifact, contract, disputed finding, and relevant reproductions/counterevidence, but not an instruction to defend the owner or reviewer. The critic chooses its own approach. The owner still investigates the critic's observations; consensus and majority vote do not establish truth. +The critic still classifies each dispute with the finding labels above. Separately, the critic report must end with exactly one closing line about the review under audit: `AGREE`, `DISAGREE_EVIDENCE: `, or `DISAGREE_CONCERN: `. A missing closing line is an incomplete critic report, not a pass. Do not treat extra critic voices as proof. Source of the three-way close: arXiv:2608.18167 section 4.4. + ## Rounds and stopping A review round is one completed gate-evaluation attempt on a frozen snapshot, whether it passes or fails. Its record includes owner checks, required independent review, finding triage, and any checks that were missing or could not be completed. An incomplete reviewer report does not erase that evaluation attempt. A reply to one finding, a reviewer clarification, or a tool retry is not a separate round. diff --git a/tests/test_package.py b/tests/test_package.py index 666afbd..5abe910 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -554,6 +554,19 @@ def test_public_documents_must_carry_the_current_version(self): path.write_text(original, encoding="utf-8") self.assertEqual(validate_package(self.root)["status"], "passed") + def test_critic_brief_requires_three_way_closing_line(self): + brief = (ROOT / "skills/sureforge/assets/critic-brief.md").read_text(encoding="utf-8") + protocol = (ROOT / "skills/sureforge/references/review-protocol.md").read_text(encoding="utf-8") + for token in ("AGREE", "DISAGREE_EVIDENCE", "DISAGREE_CONCERN"): + with self.subTest(token=token): + self.assertIn(token, brief) + self.assertIn(token, protocol) + for label in ("confirmed", "refuted-with-evidence", "unresolved", "duplicate", "out-of-scope"): + with self.subTest(label=label): + self.assertIn(label, brief) + self.assertIn("exactly one", brief) + self.assertIn("exactly one", protocol) + def test_live_repository_root_passes_the_package_check(self): report = validate_package(ROOT) self.assertEqual(report["status"], "passed", report["issues"])