fix(scaffold-verifier): refuse runnable check on observed partial read with collateral miss (#2142)

.gitleaksignore

@@ -103,6 +103,40 @@ e8ec96725c4451aa64cb97230c18d04b55d98503:packages/flow/tests/e2e/test_mcp_teach_
 8ccf2cf3360b7d7720af04c36eee4f8e756bdf6e:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:554
 8ccf2cf3360b7d7720af04c36eee4f8e756bdf6e:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:630
 8ccf2cf3360b7d7720af04c36eee4f8e756bdf6e:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:642
+# The same fixtures after PR #1928 was merged to main (commit b0fd15ee).
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:66
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:182
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:193
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:248
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:290
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:348
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:397
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:416
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.2.0/provenance.json:generic-api-key:428
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:85
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:155
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:190
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:275
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:351
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:414
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:488
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:508
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:528
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:570
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:643
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.5.0/provenance.json:generic-api-key:655
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:85
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:132
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:168
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:254
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:316
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:379
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:468
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:490
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:511
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:554
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:630
+b0fd15ee8584cc3b646ae9bab04b4f9acbc59292:packages/flow/tests/fixtures/bundle_compatibility/2.6.0/provenance.json:generic-api-key:642
 # OpenAdapt#1872 network-save redaction tests (PR #2014, first commit): synthetic
 # credentials the redactor must withhold. Later commits assemble them at runtime.
 ccc7c617e5cd8082909b455ec6176a5e4c0dd4f0:packages/flow/tests/test_extension_network.py:generic-api-key:44

packages/flow/openadapt_flow/__main__.py

@@ -6220,6 +6220,7 @@ def _cmd_scaffold_verifier(args: argparse.Namespace) -> int:
     from openadapt_flow.scaffold_verifier import (
         NEXT_COMMANDS_TEMPLATE,
         build_record_check,
+        report_draft_fallback,
         write_draft,
     )
 
@@ -6245,31 +6246,11 @@ def optional_path(name: str) -> Optional[Path]:
     if built is not None:
         print(built.summary)
         return 0 if built.working else 1
-    if getattr(args, "probe_reads", False):
-        import json as _json
-
-        from openadapt_flow import sealed_set as _sealed_set
-        from openadapt_flow.scaffold_verifier import READ_DISCOVERY_FILE
-
-        found = (out_dir or Path(args.source)) / READ_DISCOVERY_FILE
-        status, reason = "unavailable", ""
-        try:
-            data = _json.loads(_sealed_set.read_text(found))
-            status, reason = str(data.get("status")), str(data.get("reason") or "")
-        except (OSError, ValueError, _sealed_set.SealedSetError):
-            pass
-        if status == "refused":
-            print(
-                "Read discovery: REFUSED. A probed read confirmed the written "
-                f"values but can't see the whole save, so no check was written: "
-                f"{reason} ({found})."
-            )
-        else:
-            print(
-                "Read discovery: UNAVAILABLE. No probed read confirmed the "
-                "written values, so no read was proposed; a person must name "
-                f"one ({found})."
-            )
+    report_draft_fallback(
+        Path(args.source),
+        out_dir,
+        probe_reads=bool(getattr(args, "probe_reads", False)),
+    )
     out, count = write_draft(Path(args.source), out_dir)
     print(
         "No read in the recording returns the saved record (no network capture, "

packages/flow/openadapt_flow/record_check_synthesis.py

@@ -1825,6 +1825,7 @@ class Proposal:
     #: on a copy of the database with ``fault_gate``, else mapped from the
     #: recorded-state faults above.
     coverage: Optional[Any] = None
+    refusal_reason: Optional[str] = None
 
     @property
     def misses(self) -> list[str]:
@@ -1850,9 +1851,9 @@ def misses(self) -> list[str]:
 
     @property
     def refused(self) -> bool:
-        """A probed read was found but can't see the whole save: no check."""
+        """A probed or observed read can't see the whole save: no check."""
         found = self.search.read_discovery or {}
-        return found.get("status") == "refused"
+        return found.get("status") == "refused" or self.refusal_reason is not None
 
     @property
     def working(self) -> bool:
@@ -1925,6 +1926,17 @@ def proof(self) -> dict[str, Any]:
             "notes": list(self.search.notes),
             "read_discovery": self.search.read_discovery,
             "identity": self.identity,
+            **(
+                {
+                    "refusal_reason": self.refusal_reason
+                    or (self.search.read_discovery or {}).get("reason")
+                }
+                if (
+                    self.refusal_reason is not None
+                    or (self.search.read_discovery or {}).get("reason") is not None
+                )
+                else {}
+            ),
             "coverage": (
                 None
                 if self.coverage is None
@@ -2032,7 +2044,7 @@ def synthesize(
         closed_world=search.closed_world,
     )
     proposal = Proposal(params, search, effects, section, faults)
-    _refuse_partial_probed_read(proposal)
+    _refuse_partial_read(proposal)
     cfg = gate_config(proposal)
     if fault_gate and db_path is not None:
         proposal.coverage = gate_on_database(proposal, Path(db_path), cfg)
@@ -2041,40 +2053,57 @@ def synthesize(
     return proposal
 
 
-def _refuse_partial_probed_read(proposal: Proposal) -> None:
-    """Refuse a check built on a probed read that can't see the whole save.
+def _refuse_partial_read(proposal: Proposal) -> None:
+    """Refuse a check built on a read that can't see the whole save.
 
-    A probe finds the collection the save wrote, not the store. When the
-    fault test shows a write outside the reads would pass
+    A probe or partial observed read finds the collection the save wrote, not
+    the store. When the fault test shows a write outside the reads would pass
     (``collateral_unaudited``), or the capture holds a write no read covers,
     a CONFIRMED from that check could hide a wrong write elsewhere. So no
     runnable check is emitted: the read stays proposed only, with the reason,
     and a person decides (``--complete-read``, ``--db`` or a fuller read).
     """
 
-    found = proposal.search.read_discovery
-    if not found or found.get("status") != "promoted":
-        return
     unaudited = [
         f"{r.fault}: {r.detail}"
         for r in proposal.faults
         if r.fault == "collateral_unaudited" and not r.ok
     ] + [f"not covered: {item}" for item in proposal.search.uncovered]
     if not unaudited:
         return
-    found.update(
-        status="refused",
-        needs_operator_confirmation=True,
-        reason=(
-            f"the probed read {found.get('read')} covers only the written "
-            "collection, and a write outside it would pass the check ("
+
+    found = proposal.search.read_discovery
+    if found and found.get("status") == "promoted":
+        found.update(
+            status="refused",
+            needs_operator_confirmation=True,
+            reason=(
+                f"the probed read {found.get('read')} covers only the written "
+                "collection, and a write outside it would pass the check ("
+                + "; ".join(unaudited)
+                + "). No runnable check was written; confirm the read covers "
+                "everything the save can change (--complete-read), give --db, or "
+                "name a fuller read"
+            ),
+        )
+        proposal.refusal_reason = found["reason"]
+        proposal.search.notes.append(f"read discovery: {found['reason']}")
+    elif proposal.search.surfaces:
+        read_names = ", ".join(s.name for s in proposal.search.surfaces)
+        proposal.refusal_reason = (
+            f"the recorded read {read_names} covers only the recorded "
+            "records, and a write outside it would pass the check ("
             + "; ".join(unaudited)
             + "). No runnable check was written; confirm the read covers "
             "everything the save can change (--complete-read), give --db, or "
             "name a fuller read"
-        ),
-    )
-    proposal.search.notes.append(f"read discovery: {found['reason']}")
+        )
+        proposal.search.notes.append(
+            f"recorded read refusal: {proposal.refusal_reason}"
+        )
+
+
+_refuse_partial_probed_read = _refuse_partial_read
 
 
 # ---------------------------------------------------------------------------

packages/flow/openadapt_flow/scaffold_verifier.py

@@ -780,10 +780,12 @@ def build_record_check(
         _refuse_plaintext_output(recording_dir, out_dir)
         destination = out_dir or recording_dir
         destination.mkdir(parents=True, exist_ok=True)
-        _sealed.write_text(
-            destination / READ_DISCOVERY_FILE,
-            json.dumps(proposal.search.read_discovery, indent=2, sort_keys=True) + "\n",
-        )
+        if proposal.search.read_discovery is not None:
+            _sealed.write_text(
+                destination / READ_DISCOVERY_FILE,
+                json.dumps(proposal.search.read_discovery, indent=2, sort_keys=True)
+                + "\n",
+            )
         # The proof, without the check: why nothing runnable was written.
         _sealed.write_text(
             destination / rcs.PROOF_NAME,
@@ -829,6 +831,51 @@ def build_record_check(
     )
 
 
+def report_draft_fallback(
+    source: Path, out_dir: Optional[Path], *, probe_reads: bool = False
+) -> None:
+    """Print why build_record_check fell back to the TODO draft."""
+    destination = out_dir or source
+    if probe_reads:
+        import json as _json
+
+        found = destination / READ_DISCOVERY_FILE
+        status, reason = "unavailable", ""
+        try:
+            data = _json.loads(_sealed.read_text(found))
+            status, reason = str(data.get("status")), str(data.get("reason") or "")
+        except (OSError, ValueError, _sealed.SealedSetError):
+            pass
+        if status == "refused":
+            print(
+                "Read discovery: REFUSED. A probed read confirmed the written "
+                f"values but can't see the whole save, so no check was written: "
+                f"{reason} ({found})."
+            )
+        else:
+            print(
+                "Read discovery: UNAVAILABLE. No probed read confirmed the "
+                "written values, so no read was proposed; a person must name "
+                f"one ({found})."
+            )
+    else:
+        from openadapt_flow.record_check_synthesis import PROOF_NAME
+
+        proof_path = destination / PROOF_NAME
+        try:
+            import json as _json
+
+            proof_data = _json.loads(_sealed.read_text(proof_path))
+            if proof_data.get("status") == "refused":
+                reason = str(proof_data.get("refusal_reason") or "")
+                print(
+                    "Record check: REFUSED. A recorded read can't see the "
+                    f"whole save, so no check was written: {reason} ({proof_path})."
+                )
+        except (OSError, ValueError, _sealed.SealedSetError):
+            pass
+
+
 # ---------------------------------------------------------------------------
 # flow explain: read-only plain-language run summary
 # ---------------------------------------------------------------------------

packages/flow/tests/test_record_check_synthesis.py

@@ -675,13 +675,31 @@ def test_cli_writes_a_working_check_and_its_proof(tmp_path, e2e_service, capsys)
     assert (tmp_path / "out" / PROPOSED_NAME).is_file()
 
 
-def test_cli_exits_nonzero_for_a_draft(tmp_path, e2e_service, capsys):
+def test_cli_refuses_an_observed_partial_read_with_an_open_collateral_miss(
+    tmp_path, e2e_service, capsys
+):
     rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url)
     code = main(["scaffold-verifier", str(rec), "--prove", "effect-e2e"])
     out = capsys.readouterr().out
+    assert code == 0, out
+    assert "Record check: REFUSED" in out
+    assert "collateral_unaudited" in out
+    assert not (rec / PROPOSED_NAME).exists()
+    assert (rec / "effect_contract.yaml").is_file()
+    proof = json.loads((rec / PROOF_NAME).read_text())
+    assert proof["status"] == "refused"
+    assert "collateral_unaudited" in proof["refusal_reason"]
+
+
+def test_cli_exits_nonzero_for_a_draft(tmp_path, e2e_service, capsys):
+    rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url)
+    code = main(
+        ["scaffold-verifier", str(rec), "--complete-read", "--prove", "effect-e2e"]
+    )
+    out = capsys.readouterr().out
     assert code == 1
     assert "DRAFT (not working)" in out
-    assert "MISS: collateral_unaudited" in out
+    assert "Live fault suite (effect-e2e): FAILED" in out
 
 
 def test_cli_falls_back_to_the_todo_draft_without_reads(tmp_path, capsys):