From f9190abcec746acfec726be626f32483556302d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 16:11:35 +0000 Subject: [PATCH] fix(assail): strip proof comments in one pass, blanking in place strip_proof_comments stripped block comments before line comments, so a block opener inside a line comment swallowed the code after it: `code // see /* note`, `hidden();`, `*/ tail` reduced to `code `. The Rust, Idris, Lean and Agda detectors pass both markers, so such a comment could hide a later sorry, believe_me or postulate. It also deleted comment text, which moved columns and was not idempotent (`a//*x*/*b*/c` -> `a/*b*/c` -> `ac`). It is now one left-to-right scan. Whichever comment opens first wins, block comments nest (as in every caller's language), and comment bytes become spaces with newlines kept. The output has the input's byte length and is a fixed point. Tests (module strip_proof_comments_tests): - regressions for the three counterexamples, nesting, unterminated blocks, multibyte text and empty markers; - exhaustive bounded checks of length, per-byte preservation and idempotence for every marker set the analyzers pass; - a differential test against a transcription of the Idris2 model. Six planted faults are caught; one mutant is equivalent. The compiled Idris2 model and the Rust function now agree on all 349,525 strings of length <= 9 over `/`, newline, `x`, space, so Stripping.idr's header and PROOF-NEEDS.adoc record agreement instead of the earlier mismatch. Self-scan findings are unchanged (36 before and after, on main). The Idris2 model checked is the one on main. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jma5ixtpeZyJRmfoSsC98u --- PROOF-NEEDS.adoc | 85 +++++++++----- src/abi/Stripping.idr | 8 ++ src/assail/analyzer.rs | 256 +++++++++++++++++++++++++++++++++++------ 3 files changed, 288 insertions(+), 61 deletions(-) diff --git a/PROOF-NEEDS.adoc b/PROOF-NEEDS.adoc index f1b1abd..2b8097c 100644 --- a/PROOF-NEEDS.adoc +++ b/PROOF-NEEDS.adoc @@ -107,12 +107,43 @@ definitions. A mutant that blanks with `+/+` instead of a space is rejected. Nine closed sanity theorems, including two comments on different lines and `+///+`. -The model does *not* match `strip_proof_comments`, so it is evidence about -the model only. It replaces comment characters with spaces and keeps every -newline. The Rust function deletes comment text and drops a trailing -newline: on `//x` the model gives three spaces and Rust gives the empty -string. The model also covers line comments only; block comments and string -literals are not modelled. +The model now agrees with `strip_proof_comments(s, "//", None)`, the slice +it models. The compiled model and the Rust function give the same output on +every string of length at most 9 over `/`, newline, `x` and space (349,525 +inputs, local run). In CI, +`the_line_comment_slice_matches_the_idris_model` re-checks a Rust +transcription of the model's equations up to length 8. Correspondence on +that slice is therefore `TESTED`, with those bounds, not `ASSUMED`. Block +comments, nesting and string literals are not modelled. Before the +single-pass rewrite they disagreed: on `//x` the model gave three spaces and +Rust gave the empty string. + +|`strip_proof_comments` blanks comments in place. The output has the input's +byte length; every byte outside a comment, and every newline, is unchanged; +other comment bytes become spaces. Stripping is idempotent. Whichever +comment opens first wins, and block comments nest. +|`src/assail/analyzer.rs`, module `strip_proof_comments_tests` +|`TESTED` +|Exhaustive over bounded inputs, for the marker sets the analyzers pass: +`//` with `/* */`, `--` with `/- -/`, `--` with `{- -}`, `(* *)` alone, and +`//` alone. Each runs every string over a 4-letter alphabet up to length 8 +(87,381 inputs) or a 5-letter alphabet up to length 7 (97,656 inputs). Each +alphabet spells the markers, plus a newline and an ordinary character. +The three counterexamples in the open obligations are regression tests, as +are nesting, unterminated blocks and multibyte text. + +Planted faults, all caught: + +* the previous two-phase function; +* deleting comment text instead of blanking it; +* blanking that drops a newline; +* one space per character instead of per byte; +* no nesting; +* an unterminated block that keeps code. + +One mutant, a line comment that also consumes its newline, is equivalent: +the newline is kept and the scan resumes after it, so no test can tell +them apart. |Attestation chain: integrity, authenticity, non-repudiation |`src/abi/AttestationUnforgeability.idr` @@ -223,30 +254,32 @@ generation, and keep the `keep` newest older ones |*OPEN*. The model check above covers the protocol, not the Rust code; these contracts tie the implementation to P1 and P2. -|`strip_proof_comments` is idempotent and preserves positions outside -comments and strings -|Fix the implementation first; then Creusot/Pearlite and Verus -|Neither property holds of the current Rust function, so neither can be -proved until it changes. All three cases below were run against the -function as it is on `main`: +|Prove the `strip_proof_comments` properties in the ledger for all inputs +|Creusot/Pearlite and Verus +|The single-pass rewrite makes both properties true of the implementation; +the ledger row has them `TESTED` within bounds. Pearlite contracts can +state them directly: equal length, the per-byte relation, and +idempotence. Three counterexamples held of the previous two-phase function +and are now regression tests: -* Not idempotent: `a//*x*/*b*/c` gives `a/*b*/c`, and a second pass gives +* Not idempotent: `a//*x*/*b*/c` gave `a/*b*/c`, and a second pass gave `ac`. -* Positions move: `a/*x*/b` gives `ab`, moving `b` from offset 6 to 1. -* Ordering bug: block comments are stripped before line comments, so a - block opener inside a line comment swallows the code after it. - `code // see /* note`, then `hidden();`, then `*/ tail`, gives `code `. - The callers that pass both a line and a block marker are affected: +* Positions moved: `a/*x*/b` gave `ab`, moving `b` from offset 6 to 1. +* Ordering bug: block comments were stripped before line comments, so a + block opener inside a line comment swallowed the code after it. + `code // see /* note`, then `hidden();`, then `*/ tail`, gave `code `. + The callers that pass both a line and a block marker were affected: `analyze_rust`, `is_rust_ffi_safe_wrapper`, `analyze_idris`, `analyze_lean` and `analyze_agda`. In the Idris, Lean and Agda ProofDrift - detectors such a comment can hide a later `sorry`, `believe_me` or - postulate. `analyze_isabelle` and `analyze_coq` pass no line marker and - are not affected. - -The Idris2 model above can guide the contracts once the function is -fixed; it does not discharge them. The model's own remaining slices -(block comments, strings, composition, position preservation; issue #114) -are optional once the implementation proof exists. + detectors such a comment could hide a later `sorry`, `believe_me` or + postulate. + +The function does not know about string literals. The Rust and Zig callers +blank strings first; the Lean, Agda, Idris, Coq and Isabelle callers do +not, so a comment marker inside a string literal there still starts a +comment. The Idris2 model's remaining slices (block comments, strings, +composition; issue #114) are optional once the implementation proof +exists. |Patch Bridge reachability is sound: no reachable dependency is classified as phantom diff --git a/src/abi/Stripping.idr b/src/abi/Stripping.idr index 32bf438..1fb08b0 100644 --- a/src/abi/Stripping.idr +++ b/src/abi/Stripping.idr @@ -15,6 +15,14 @@ ||| The model strips ALL `//` comments: `stripLineCommentBody` hands back ||| to `stripLineComments` after each preserved newline (issue #113). ||| +||| Correspondence: the model is `strip_proof_comments(s, "//", None)`, the +||| line-comment slice only. Block comments, their nesting and string +||| literals are not modelled, so the theorem is about that slice. On it the +||| compiled model and the Rust function agree on every string of length at +||| most 9 over `/`, newline, `x` and space, and a Rust test +||| (`the_line_comment_slice_matches_the_idris_model`) re-checks these +||| equations up to length 8. See PROOF-NEEDS.adoc. +||| ||| Re-proved 2026-11-09. The previous proof (PR #111, issue #113) never ||| type-checked: its `//` case relied on reductions that do not hold for a ||| variable character, and it used `decEq` without importing it. The proof diff --git a/src/assail/analyzer.rs b/src/assail/analyzer.rs index d375d0c..7a91b3a 100644 --- a/src/assail/analyzer.rs +++ b/src/assail/analyzer.rs @@ -6187,51 +6187,63 @@ fn push_line_anchored_proof_drift( }); } } - -/// Remove proof-language comments while keeping every newline, so a line -/// number taken from the result is a line number in `content`. +/// Blank out proof-language comments in one left-to-right pass. +/// +/// Every byte inside a comment becomes a space, except `'\n'`, which is +/// kept. So the result has exactly the length of `content`, every byte +/// outside a comment is unchanged, and a line or column taken from the +/// result is a line or column in `content`. +/// +/// Whichever comment opens first wins: a block opener inside a line comment +/// is comment text, and so is a line marker inside a block comment. Block +/// comments nest, as they do in every caller's language (Rust, Lean, Agda, +/// Idris, Coq/Rocq and Isabelle). An unterminated block comment runs to the +/// end of `content`. An empty `line_marker`, or an empty block marker, +/// switches that kind of comment off. Markers must not contain whitespace. +/// +/// The result is a fixed point: stripping it again changes nothing. fn strip_proof_comments(content: &str, line_marker: &str, block: Option<(&str, &str)>) -> String { - // First strip block comments greedily, left-to-right. - let mut without_blocks = String::with_capacity(content.len()); - if let Some((open, close)) = block { - let mut rest = content; - while let Some(start) = rest.find(open) { - without_blocks.push_str(&rest[..start]); - let after_open = &rest[start + open.len()..]; - if let Some(end) = after_open.find(close) { - push_newlines_of(&mut without_blocks, &after_open[..end]); - rest = &after_open[end + close.len()..]; + fn blank(out: &mut String, span: &str) { + for c in span.chars() { + if c == '\n' { + out.push('\n'); } else { - // Unterminated block comment — drop the rest, keep its lines. - push_newlines_of(&mut without_blocks, after_open); - rest = ""; - break; + // One space per byte keeps byte offsets, not just columns. + out.extend(std::iter::repeat_n(' ', c.len_utf8())); } } - without_blocks.push_str(rest); - } else { - without_blocks.push_str(content); } - // Then strip line comments. Split on '\n' (not `lines()`) so a trailing - // newline survives and the newline count is exactly preserved. - if line_marker.is_empty() { - return without_blocks; - } - let mut out = String::with_capacity(without_blocks.len()); - for (i, line) in without_blocks.split('\n').enumerate() { - let kept = match line.find(line_marker) { - Some(idx) => &line[..idx], - None => line, + let block = block.filter(|(open, close)| !open.is_empty() && !close.is_empty()); + let mut out = String::with_capacity(content.len()); + let mut depth = 0usize; + let mut i = 0; + while let Some(rest) = content.get(i..).filter(|r| !r.is_empty()) { + let step = match block { + Some((_, close)) if depth > 0 && rest.starts_with(close) => { + depth -= 1; + close.len() + } + Some((open, _)) if rest.starts_with(open) => { + depth += 1; + open.len() + } + _ if depth > 0 => rest.chars().next().map_or(1, char::len_utf8), + _ if !line_marker.is_empty() && rest.starts_with(line_marker) => { + rest.find('\n').unwrap_or(rest.len()) + } + _ => { + let len = rest.chars().next().map_or(1, char::len_utf8); + out.push_str(&rest[..len]); + i += len; + continue; + } }; - if i > 0 { - out.push('\n'); - } - out.push_str(kept); + blank(&mut out, &rest[..step]); + i += step; } out } - /// Strip C-family line comments (`//`, `///`, `//!`) from each line, leaving /// the rest of the source intact. String literals are detected so a `//` /// inside `"http://example.com"` is NOT treated as a comment. @@ -8885,3 +8897,177 @@ mod proof_line_anchor_tests { assert_eq!(occurrence_lines(&code, "sorry"), vec![6]); } } + +#[cfg(test)] +mod strip_proof_comments_tests { + use super::strip_proof_comments; + + type Markers = (&'static str, Option<(&'static str, &'static str)>); + + /// The marker pairs the analyzers pass, each with an alphabet that can + /// spell every marker, plus a newline and an ordinary character. + const CONFIGS: [(Markers, &[char]); 5] = [ + (("//", Some(("/*", "*/"))), &['/', '*', '\n', 'x']), + (("--", Some(("/-", "-/"))), &['/', '-', '\n', 'x']), + (("--", Some(("{-", "-}"))), &['{', '-', '}', '\n', 'x']), + (("", Some(("(*", "*)"))), &['(', '*', ')', '\n', 'x']), + (("//", None), &['/', '\n', 'x', ' ']), + ]; + + /// Every string over `alphabet` of length at most `max_len`. + fn all_strings(alphabet: &[char], max_len: usize) -> Vec { + let mut all = vec![String::new()]; + let mut layer = vec![String::new()]; + for _ in 0..max_len { + layer = layer + .iter() + .flat_map(|s| { + alphabet.iter().map(move |c| { + let mut t = s.clone(); + t.push(*c); + t + }) + }) + .collect(); + all.extend(layer.iter().cloned()); + } + all + } + + fn strip(s: &str, (line, block): Markers) -> String { + strip_proof_comments(s, line, block) + } + + #[test] + fn a_block_opener_inside_a_line_comment_hides_nothing() { + let src = "code // see /* note\nhidden();\n*/ tail"; + let out = strip(src, ("//", Some(("/*", "*/")))); + let blanked = " ".repeat("// see /* note".len()); + assert_eq!(out, format!("code {blanked}\nhidden();\n*/ tail")); + } + + #[test] + fn a_line_marker_inside_a_block_comment_hides_nothing() { + let src = "/- -- -/ sorry"; + assert_eq!(strip(src, ("--", Some(("/-", "-/")))), " sorry"); + } + + #[test] + fn the_former_idempotence_counterexample_is_now_a_fixed_point() { + let markers = ("//", Some(("/*", "*/"))); + let once = strip("a//*x*/*b*/c", markers); + assert_eq!(once, "a "); + assert_eq!(strip(&once, markers), once); + } + + #[test] + fn text_after_a_block_comment_keeps_its_offset() { + let out = strip("a/*x*/b", ("//", Some(("/*", "*/")))); + assert_eq!(out, "a b"); + assert_eq!(out.find('b'), Some(6)); + } + + #[test] + fn block_comments_nest() { + let src = "/- a /- b -/ sorry -/ x"; + assert_eq!( + strip(src, ("--", Some(("/-", "-/")))), + " x" + ); + let src = "(* (* *) sorry *) oops"; + assert_eq!( + strip(src, ("", Some(("(*", "*)")))), + " oops" + ); + } + + #[test] + fn an_unterminated_block_comment_runs_to_the_end_keeping_newlines() { + let out = strip("a {- b\nc\n", ("--", Some(("{-", "-}")))); + assert_eq!(out, "a \n \n"); + } + + #[test] + fn multibyte_text_keeps_its_byte_offsets() { + let src = "/* é */x // ü\ny"; + let out = strip(src, ("//", Some(("/*", "*/")))); + assert_eq!(out.len(), src.len()); + assert_eq!(out.find('x'), src.find('x')); + assert_eq!(out.find('y'), src.find('y')); + assert!(!out.contains('é') && !out.contains('ü')); + } + + #[test] + fn empty_markers_switch_that_comment_kind_off() { + assert_eq!(strip("a // b", ("", None)), "a // b"); + assert_eq!(strip("a /* b */", ("", Some(("", "*/")))), "a /* b */"); + } + + /// Exhaustive over every string up to the bound, for every marker set the + /// analyzers use: the result has the input's length, keeps every byte + /// outside a comment and every newline, blanks the rest with spaces, and + /// is a fixed point. + #[test] + fn stripping_preserves_positions_and_is_idempotent_exhaustively() { + for (markers, alphabet) in CONFIGS { + let max_len = if alphabet.len() > 4 { 7 } else { 8 }; + for src in all_strings(alphabet, max_len) { + let out = strip(&src, markers); + assert_eq!(out.len(), src.len(), "{markers:?} {src:?}"); + for (a, b) in src.bytes().zip(out.bytes()) { + assert!( + a == b || (b == b' ' && a != b'\n'), + "{markers:?} {src:?} -> {out:?}" + ); + } + assert_eq!(strip(&out, markers), out, "{markers:?} {src:?}"); + } + } + } + + /// The line-comment model in `src/abi/Stripping.idr`, + /// transcribed equation by equation. Idris2 proves the model idempotent. + fn idris_model(s: &[char]) -> String { + fn code(s: &[char], out: &mut String) { + match s { + [] => {} + ['/', '/', rest @ ..] => { + out.push_str(" "); + body(rest, out); + } + [c, rest @ ..] => { + out.push(*c); + code(rest, out); + } + } + } + fn body(s: &[char], out: &mut String) { + match s { + [] => {} + ['\n', rest @ ..] => { + out.push('\n'); + code(rest, out); + } + [_, rest @ ..] => { + out.push(' '); + body(rest, out); + } + } + } + let mut out = String::new(); + code(s, &mut out); + out + } + + /// With only a `//` marker, the Rust function and the Idris2 model agree + /// on every input up to the bound, so the model's idempotence theorem is + /// about this function's behaviour on that slice, not about a different + /// function. + #[test] + fn the_line_comment_slice_matches_the_idris_model() { + for src in all_strings(&['/', '\n', 'x', ' '], 8) { + let chars: Vec = src.chars().collect(); + assert_eq!(strip(&src, ("//", None)), idris_model(&chars), "{src:?}"); + } + } +}