From 2ccea41b54e662b19cea71e766d5360ebf54a988 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 12 Sep 2026 10:18:10 -0500 Subject: [PATCH] fix(latex): tokenize verbatim verbatiminput like lstinputlisting verbatim.sty \verbatiminput{file} / \verbatiminput*{file} is one leftover command. Tokenize it before \verb so the span stays atomic. A leftover walker emits the command as Structure so following flush After. does not join. After. / Next. still split. lstinputlisting / inputminted unchanged. GitHub #398. --- docs/orgmode/reference/config.org | 3 +- docs/orgmode/reference/formats.org | 3 +- src/config.rs | 2 +- src/lib.rs | 2 +- src/parser/latex.rs | 138 ++++++++++++++++++++++++++++- src/sentence/unicode.rs | 94 ++++++++++++++++++-- tests/latex_verbatim_envs.rs | 104 ++++++++++++++++++++++ 7 files changed, 335 insertions(+), 11 deletions(-) diff --git a/docs/orgmode/reference/config.org b/docs/orgmode/reference/config.org index 08f7438b..c9115261 100644 --- a/docs/orgmode/reference/config.org +++ b/docs/orgmode/reference/config.org @@ -187,12 +187,13 @@ Adding =algorithm= stops reflow of that environment. String array of extra command names tokenized like the built-in verb command before sentence split. The next character after the name is the delimiter. -Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb=, and piton.sty =\piton=; =lstinline= still accepts optional =[...]= and ={...}=. +Built-in names are =verb=, =lstinline=, =lstinputlisting=, =verbatiminput=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb=, and piton.sty =\piton=; =lstinline= still accepts optional =[...]= and ={...}=. =mintinline= / =mint= take optional =[...]=, a ={lang}= argument, then a delimiter or ={...}= body. =\inputminted= takes the same optional =[...]= and ={lang}= then a required ={filename}=; following flush prose stays on its own line. =SaveVerb= takes optional =[...]=, a ={name}= argument, then the same delimiter body as =Verb=. =\piton|...|= is verb-like (interior =.!?%= stay one token); =\piton{...}= stays one token via the generic command argument. =\lstinputlisting= takes optional =[...]= then a required ={filename}=; following flush prose stays on its own line. +=\verbatiminput= / =\verbatiminput*= take a required ={filename}=; following flush prose stays on its own line. Adding another name keeps that command's delimited body atomic and treats an inner =%= as content, not a comment. * Code-block sections (=[code.]=) diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 0996ddd6..97dcf05e 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -135,10 +135,11 @@ These tokens within prose are not split across lines: - piton.sty =Piton= is a built-in code region (verbatim listing env) - Extra names from =[latex].verbatim_envs= are code regions too - Body follows the same comment-reflow and optional =--format-code= rules as other formats when language is known (=minted= / =minted*= language arg, =lstlisting= / =lstlisting*= =language== option) -- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / piton.sty =\piton= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. +- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / piton.sty =\piton= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= / verbatim.sty =\verbatiminput= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. =\piton|...|= is verb-like; =\piton{...}= stays one token via the generic command argument =\lstinputlisting= / =\lstinputlisting*= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\inputminted= / =\inputminted*= take optional =[...]=, ={lang}=, then a ={filename}=; a flush following sentence stays on its own line + =\verbatiminput= / =\verbatiminput*= take a ={filename}=; a flush following sentence stays on its own line *** Prose regions (reflowed) :PROPERTIES: diff --git a/src/config.rs b/src/config.rs index be8c6aeb..c30d948b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -49,7 +49,7 @@ pub struct FormatOverrides { pub structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. /// Meaningful under `[latex]` only. Missing or empty keeps - /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. + /// verb/lstinline/lstinputlisting/verbatiminput/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. pub verbatim_commands: Vec, } diff --git a/src/lib.rs b/src/lib.rs index d080441d..fb60aa1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ pub struct FormatConfig { /// to `NON_PROSE_ENVS`. Empty keeps the built-in list. pub latex_structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. - /// Empty keeps verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. + /// Empty keeps verb/lstinline/lstinputlisting/verbatiminput/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. pub latex_verbatim_commands: Vec, } diff --git a/src/parser/latex.rs b/src/parser/latex.rs index 0ffcb453..f427144c 100644 --- a/src/parser/latex.rs +++ b/src/parser/latex.rs @@ -502,7 +502,7 @@ impl LatexParser { /// Byte offset of the first `%` that is not escaped as `\%` and is not /// inside `\verb` / `\lstinline` / `\spverb` / `\mintinline` / `\mint` / /// `\Verb` / `\SaveVerb` / `\piton` / `\lstinputlisting` / - /// `\inputminted` / configured verbatim commands. + /// `\inputminted` / `\verbatiminput` / configured verbatim commands. fn unescaped_percent(&self, line: &str) -> Option { unescaped_percent_with(line, &self.extra_verbatim_commands) } @@ -875,7 +875,7 @@ fn find_tex_cs(line: &str, from: usize, cs: &str) -> Option { /// `\iffalse` in ordinary TeX, skipping `\verb` / `\lstinline` / /// `\spverb` / `\mintinline` / `\mint` / `\inputminted` / `\Verb` / -/// `\SaveVerb` / `\piton` / `\lstinputlisting` spans. +/// `\SaveVerb` / `\piton` / `\lstinputlisting` / `\verbatiminput` spans. fn find_iffalse_at(line: &str, from: usize, extra_cmds: &[String]) -> Option { let bytes = line.as_bytes(); let mut i = from; @@ -917,6 +917,14 @@ fn find_inputminted_at(line: &str, from: usize, extra_cmds: &[String]) -> Option find_leftover_cmd_at(line, from, extra_cmds, inputminted_cs_at) } +/// Leftover verbatim.sty `\verbatiminput` / `\verbatiminput*` +/// (required `{file}`; GitHub #398). Other verb spans are skipped +/// so `\verb|\verbatiminput{x}|` is not stolen. Walk stops at an +/// unescaped `%` so a comment is not a command tail. +fn find_verbatiminput_at(line: &str, from: usize, extra_cmds: &[String]) -> Option<(usize, usize)> { + find_leftover_cmd_at(line, from, extra_cmds, verbatiminput_cs_at) +} + fn find_leftover_cmd_at( line: &str, from: usize, @@ -967,6 +975,16 @@ fn inputminted_cs_at(line: &str, at: usize) -> bool { !after.starts_with(|c: char| c.is_ascii_alphabetic()) } +fn verbatiminput_cs_at(line: &str, at: usize) -> bool { + let Some(tail) = line.get(at..).and_then(|s| s.strip_prefix('\\')) else { + return false; + }; + let Some(after) = tail.strip_prefix("verbatiminput") else { + return false; + }; + !after.starts_with(|c: char| c.is_ascii_alphabetic()) +} + struct ParseState<'a> { input: &'a str, parser: &'a LatexParser, @@ -1652,6 +1670,9 @@ impl<'a> ParseState<'a> { if let Some((start, end)) = find_lstinputlisting_at(code, i, &self.parser.extra_verbatim_commands) .or_else(|| find_inputminted_at(code, i, &self.parser.extra_verbatim_commands)) + .or_else(|| { + find_verbatiminput_at(code, i, &self.parser.extra_verbatim_commands) + }) { self.append_item_or_prose(line.start + i, &code[i..start]); self.push_structure(ByteSpan::new( @@ -2580,6 +2601,119 @@ Some text. assert_eq!(format_text(&minted_out, &latex_cfg()).unwrap(), minted_out); } + /// Ticket fixture (GitHub #398): verbatim.sty `\verbatiminput{file}` + /// is one leftover command. Following flush prose does not join the + /// command line. `After.` / `Next.` still split. `lstinputlisting` / + /// `inputminted` unchanged. + #[test] + fn verbatiminput_does_not_join_following_prose() { + use crate::format_text; + + let input = concat!( + "Before. Next.\n", + "\\verbatiminput{foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\verbatiminput{foo.py}\n"), + "verbatiminput must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\verbatiminput{foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before verbatiminput must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after verbatiminput must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let star = concat!( + "Before. Next.\n", + "\\verbatiminput*{foo.py}\n", + "After. Next.\n", + ); + let star_out = format_text(star, &latex_cfg()).unwrap(); + assert!( + star_out.contains("\\verbatiminput*{foo.py}\n"), + "verbatiminput* must stay one atomic command, got:\n{star_out}" + ); + assert!( + !star_out.contains("\\verbatiminput*{foo.py} After."), + "starred verbatiminput must not join following prose, got:\n{star_out}" + ); + assert!( + star_out.contains("After.\nNext."), + "prose after starred verbatiminput must still split, got:\n{star_out}" + ); + + let lst = concat!( + "Before. Next.\n", + "\\lstinputlisting{foo.py}\n", + "After. Next.\n", + ); + let lst_out = format_text(lst, &latex_cfg()).unwrap(); + assert!( + lst_out.contains("\\lstinputlisting{foo.py}\n"), + "lstinputlisting must stay unchanged, got:\n{lst_out}" + ); + assert!( + !lst_out.contains("\\lstinputlisting{foo.py} After."), + "lstinputlisting must not join following prose, got:\n{lst_out}" + ); + assert!( + lst_out.contains("After.\nNext."), + "prose after lstinputlisting must still split, got:\n{lst_out}" + ); + + let minted = concat!( + "Before. Next.\n", + "\\inputminted{python}{foo.py}\n", + "After. Next.\n", + ); + let minted_out = format_text(minted, &latex_cfg()).unwrap(); + assert!( + minted_out.contains("\\inputminted{python}{foo.py}\n"), + "inputminted must stay unchanged, got:\n{minted_out}" + ); + assert!( + !minted_out.contains("\\inputminted{python}{foo.py} After."), + "inputminted must not join following prose, got:\n{minted_out}" + ); + assert!( + minted_out.contains("After.\nNext."), + "prose after inputminted must still split, got:\n{minted_out}" + ); + assert_eq!(format_text(&minted_out, &latex_cfg()).unwrap(), minted_out); + } + /// Ticket fixture (GitHub #245): minted `\mintinline{lang}|body|` is /// one token; following `Next sentence.` still splits. #[test] diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 66062acb..9f7409fd 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -200,9 +200,9 @@ pub fn protect_inline_tokens_with( /// `\mintinline{lang}|...|` / `\mint{lang}{...}` / /// `\inputminted{lang}{file}` / `\Verb|...|` / /// `\SaveVerb{name}|...|` / `\piton|...|` / -/// `\lstinputlisting[...]{file}` so inner `.!?%` cannot -/// split or comment. `\piton{...}` stays on the generic `\cmd{arg}` -/// path (piton.sty brace syntax is not verbatim; GitHub #305). +/// `\lstinputlisting[...]{file}` / `\verbatiminput{file}` so inner +/// `.!?%` cannot split or comment. `\piton{...}` stays on the generic +/// `\cmd{arg}` path (piton.sty brace syntax is not verbatim; GitHub #305). fn protect_latex_verbatim( text: &str, placeholders: &mut Vec, @@ -228,7 +228,7 @@ fn protect_latex_verbatim( /// Byte end of a `\verb` / `\lstinline` / `\spverb` / `\mintinline` / /// `\mint` / `\inputminted` / `\Verb` / `\SaveVerb` / `\piton` / -/// `\lstinputlisting` / +/// `\lstinputlisting` / `\verbatiminput` / /// extra-name span starting at `at`. /// /// `\verb` / `\verb*` / `\spverb` / `\spverb*` / `\Verb` / `\Verb*`: next @@ -247,7 +247,10 @@ fn protect_latex_verbatim( /// delimiter so `\piton{...}` stays on the generic `\cmd{arg}` path. /// `\lstinputlisting` / `\lstinputlisting*` (listings.sty leftover; /// GitHub #391) take optional `[...]` then a required `{filename}`; -/// no brace is not a span. Extra names are tokenized like `\verb`. +/// no brace is not a span. `\verbatiminput` / `\verbatiminput*` +/// (verbatim.sty leftover; GitHub #398) take a required `{filename}`; +/// no brace is not a span. Matched before `\verb` so the name is not +/// `\verb` plus leftover. Extra names are tokenized like `\verb`. /// With no closer, the span runs to end of line so an inner `%` is /// not a comment. pub(crate) fn latex_verb_span_end_with( @@ -312,6 +315,12 @@ pub(crate) fn latex_verb_span_end_with( return None; } (after_bs + "SaveVerb".len(), VerbKind::SaveVerb) + } else if let Some(stripped) = tail.strip_prefix("verbatiminput") { + // Before `verb` so `\verbatiminput` is not `\verb` + leftover. + if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { + return None; + } + (after_bs + "verbatiminput".len(), VerbKind::Verbatiminput) } else if let Some(stripped) = tail.strip_prefix("verb") { if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { return None; @@ -358,6 +367,17 @@ pub(crate) fn latex_verb_span_end_with( return Some(find_unescaped_brace_close(text, i).unwrap_or_else(|| line_end(text, i))); } + // verbatim.sty `\verbatiminput{file}` / `\verbatiminput*{file}` + // is brace-only (GitHub #398). No optional `[...]`. + if kind == VerbKind::Verbatiminput { + i = skip_ascii_ws(text, i); + if !text.get(i..).is_some_and(|s| s.starts_with('{')) { + return None; + } + i += 1; + return Some(find_unescaped_brace_close(text, i).unwrap_or_else(|| line_end(text, i))); + } + if kind == VerbKind::Mint { if !text.get(i..).is_some_and(|s| s.starts_with('{')) { return None; @@ -414,6 +434,8 @@ enum VerbKind { Lstinline, /// `\lstinputlisting`: optional `[...]` then required `{filename}`. Lstinputlisting, + /// `\verbatiminput`: required `{filename}` (verbatim.sty leftover). + Verbatiminput, /// `\mintinline` / `\mint` / `\inputminted`: optional `[...]`, /// `{lang}`, then body. Mint, @@ -439,6 +461,7 @@ fn match_extra_verb_command<'a>(tail: &'a str, extras: &'a [String]) -> Option<& || name == "verb" || name == "lstinline" || name == "lstinputlisting" + || name == "verbatiminput" || name == "spverb" || name == "mintinline" || name == "inputminted" @@ -2336,6 +2359,67 @@ mod tests { ); } + /// Ticket fixture (GitHub #398): verbatim.sty `\verbatiminput{file}` + /// is one token like `\lstinputlisting`; following `After.` still + /// splits. Matched before `\verb` so the name is not a delim span. + #[test] + fn latex_verbatiminput_stays_atomic() { + let text = r"See \verbatiminput{foo.py} here. After."; + let (_, placeholders) = protect_inline_tokens(text); + assert!( + placeholders.iter().any(|p| p == r"\verbatiminput{foo.py}"), + "verbatiminput span must be protected, got {placeholders:?}" + ); + assert_eq!( + latex_verb_span_end_with(r"\verbatiminput{foo.py}", 0, &[]), + Some(r"\verbatiminput{foo.py}".len()) + ); + assert_eq!( + split(text), + vec![ + r"See \verbatiminput{foo.py} here.".to_string(), + "After.".to_string() + ] + ); + assert_eq!( + latex_verb_span_end_with(r"\verbatiminput*{foo.py}", 0, &[]), + Some(r"\verbatiminput*{foo.py}".len()) + ); + let star = r"See \verbatiminput*{foo.py} here. After."; + assert_eq!( + split(star), + vec![ + r"See \verbatiminput*{foo.py} here.".to_string(), + "After.".to_string() + ] + ); + assert_eq!( + latex_verb_span_end_with(r"\verbatiminput foo.py", 0, &[]), + None, + "verbatiminput without a brace file arg is not a verb span" + ); + assert_eq!( + latex_verb_span_end_with(r"\verb|a.b!|", 0, &[]), + Some(r"\verb|a.b!|".len()), + "verbatiminput must not steal verb" + ); + assert_eq!( + latex_verb_span_end_with(r"\lstinputlisting{foo.py}", 0, &[]), + Some(r"\lstinputlisting{foo.py}".len()), + "verbatiminput must not steal lstinputlisting" + ); + assert_eq!( + latex_verb_span_end_with(r"\inputminted{python}{foo.py}", 0, &[]), + Some(r"\inputminted{python}{foo.py}".len()), + "verbatiminput must not steal inputminted" + ); + assert_eq!( + latex_verb_span_end_with(r"\VerbatimInput{foo.py}", 0, &[]), + None, + "verbatiminput must not steal fancyvrb VerbatimInput" + ); + } + /// Ticket fixture (GitHub #275): fancyvrb `\SaveVerb{name}|body|` /// is one token; following `After.` still splits. #[test] diff --git a/tests/latex_verbatim_envs.rs b/tests/latex_verbatim_envs.rs index b7e758ec..6290eecf 100644 --- a/tests/latex_verbatim_envs.rs +++ b/tests/latex_verbatim_envs.rs @@ -70,6 +70,8 @@ //! VerbatimEnvironment plus VerbatimOut. //! GitHub #391: listings.sty \\lstinputlisting is one atomic command; //! following flush prose does not join the command line. +//! GitHub #398: verbatim.sty \\verbatiminput / \\verbatiminput* is one +//! leftover command; following flush prose does not join the command line. use snapper_fmt::format::Format; use snapper_fmt::parser::latex::LatexParser; @@ -3116,6 +3118,108 @@ fn lstinputlisting_fixture_does_not_join_following_prose() { ); } +/// Ticket fixture (GitHub #398): verbatim.sty `\verbatiminput{file}` +/// stays one atomic command. Following flush `After.` does not join +/// the command line. `After.` / `Next.` still split. lstinputlisting / +/// inputminted unchanged. +#[test] +fn verbatiminput_fixture_does_not_join_following_prose() { + let input = concat!( + "Before. Next.\n", + "\\verbatiminput{foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\verbatiminput{foo.py}\n"), + "verbatiminput must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\verbatiminput{foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before verbatiminput must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after verbatiminput must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let star = concat!( + "Before. Next.\n", + "\\verbatiminput*{foo.py}\n", + "After. Next.\n", + ); + let star_out = format_text(star, &latex_cfg()).unwrap(); + assert!( + star_out.contains("\\verbatiminput*{foo.py}\n"), + "verbatiminput* must stay one atomic command, got:\n{star_out}" + ); + assert!( + !star_out.contains("\\verbatiminput*{foo.py} After."), + "starred verbatiminput must not join following prose, got:\n{star_out}" + ); + assert!( + star_out.contains("After.\nNext."), + "prose after starred verbatiminput must still split, got:\n{star_out}" + ); + + let lst = concat!( + "Before. Next.\n", + "\\lstinputlisting{foo.py}\n", + "After. Next.\n", + ); + let lst_out = format_text(lst, &latex_cfg()).unwrap(); + assert!( + lst_out.contains("\\lstinputlisting{foo.py}\n"), + "lstinputlisting must stay unchanged, got:\n{lst_out}" + ); + assert!( + !lst_out.contains("\\lstinputlisting{foo.py} After."), + "lstinputlisting must not join following prose, got:\n{lst_out}" + ); + + let minted = concat!( + "Before. Next.\n", + "\\inputminted{python}{foo.py}\n", + "After. Next.\n", + ); + let minted_out = format_text(minted, &latex_cfg()).unwrap(); + assert!( + minted_out.contains("\\inputminted{python}{foo.py}\n"), + "inputminted must stay unchanged, got:\n{minted_out}" + ); + assert!( + !minted_out.contains("\\inputminted{python}{foo.py} After."), + "inputminted must not join following prose, got:\n{minted_out}" + ); +} + /// Ticket fixture (GitHub #305): piton.sty `{Piton}` body stays Code /// on one source line; `\piton|done. Next|` is one token; following /// prose still splits. minted / lstlisting / `\verb` unchanged.