From 030dca9c036ae402734eae4a478cfbcbddf3d97e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 12 Sep 2026 11:34:44 -0500 Subject: [PATCH] fix(latex): tokenize piton PitonInputFile like lstinputlisting piton.sty \PitonInputFile (NewDocumentCommand d<> O{} m) is one leftover command. Tokenize it like lstinputlisting so the span stays atomic including optional <...> / [...]. A leftover walker emits the command as Structure so following flush After. does not join. After. / Next. still split. Piton env, \piton, verbatiminput, VerbatimInput unchanged. GitHub #406. --- docs/orgmode/reference/config.org | 3 +- docs/orgmode/reference/formats.org | 3 +- src/config.rs | 2 +- src/lib.rs | 2 +- src/parser/latex.rs | 208 +++++++++++++++- src/sentence/unicode.rs | 152 +++++++++++- tests/latex_verbatim_envs.rs | 371 +++++++++++++++++++++-------- 7 files changed, 628 insertions(+), 113 deletions(-) diff --git a/docs/orgmode/reference/config.org b/docs/orgmode/reference/config.org index f64f86f4..ba6f578b 100644 --- a/docs/orgmode/reference/config.org +++ b/docs/orgmode/reference/config.org @@ -187,7 +187,7 @@ Adding =algorithm= stops reflow of that environment. String array of extra command names tokenized like the built-in verb command before sentence split. The next character after the name is the delimiter. -Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb= / =VerbatimInput= / =BVerbatimInput= / =LVerbatimInput=, piton.sty =\piton=, and tools/verbatim.sty =\verbatiminput=; =lstinline= still accepts optional =[...]= and ={...}=. +Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb= / =VerbatimInput= / =BVerbatimInput= / =LVerbatimInput=, piton.sty =\piton= / =\PitonInputFile=, and tools/verbatim.sty =\verbatiminput=; =lstinline= still accepts optional =[...]= and ={...}=. =mintinline= / =mint= take optional =[...]=, a ={lang}= argument, then a delimiter or ={...}= body. =\inputminted= takes the same optional =[...]= and ={lang}= then a required ={filename}=; following flush prose stays on its own line. =SaveVerb= takes optional =[...]=, a ={name}= argument, then the same delimiter body as =Verb=. @@ -195,6 +195,7 @@ Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline =\lstinputlisting= takes optional =[...]= then a required ={filename}=; following flush prose stays on its own line. =\VerbatimInput= / =\BVerbatimInput= / =\LVerbatimInput= take optional =[...]= then a required ={filename}=; following flush prose stays on its own line. =\verbatiminput= / =\verbatiminput*= take a required ={filename}=; following flush prose stays on its own line. +=\PitonInputFile= takes optional =<...>= / =[...]= then a required ={filename}=; following flush prose stays on its own line. Adding another name keeps that command's delimited body atomic and treats an inner =%= as content, not a comment. * Code-block sections (=[code.]=) diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 8a30dfb2..bfc6dd55 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -135,12 +135,13 @@ These tokens within prose are not split across lines: - piton.sty =Piton= is a built-in code region (verbatim listing env) - Extra names from =[latex].verbatim_envs= are code regions too - Body follows the same comment-reflow and optional =--format-code= rules as other formats when language is known (=minted= / =minted*= language arg, =lstlisting= / =lstlisting*= =language== option) -- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / =\VerbatimInput= / =\BVerbatimInput= / =\LVerbatimInput= / piton.sty =\piton= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= / tools/verbatim.sty =\verbatiminput= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. +- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / =\VerbatimInput= / =\BVerbatimInput= / =\LVerbatimInput= / piton.sty =\piton= / =\PitonInputFile= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= / tools/verbatim.sty =\verbatiminput= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. =\piton|...|= is verb-like; =\piton{...}= stays one token via the generic command argument =\lstinputlisting= / =\lstinputlisting*= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\inputminted= / =\inputminted*= take optional =[...]=, ={lang}=, then a ={filename}=; a flush following sentence stays on its own line =\VerbatimInput= / =\BVerbatimInput= / =\LVerbatimInput= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\verbatiminput= / =\verbatiminput*= take a ={filename}=; a flush following sentence stays on its own line + =\PitonInputFile= takes optional =<...>=, optional =[...]=, then a ={filename}=; a flush following sentence stays on its own line *** Prose regions (reflowed) :PROPERTIES: diff --git a/src/config.rs b/src/config.rs index ab385a2b..76a02484 100644 --- a/src/config.rs +++ b/src/config.rs @@ -49,7 +49,7 @@ pub struct FormatOverrides { pub structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. /// Meaningful under `[latex]` only. Missing or empty keeps - /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/VerbatimInput/piton/verbatiminput. + /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/VerbatimInput/piton/verbatiminput/PitonInputFile. pub verbatim_commands: Vec, } diff --git a/src/lib.rs b/src/lib.rs index dac38103..ddd23ce6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ pub struct FormatConfig { /// to `NON_PROSE_ENVS`. Empty keeps the built-in list. pub latex_structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. - /// Empty keeps verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/VerbatimInput/piton/verbatiminput. + /// Empty keeps verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/VerbatimInput/piton/verbatiminput/PitonInputFile. pub latex_verbatim_commands: Vec, } diff --git a/src/parser/latex.rs b/src/parser/latex.rs index 945f56d0..03f205aa 100644 --- a/src/parser/latex.rs +++ b/src/parser/latex.rs @@ -503,7 +503,7 @@ impl LatexParser { /// inside `\verb` / `\lstinline` / `\spverb` / `\mintinline` / `\mint` / /// `\Verb` / `\SaveVerb` / `\piton` / `\lstinputlisting` / /// `\inputminted` / `\verbatiminput` / `\VerbatimInput` / - /// configured verbatim commands. + /// `\PitonInputFile` / configured verbatim commands. fn unescaped_percent(&self, line: &str) -> Option { unescaped_percent_with(line, &self.extra_verbatim_commands) } @@ -877,7 +877,7 @@ fn find_tex_cs(line: &str, from: usize, cs: &str) -> Option { /// `\iffalse` in ordinary TeX, skipping `\verb` / `\lstinline` / /// `\spverb` / `\mintinline` / `\mint` / `\inputminted` / `\Verb` / /// `\SaveVerb` / `\piton` / `\lstinputlisting` / `\verbatiminput` / -/// `\VerbatimInput` spans. +/// `\VerbatimInput` / `\PitonInputFile` spans. fn find_iffalse_at(line: &str, from: usize, extra_cmds: &[String]) -> Option { let bytes = line.as_bytes(); let mut i = from; @@ -941,6 +941,19 @@ fn find_fancyvrb_input_at( find_leftover_cmd_at(line, from, extra_cmds, fancyvrb_input_cs_at) } +/// Leftover piton.sty `\PitonInputFile` (optional `<...>`, optional +/// `[...]`, required `{file}`; `d < > O { } m`; GitHub #406). Other +/// verb spans are skipped so `\verb|\PitonInputFile{x}|` is not +/// stolen. Walk stops at an unescaped `%` so a comment is not a +/// command tail. +fn find_pitoninputfile_at( + line: &str, + from: usize, + extra_cmds: &[String], +) -> Option<(usize, usize)> { + find_leftover_cmd_at(line, from, extra_cmds, pitoninputfile_cs_at) +} + fn find_leftover_cmd_at( line: &str, from: usize, @@ -1013,6 +1026,16 @@ fn fancyvrb_input_cs_at(line: &str, at: usize) -> bool { false } +fn pitoninputfile_cs_at(line: &str, at: usize) -> bool { + let Some(tail) = line.get(at..).and_then(|s| s.strip_prefix('\\')) else { + return false; + }; + let Some(after) = tail.strip_prefix("PitonInputFile") else { + return false; + }; + !after.starts_with(|c: char| c.is_ascii_alphabetic()) +} + struct ParseState<'a> { input: &'a str, parser: &'a LatexParser, @@ -1704,6 +1727,9 @@ impl<'a> ParseState<'a> { .or_else(|| { find_fancyvrb_input_at(code, i, &self.parser.extra_verbatim_commands) }) + .or_else(|| { + find_pitoninputfile_at(code, i, &self.parser.extra_verbatim_commands) + }) { self.append_item_or_prose(line.start + i, &code[i..start]); self.push_structure(ByteSpan::new( @@ -2875,6 +2901,184 @@ Some text. assert_eq!(format_text(&minted_out, &latex_cfg()).unwrap(), minted_out); } + /// Ticket fixture (GitHub #406): piton.sty `\PitonInputFile{file}` + /// is one leftover command. Following flush prose does not join the + /// command line. `After.` / `Next.` still split. Optional `[...]` + /// and `d < >` stay atomic. Piton env, `\piton`, `\verbatiminput`, + /// and fancyvrb `\VerbatimInput` unchanged. + #[test] + fn pitoninputfile_does_not_join_following_prose() { + use crate::format_text; + + let input = concat!( + "Before. Next.\n", + "\\PitonInputFile{foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\PitonInputFile{foo.py}") + )), + "PitonInputFile must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\PitonInputFile{foo.py}") + )), + "PitonInputFile must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\PitonInputFile{foo.py}\n"), + "PitonInputFile must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\PitonInputFile{foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before PitonInputFile must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after PitonInputFile must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let opts = concat!( + "Before. Next.\n", + "\\PitonInputFile[language=python]{foo.py}\n", + "After. Next.\n", + ); + let opts_out = format_text(opts, &latex_cfg()).unwrap(); + assert!( + opts_out.contains("\\PitonInputFile[language=python]{foo.py}\n"), + "PitonInputFile optional args must stay atomic, got:\n{opts_out}" + ); + assert!( + !opts_out.contains("\\PitonInputFile[language=python]{foo.py} After."), + "optional-arg PitonInputFile must not join following prose, got:\n{opts_out}" + ); + assert!( + opts_out.contains("After.\nNext."), + "prose after optional-arg PitonInputFile must still split, got:\n{opts_out}" + ); + + let range = concat!( + "Before. Next.\n", + "\\PitonInputFile<1-10>[language=python]{foo.py}\n", + "After. Next.\n", + ); + let range_out = format_text(range, &latex_cfg()).unwrap(); + assert!( + range_out.contains("\\PitonInputFile<1-10>[language=python]{foo.py}\n"), + "PitonInputFile d<> plus optional args must stay atomic, got:\n{range_out}" + ); + assert!( + !range_out.contains("\\PitonInputFile<1-10>[language=python]{foo.py} After."), + "d<> PitonInputFile must not join following prose, got:\n{range_out}" + ); + assert!( + range_out.contains("After.\nNext."), + "prose after d<> PitonInputFile must still split, got:\n{range_out}" + ); + + let angle = concat!( + "Before. Next.\n", + "\\PitonInputFile{foo.py}\n", + "After. Next.\n", + ); + let angle_out = format_text(angle, &latex_cfg()).unwrap(); + assert!( + angle_out.contains("\\PitonInputFile{foo.py}\n"), + "PitonInputFile standalone angle must stay atomic, got:\n{angle_out}" + ); + assert!( + !angle_out.contains("\\PitonInputFile{foo.py} After."), + "standalone-angle PitonInputFile must not join following prose, got:\n{angle_out}" + ); + assert!( + angle_out.contains("After.\nNext."), + "prose after standalone-angle PitonInputFile must still split, got:\n{angle_out}" + ); + + let piton_env = concat!( + "\\begin{Piton}\n", + "First line. Second line.\n", + "\\end{Piton}\n", + "After the block. Next.\n", + ); + let piton_env_out = format_text(piton_env, &latex_cfg()).unwrap(); + assert!( + piton_env_out.contains("\\begin{Piton}\nFirst line. Second line.\n\\end{Piton}"), + "Piton env must stay a code env, got:\n{piton_env_out}" + ); + assert!( + piton_env_out.contains("After the block.\nNext."), + "prose after Piton env must still split, got:\n{piton_env_out}" + ); + + let piton_cmd = "See \\piton|done. Next| here. After.\n"; + let piton_cmd_out = format_text(piton_cmd, &latex_cfg()).unwrap(); + assert!( + piton_cmd_out.contains("See \\piton|done. Next| here.\nAfter."), + "\\piton must stay intact and still split, got:\n{piton_cmd_out}" + ); + + let verb_in = concat!( + "Before. Next.\n", + "\\verbatiminput{foo.py}\n", + "After. Next.\n", + ); + let verb_out = format_text(verb_in, &latex_cfg()).unwrap(); + assert!( + verb_out.contains("\\verbatiminput{foo.py}\n"), + "verbatiminput must stay unchanged, got:\n{verb_out}" + ); + assert!( + !verb_out.contains("\\verbatiminput{foo.py} After."), + "verbatiminput must not join following prose, got:\n{verb_out}" + ); + assert!( + verb_out.contains("After.\nNext."), + "prose after verbatiminput must still split, got:\n{verb_out}" + ); + + let fancy = concat!( + "Before. Next.\n", + "\\VerbatimInput{foo.py}\n", + "After. Next.\n", + ); + let fancy_out = format_text(fancy, &latex_cfg()).unwrap(); + assert!( + fancy_out.contains("\\VerbatimInput{foo.py}\n"), + "VerbatimInput must stay unchanged, got:\n{fancy_out}" + ); + assert!( + !fancy_out.contains("\\VerbatimInput{foo.py} After."), + "VerbatimInput must not join following prose, got:\n{fancy_out}" + ); + assert!( + !fancy_out.contains("\\PitonInputFile"), + "PitonInputFile walker must not rewrite VerbatimInput, got:\n{fancy_out}" + ); + assert!( + fancy_out.contains("After.\nNext."), + "prose after VerbatimInput must still split, got:\n{fancy_out}" + ); + } + /// Ticket fixture (GitHub #245): minted `\mintinline{lang}|body|` is /// one token; following `Next sentence.` still splits. #[test] diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index cc361b09..694c0a7b 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -201,7 +201,8 @@ pub fn protect_inline_tokens_with( /// `\inputminted{lang}{file}` / `\Verb|...|` / /// `\SaveVerb{name}|...|` / `\piton|...|` / /// `\lstinputlisting[...]{file}` / -/// `\verbatiminput{file}` / `\VerbatimInput[...]{file}` so inner `.!?%` cannot +/// `\verbatiminput{file}` / `\VerbatimInput[...]{file}` / +/// `\PitonInputFile<...>[...]{file}` so inner `.!?%` cannot /// split or comment. `\piton{...}` stays on the generic `\cmd{arg}` /// path (piton.sty brace syntax is not verbatim; GitHub #305). fn protect_latex_verbatim( @@ -229,8 +230,8 @@ fn protect_latex_verbatim( /// Byte end of a `\verb` / `\lstinline` / `\spverb` / `\mintinline` / /// `\mint` / `\inputminted` / `\Verb` / `\SaveVerb` / `\piton` / -/// `\lstinputlisting` / `\VerbatimInput` / `\BVerbatimInput` / -/// `\LVerbatimInput` / +/// `\lstinputlisting` / `\verbatiminput` / `\VerbatimInput` / +/// `\BVerbatimInput` / `\LVerbatimInput` / `\PitonInputFile` / /// extra-name span starting at `at`. /// /// `\verb` / `\verb*` / `\spverb` / `\spverb*` / `\Verb` / `\Verb*`: next @@ -249,12 +250,17 @@ fn protect_latex_verbatim( /// delimiter so `\piton{...}` stays on the generic `\cmd{arg}` path. /// `\lstinputlisting` / `\lstinputlisting*` (listings.sty leftover; /// GitHub #391) take optional `[...]` then a required `{filename}`; -/// no brace is not a span. `\VerbatimInput` / `\BVerbatimInput` / -/// `\LVerbatimInput` (fancyvrb.sty leftover; GitHub #399) use the same -/// optional `[...]` then `{filename}` walk. Matched before `\Verb` so -/// `\VerbatimInput` is not `\Verb` plus leftover letters. Extra names -/// are tokenized like `\verb`. With no closer, the span runs to end of -/// line so an inner `%` is not a comment. +/// no brace is not a span. `\verbatiminput` / `\verbatiminput*` +/// (tools/verbatim.sty leftover; GitHub #398) take a required +/// `{filename}`; no brace is not a span. `\VerbatimInput` / +/// `\BVerbatimInput` / `\LVerbatimInput` (fancyvrb.sty leftover; +/// GitHub #399) use the same optional `[...]` then `{filename}` walk. +/// Matched before `\Verb` so `\VerbatimInput` is not `\Verb` plus +/// leftover letters. `\PitonInputFile` (piton.sty leftover; +/// `d < > O { } m`; GitHub #406) takes optional `<...>`, optional +/// `[...]`, then a required `{filename}`; no brace is not a span. +/// Extra names are tokenized like `\verb`. With no closer, the span +/// runs to end of line so an inner `%` is not a comment. pub(crate) fn latex_verb_span_end_with( text: &str, at: usize, @@ -293,6 +299,13 @@ pub(crate) fn latex_verb_span_end_with( return None; } (after_bs + "verbatiminput".len(), VerbKind::Verbatiminput) + } else if let Some(stripped) = tail.strip_prefix("PitonInputFile") { + // Longer than `\piton` and a different case; match by full name + // so this leftover is not `\piton` + alphabetic leftover. + if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { + return None; + } + (after_bs + "PitonInputFile".len(), VerbKind::PitonInputFile) } else if let Some(stripped) = tail.strip_prefix("lstinline") { if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { return None; @@ -363,6 +376,28 @@ pub(crate) fn latex_verb_span_end_with( } } + // piton.sty `\PitonInputFile[opts]{file}` (`d < > O { } m`). + if kind == VerbKind::PitonInputFile { + i = skip_ascii_ws(text, i); + if text.get(i..).is_some_and(|s| s.starts_with('<')) { + match skip_angle_group(text, i) { + Some(end) => i = skip_ascii_ws(text, end), + None => return Some(line_end(text, i)), + } + } + if text.get(i..).is_some_and(|s| s.starts_with('[')) { + match skip_bracket_group(text, i) { + Some(end) => i = skip_ascii_ws(text, end), + None => return Some(line_end(text, i)), + } + } + if !text.get(i..).is_some_and(|s| s.starts_with('{')) { + return None; + } + i += 1; + return Some(find_unescaped_brace_close(text, i).unwrap_or_else(|| line_end(text, i))); + } + // listings.sty `\lstinputlisting[opts]{file}` and fancyvrb // `\VerbatimInput[opts]{file}` (B/L twins) are brace-only. if matches!(kind, VerbKind::Lstinputlisting | VerbKind::Verbatiminput) { @@ -435,6 +470,9 @@ enum VerbKind { Lstinputlisting, /// `\\verbatiminput`: required `{filename}` (verbatim.sty leftover). Verbatiminput, + /// `\PitonInputFile`: optional `<...>`, optional `[...]`, required + /// `{filename}` (`d < > O { } m`; GitHub #406). + PitonInputFile, /// `\mintinline` / `\mint` / `\inputminted`: optional `[...]`, /// `{lang}`, then body. Mint, @@ -484,6 +522,8 @@ fn match_extra_verb_command<'a>(tail: &'a str, extras: &'a [String]) -> Option<& || name == "BVerbatimInput" || name == "LVerbatimInput" || name == "piton" + || name == "verbatiminput" + || name == "PitonInputFile" { continue; } @@ -531,6 +571,24 @@ fn skip_bracket_group(text: &str, open_at: usize) -> Option { None } +/// xparse `d < >`: one optional delimited group. First `>` closes; +/// not nested (piton.sty `\PitonInputFile`; GitHub #406). +fn skip_angle_group(text: &str, open_at: usize) -> Option { + let bytes = text.as_bytes(); + if bytes.get(open_at) != Some(&b'<') { + return None; + } + let mut i = open_at + 1; + while i < bytes.len() { + match bytes[i] { + b'\n' => return None, + b'>' => return Some(i + 1), + _ => i += 1, + } + } + None +} + fn find_unescaped_brace_close(text: &str, mut i: usize) -> Option { let bytes = text.as_bytes(); while i < bytes.len() { @@ -2445,6 +2503,82 @@ mod tests { ); } + /// Ticket fixture (GitHub #406): piton.sty `\PitonInputFile{file}` + /// is one leftover command; following `After.` still splits. + /// Optional `[...]` and `d < >` stay in the span (A combined + B + /// standalone angle). `\piton` / tools/verbatim.sty + /// `\verbatiminput` / fancyvrb `\VerbatimInput` stay spans. + #[test] + fn latex_pitoninputfile_stays_atomic() { + let text = r"See \PitonInputFile{foo.py} here. After."; + let (_, placeholders) = protect_inline_tokens(text); + assert!( + placeholders.iter().any(|p| p == r"\PitonInputFile{foo.py}"), + "PitonInputFile span must be protected, got {placeholders:?}" + ); + assert_eq!( + latex_verb_span_end_with(r"\PitonInputFile{foo.py}", 0, &[]), + Some(r"\PitonInputFile{foo.py}".len()) + ); + assert_eq!( + split(text), + vec![ + r"See \PitonInputFile{foo.py} here.".to_string(), + "After.".to_string() + ] + ); + let opts = r"See \PitonInputFile[language=python]{foo.py} here. After."; + let (_, opt_ph) = protect_inline_tokens(opts); + assert!( + opt_ph + .iter() + .any(|p| p == r"\PitonInputFile[language=python]{foo.py}"), + "PitonInputFile optional args must be protected, got {opt_ph:?}" + ); + assert_eq!( + latex_verb_span_end_with(r"\PitonInputFile[language=python]{foo.py}", 0, &[]), + Some(r"\PitonInputFile[language=python]{foo.py}".len()) + ); + assert_eq!( + split(opts), + vec![ + r"See \PitonInputFile[language=python]{foo.py} here.".to_string(), + "After.".to_string() + ] + ); + assert_eq!( + latex_verb_span_end_with(r"\PitonInputFile{foo.py}", 0, &[]), + Some(r"\PitonInputFile{foo.py}".len()), + "PitonInputFile standalone angle spec must stay in the span" + ); + let range = r"\PitonInputFile<1-10>[language=python]{foo.py}"; + assert_eq!( + latex_verb_span_end_with(range, 0, &[]), + Some(range.len()), + "PitonInputFile d<> plus optional args must stay one span" + ); + assert_eq!( + latex_verb_span_end_with(r"\PitonInputFile foo.py", 0, &[]), + None, + "PitonInputFile without a brace file arg is not a verb span" + ); + assert_eq!( + latex_verb_span_end_with(r"\piton|done. Next|", 0, &[]), + Some(r"\piton|done. Next|".len()), + "PitonInputFile must not steal piton" + ); + assert_eq!( + latex_verb_span_end_with(r"\verbatiminput{foo.py}", 0, &[]), + Some(r"\verbatiminput{foo.py}".len()), + "PitonInputFile must not steal verbatiminput" + ); + assert_eq!( + latex_verb_span_end_with(r"\VerbatimInput{foo.py}", 0, &[]), + Some(r"\VerbatimInput{foo.py}".len()), + "PitonInputFile must not steal VerbatimInput" + ); + } + /// Ticket fixture (GitHub #275): fancyvrb `\SaveVerb{name}|body|` /// is one token; following `After.` still splits. #[test] diff --git a/tests/latex_verbatim_envs.rs b/tests/latex_verbatim_envs.rs index 1bbc1545..dbc042e6 100644 --- a/tests/latex_verbatim_envs.rs +++ b/tests/latex_verbatim_envs.rs @@ -14,6 +14,8 @@ //! GitHub #399: fancyvrb.sty \\VerbatimInput / \\BVerbatimInput / //! \\LVerbatimInput is one leftover command; following flush prose //! stays on its own line. +//! GitHub #406: piton.sty \\PitonInputFile is one leftover command; +//! following flush prose stays on its own line. //! GitHub #273: minted.sty minted* is the starred twin of minted (same raw body). //! GitHub #275: fancyvrb \\SaveVerb{name}|body| is the same delimiter body as \\Verb. //! GitHub #246: tcolorbox listings tcblisting* is the starred twin of tcblisting. @@ -3086,6 +3088,178 @@ fn verbatiminput_fixture_does_not_join_following_prose() { ); } +/// Ticket fixture (GitHub #406): piton.sty `\PitonInputFile{file}` +/// stays one atomic command. Following flush `After.` does not join +/// the command line. `After.` / `Next.` still split. Piton env, +/// `\piton`, tools/verbatim.sty `\verbatiminput`, and fancyvrb +/// `\VerbatimInput` unchanged. +#[test] +fn pitoninputfile_fixture_does_not_join_following_prose() { + let input = concat!( + "Before. Next.\n", + "\\PitonInputFile{foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\PitonInputFile{foo.py}") + )), + "PitonInputFile must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\PitonInputFile{foo.py}") + )), + "PitonInputFile must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\PitonInputFile{foo.py}\n"), + "PitonInputFile must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\PitonInputFile{foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before PitonInputFile must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after PitonInputFile must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let opts = concat!( + "Before. Next.\n", + "\\PitonInputFile[language=python]{foo.py}\n", + "After. Next.\n", + ); + let opts_out = format_text(opts, &latex_cfg()).unwrap(); + assert!( + opts_out.contains("\\PitonInputFile[language=python]{foo.py}\n"), + "PitonInputFile optional args must stay atomic, got:\n{opts_out}" + ); + assert!( + !opts_out.contains("\\PitonInputFile[language=python]{foo.py} After."), + "optional-arg PitonInputFile must not join following prose, got:\n{opts_out}" + ); + assert!( + opts_out.contains("After.\nNext."), + "prose after optional-arg PitonInputFile must still split, got:\n{opts_out}" + ); + + let range = concat!( + "Before. Next.\n", + "\\PitonInputFile<1-10>[language=python]{foo.py}\n", + "After. Next.\n", + ); + let range_out = format_text(range, &latex_cfg()).unwrap(); + assert!( + range_out.contains("\\PitonInputFile<1-10>[language=python]{foo.py}\n"), + "PitonInputFile d<> plus optional args must stay atomic, got:\n{range_out}" + ); + assert!( + !range_out.contains("\\PitonInputFile<1-10>[language=python]{foo.py} After."), + "d<> PitonInputFile must not join following prose, got:\n{range_out}" + ); + assert!( + range_out.contains("After.\nNext."), + "prose after d<> PitonInputFile must still split, got:\n{range_out}" + ); + + let angle = concat!( + "Before. Next.\n", + "\\PitonInputFile{foo.py}\n", + "After. Next.\n", + ); + let angle_out = format_text(angle, &latex_cfg()).unwrap(); + assert!( + angle_out.contains("\\PitonInputFile{foo.py}\n"), + "PitonInputFile standalone angle must stay atomic, got:\n{angle_out}" + ); + assert!( + !angle_out.contains("\\PitonInputFile{foo.py} After."), + "standalone-angle PitonInputFile must not join following prose, got:\n{angle_out}" + ); + + let piton_env = concat!( + "\\begin{Piton}\n", + "First line. Second line.\n", + "\\end{Piton}\n", + "After the block. Next.\n", + ); + let piton_env_out = format_text(piton_env, &latex_cfg()).unwrap(); + assert!( + piton_env_out.contains("\\begin{Piton}\nFirst line. Second line.\n\\end{Piton}"), + "Piton env must stay a code env, got:\n{piton_env_out}" + ); + assert!( + piton_env_out.contains("After the block.\nNext."), + "prose after Piton env must still split, got:\n{piton_env_out}" + ); + + let piton_cmd = "See \\piton|done. Next| here. After.\n"; + let piton_cmd_out = format_text(piton_cmd, &latex_cfg()).unwrap(); + assert!( + piton_cmd_out.contains("See \\piton|done. Next| here.\nAfter."), + "\\piton must stay intact and still split, got:\n{piton_cmd_out}" + ); + + let verb_in = concat!( + "Before. Next.\n", + "\\verbatiminput{foo.py}\n", + "After. Next.\n", + ); + let verb_out = format_text(verb_in, &latex_cfg()).unwrap(); + assert!( + verb_out.contains("\\verbatiminput{foo.py}\n"), + "verbatiminput must stay one atomic command, got:\n{verb_out}" + ); + assert!( + !verb_out.contains("\\verbatiminput{foo.py} After."), + "verbatiminput must not join following prose, got:\n{verb_out}" + ); + assert!( + verb_out.contains("After.\nNext."), + "prose after verbatiminput must still split, got:\n{verb_out}" + ); + + let fancy = concat!( + "Before. Next.\n", + "\\VerbatimInput{foo.py}\n", + "After. Next.\n", + ); + let fancy_out = format_text(fancy, &latex_cfg()).unwrap(); + assert!( + fancy_out.contains("\\VerbatimInput{foo.py}\n"), + "VerbatimInput must stay one atomic command, got:\n{fancy_out}" + ); + assert!( + !fancy_out.contains("\\VerbatimInput{foo.py} After."), + "VerbatimInput must not join following prose, got:\n{fancy_out}" + ); + assert!( + !fancy_out.contains("\\PitonInputFile"), + "PitonInputFile walker must not rewrite VerbatimInput, got:\n{fancy_out}" + ); + assert!( + fancy_out.contains("After.\nNext."), + "prose after VerbatimInput must still split, got:\n{fancy_out}" + ); +} + /// Ticket fixture (GitHub #245): `\mintinline{python}|a.b! c|` is one /// token; following `Next sentence.` still splits. `{lang}{body}` and /// `\mint` are the same class. @@ -3358,105 +3532,106 @@ fn piton_env_and_pipe_cmd_fixture_is_code_and_does_not_reflow() { verb_out.contains("Use \\verb|a.b! c| here.\nNext sentence."), "prose after \\verb must still split, got:\n{verb_out}" ); - /// Ticket fixture (GitHub #398): tools/verbatim.sty `\verbatiminput{file}` - /// stays one atomic command. Following flush `After.` does not join - /// the command line. `After.` / `Next.` still split. lstinputlisting / - /// inputminted unchanged. - #[test] - fn verbatiminput_fixture_does_not_join_following_prose() { - let input = concat!( - "Before. Next.\n", - "\\verbatiminput{foo.py}\n", - "After. Next.\n", - ); - let regions = LatexParser::default().parse(input); - assert!( - regions.iter().any(|r| matches!( - r, - Region::Structure(s) if s.contains(r"\verbatiminput{foo.py}") - )), - "verbatiminput must stay one Structure command, got: {regions:?}" - ); - assert!( - !regions.iter().any(|r| matches!( - r, - Region::Prose(p) if p.contains(r"\verbatiminput{foo.py}") - )), - "verbatiminput must not leak into Prose, got: {regions:?}" - ); - assert!( - regions.iter().any(|r| matches!( - r, - Region::Prose(p) if p.contains("After.") && p.contains("Next.") - )), - "After. / Next. must stay Prose, got: {regions:?}" - ); - let out = format_text(input, &latex_cfg()).unwrap(); - assert!( - out.contains("\\verbatiminput{foo.py}\n"), - "verbatiminput must stay one atomic command, got:\n{out}" - ); - assert!( - !out.contains("\\verbatiminput{foo.py} After."), - "following flush prose must not join the command line, got:\n{out}" - ); - assert!( - out.contains("Before.\nNext."), - "prose before verbatiminput must still split, got:\n{out}" - ); - assert!( - out.contains("After.\nNext."), - "prose after verbatiminput must still split, got:\n{out}" - ); - assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); +} - let star = concat!( - "Before. Next.\n", - "\\verbatiminput*{foo.py}\n", - "After. Next.\n", - ); - let star_out = format_text(star, &latex_cfg()).unwrap(); - assert!( - star_out.contains("\\verbatiminput*{foo.py}\n"), - "verbatiminput* must stay one atomic command, got:\n{star_out}" - ); - assert!( - !star_out.contains("\\verbatiminput*{foo.py} After."), - "starred verbatiminput must not join following prose, got:\n{star_out}" - ); - assert!( - star_out.contains("After.\nNext."), - "prose after starred verbatiminput must still split, got:\n{star_out}" - ); +/// Ticket fixture (GitHub #398): tools/verbatim.sty `\verbatiminput{file}` +/// stays one atomic command. Following flush `After.` does not join +/// the command line. `After.` / `Next.` still split. lstinputlisting / +/// inputminted unchanged. +#[test] +fn tools_verbatiminput_fixture_does_not_join_following_prose() { + let input = concat!( + "Before. Next.\n", + "\\verbatiminput{foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\verbatiminput{foo.py}") + )), + "verbatiminput must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\verbatiminput{foo.py}\n"), + "verbatiminput must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\verbatiminput{foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before verbatiminput must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after verbatiminput must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); - let lst = concat!( - "Before. Next.\n", - "\\lstinputlisting{foo.py}\n", - "After. Next.\n", - ); - let lst_out = format_text(lst, &latex_cfg()).unwrap(); - assert!( - lst_out.contains("\\lstinputlisting{foo.py}\n"), - "lstinputlisting must stay unchanged, got:\n{lst_out}" - ); - assert!( - !lst_out.contains("\\lstinputlisting{foo.py} After."), - "lstinputlisting must not join following prose, got:\n{lst_out}" - ); + let star = concat!( + "Before. Next.\n", + "\\verbatiminput*{foo.py}\n", + "After. Next.\n", + ); + let star_out = format_text(star, &latex_cfg()).unwrap(); + assert!( + star_out.contains("\\verbatiminput*{foo.py}\n"), + "verbatiminput* must stay one atomic command, got:\n{star_out}" + ); + assert!( + !star_out.contains("\\verbatiminput*{foo.py} After."), + "starred verbatiminput must not join following prose, got:\n{star_out}" + ); + assert!( + star_out.contains("After.\nNext."), + "prose after starred verbatiminput must still split, got:\n{star_out}" + ); - let minted = concat!( - "Before. Next.\n", - "\\inputminted{python}{foo.py}\n", - "After. Next.\n", - ); - let minted_out = format_text(minted, &latex_cfg()).unwrap(); - assert!( - minted_out.contains("\\inputminted{python}{foo.py}\n"), - "inputminted must stay unchanged, got:\n{minted_out}" - ); - assert!( - !minted_out.contains("\\inputminted{python}{foo.py} After."), - "inputminted must not join following prose, got:\n{minted_out}" - ); - } + let lst = concat!( + "Before. Next.\n", + "\\lstinputlisting{foo.py}\n", + "After. Next.\n", + ); + let lst_out = format_text(lst, &latex_cfg()).unwrap(); + assert!( + lst_out.contains("\\lstinputlisting{foo.py}\n"), + "lstinputlisting must stay unchanged, got:\n{lst_out}" + ); + assert!( + !lst_out.contains("\\lstinputlisting{foo.py} After."), + "lstinputlisting must not join following prose, got:\n{lst_out}" + ); + + let minted = concat!( + "Before. Next.\n", + "\\inputminted{python}{foo.py}\n", + "After. Next.\n", + ); + let minted_out = format_text(minted, &latex_cfg()).unwrap(); + assert!( + minted_out.contains("\\inputminted{python}{foo.py}\n"), + "inputminted must stay unchanged, got:\n{minted_out}" + ); + assert!( + !minted_out.contains("\\inputminted{python}{foo.py} After."), + "inputminted must not join following prose, got:\n{minted_out}" + ); }