From d8b2eada790f6f69c0f10fadf6e6b2b64f486bcd Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 12 Sep 2026 10:17:00 -0500 Subject: [PATCH] fix(latex): tokenize tcolorbox tcbinputlisting like lstinputlisting tcolorbox \tcbinputlisting{keyvals} is one leftover command (one keyval group). Tokenize it so the span stays atomic. A leftover walker emits the command as Structure so following flush After. does not join. After. / Next. still split. tcboutputlisting / lstinputlisting / inputminted unchanged. GitHub #400. --- docs/orgmode/reference/config.org | 3 +- docs/orgmode/reference/formats.org | 3 +- src/config.rs | 2 +- src/lib.rs | 2 +- src/parser/latex.rs | 140 ++++++++++++++++++++++++++++- src/sentence/unicode.rs | 76 +++++++++++++++- tests/latex_verbatim_envs.rs | 111 +++++++++++++++++++++++ 7 files changed, 328 insertions(+), 9 deletions(-) diff --git a/docs/orgmode/reference/config.org b/docs/orgmode/reference/config.org index 08f7438b..7a08988c 100644 --- a/docs/orgmode/reference/config.org +++ b/docs/orgmode/reference/config.org @@ -187,12 +187,13 @@ Adding =algorithm= stops reflow of that environment. String array of extra command names tokenized like the built-in verb command before sentence split. The next character after the name is the delimiter. -Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb=, and piton.sty =\piton=; =lstinline= still accepts optional =[...]= and ={...}=. +Built-in names are =verb=, =lstinline=, =lstinputlisting=, =spverb=, =mintinline=, =inputminted=, =mint=, fancyvrb =Verb= / =Verb*= / =SaveVerb=, piton.sty =\piton=, and tcolorbox =\tcbinputlisting=; =lstinline= still accepts optional =[...]= and ={...}=. =mintinline= / =mint= take optional =[...]=, a ={lang}= argument, then a delimiter or ={...}= body. =\inputminted= takes the same optional =[...]= and ={lang}= then a required ={filename}=; following flush prose stays on its own line. =SaveVerb= takes optional =[...]=, a ={name}= argument, then the same delimiter body as =Verb=. =\piton|...|= is verb-like (interior =.!?%= stay one token); =\piton{...}= stays one token via the generic command argument. =\lstinputlisting= takes optional =[...]= then a required ={filename}=; following flush prose stays on its own line. +=\tcbinputlisting= takes one required ={keyval}= group; following flush prose stays on its own line. Adding another name keeps that command's delimited body atomic and treats an inner =%= as content, not a comment. * Code-block sections (=[code.]=) diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 0996ddd6..5946e43d 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -135,10 +135,11 @@ These tokens within prose are not split across lines: - piton.sty =Piton= is a built-in code region (verbatim listing env) - Extra names from =[latex].verbatim_envs= are code regions too - Body follows the same comment-reflow and optional =--format-code= rules as other formats when language is known (=minted= / =minted*= language arg, =lstlisting= / =lstlisting*= =language== option) -- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / piton.sty =\piton= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. +- Inline =\verb= / =\lstinline= / =\spverb= / =\mintinline= / =\mint= / fancyvrb =\Verb= / =\Verb*= / =\SaveVerb= / piton.sty =\piton= / listings.sty =\lstinputlisting= / minted.sty =\inputminted= / tcolorbox =\tcbinputlisting= (and extra =[latex].verbatim_commands=) stay atomic; inner =.!?%= do not split or comment. =\piton|...|= is verb-like; =\piton{...}= stays one token via the generic command argument =\lstinputlisting= / =\lstinputlisting*= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\inputminted= / =\inputminted*= take optional =[...]=, ={lang}=, then a ={filename}=; a flush following sentence stays on its own line + =\tcbinputlisting= takes one required ={keyval}= group; a flush following sentence stays on its own line *** Prose regions (reflowed) :PROPERTIES: diff --git a/src/config.rs b/src/config.rs index be8c6aeb..c396e79c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -49,7 +49,7 @@ pub struct FormatOverrides { pub structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. /// Meaningful under `[latex]` only. Missing or empty keeps - /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. + /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton/tcbinputlisting. pub verbatim_commands: Vec, } diff --git a/src/lib.rs b/src/lib.rs index d080441d..48c98058 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ pub struct FormatConfig { /// to `NON_PROSE_ENVS`. Empty keeps the built-in list. pub latex_structure_envs: Vec, /// Extra LaTeX command names tokenized like `\verb` before split. - /// Empty keeps verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton. + /// Empty keeps verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/piton/tcbinputlisting. pub latex_verbatim_commands: Vec, } diff --git a/src/parser/latex.rs b/src/parser/latex.rs index 0ffcb453..cdec74dd 100644 --- a/src/parser/latex.rs +++ b/src/parser/latex.rs @@ -502,7 +502,7 @@ impl LatexParser { /// Byte offset of the first `%` that is not escaped as `\%` and is not /// inside `\verb` / `\lstinline` / `\spverb` / `\mintinline` / `\mint` / /// `\Verb` / `\SaveVerb` / `\piton` / `\lstinputlisting` / - /// `\inputminted` / configured verbatim commands. + /// `\inputminted` / `\tcbinputlisting` / configured verbatim commands. fn unescaped_percent(&self, line: &str) -> Option { unescaped_percent_with(line, &self.extra_verbatim_commands) } @@ -875,7 +875,7 @@ fn find_tex_cs(line: &str, from: usize, cs: &str) -> Option { /// `\iffalse` in ordinary TeX, skipping `\verb` / `\lstinline` / /// `\spverb` / `\mintinline` / `\mint` / `\inputminted` / `\Verb` / -/// `\SaveVerb` / `\piton` / `\lstinputlisting` spans. +/// `\SaveVerb` / `\piton` / `\lstinputlisting` / `\tcbinputlisting` spans. fn find_iffalse_at(line: &str, from: usize, extra_cmds: &[String]) -> Option { let bytes = line.as_bytes(); let mut i = from; @@ -917,6 +917,17 @@ fn find_inputminted_at(line: &str, from: usize, extra_cmds: &[String]) -> Option find_leftover_cmd_at(line, from, extra_cmds, inputminted_cs_at) } +/// Leftover tcolorbox `\tcbinputlisting` (one keyval group; GitHub #400). +/// Other verb spans are skipped so `\verb|\tcbinputlisting{x}|` is not +/// stolen. Walk stops at an unescaped `%` so a comment is not a tail. +fn find_tcbinputlisting_at( + line: &str, + from: usize, + extra_cmds: &[String], +) -> Option<(usize, usize)> { + find_leftover_cmd_at(line, from, extra_cmds, tcbinputlisting_cs_at) +} + fn find_leftover_cmd_at( line: &str, from: usize, @@ -967,6 +978,16 @@ fn inputminted_cs_at(line: &str, at: usize) -> bool { !after.starts_with(|c: char| c.is_ascii_alphabetic()) } +fn tcbinputlisting_cs_at(line: &str, at: usize) -> bool { + let Some(tail) = line.get(at..).and_then(|s| s.strip_prefix('\\')) else { + return false; + }; + let Some(after) = tail.strip_prefix("tcbinputlisting") else { + return false; + }; + !after.starts_with(|c: char| c.is_ascii_alphabetic()) +} + struct ParseState<'a> { input: &'a str, parser: &'a LatexParser, @@ -1652,6 +1673,9 @@ impl<'a> ParseState<'a> { if let Some((start, end)) = find_lstinputlisting_at(code, i, &self.parser.extra_verbatim_commands) .or_else(|| find_inputminted_at(code, i, &self.parser.extra_verbatim_commands)) + .or_else(|| { + find_tcbinputlisting_at(code, i, &self.parser.extra_verbatim_commands) + }) { self.append_item_or_prose(line.start + i, &code[i..start]); self.push_structure(ByteSpan::new( @@ -2580,6 +2604,118 @@ Some text. assert_eq!(format_text(&minted_out, &latex_cfg()).unwrap(), minted_out); } + /// Ticket fixture (GitHub #400): tcolorbox `\tcbinputlisting{keys}` + /// is one leftover command (one keyval group). Following flush + /// prose does not join the command line. `After.` / `Next.` still + /// split. `tcboutputlisting` / `lstinputlisting` / `inputminted` + /// unchanged. + #[test] + fn tcbinputlisting_does_not_join_following_prose() { + use crate::format_text; + + let input = concat!( + "Before. Next.\n", + "\\tcbinputlisting{listing file=foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\tcbinputlisting{listing file=foo.py}") + )), + "tcbinputlisting must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\tcbinputlisting{listing file=foo.py}") + )), + "tcbinputlisting must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\tcbinputlisting{listing file=foo.py}\n"), + "tcbinputlisting must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\tcbinputlisting{listing file=foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before tcbinputlisting must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after tcbinputlisting must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let tcbout = concat!( + "\\begin{tcboutputlisting}\n", + "First line. Second line.\n", + "\\end{tcboutputlisting}\n", + "After the block. Next.\n", + ); + let tcbout_out = format_text(tcbout, &latex_cfg()).unwrap(); + assert!( + tcbout_out.contains( + "\\begin{tcboutputlisting}\nFirst line. Second line.\n\\end{tcboutputlisting}" + ), + "tcboutputlisting must stay a code env, got:\n{tcbout_out}" + ); + assert!( + tcbout_out.contains("After the block.\nNext."), + "prose after tcboutputlisting must still split, got:\n{tcbout_out}" + ); + + let lst = concat!( + "Before. Next.\n", + "\\lstinputlisting{foo.py}\n", + "After. Next.\n", + ); + let lst_out = format_text(lst, &latex_cfg()).unwrap(); + assert!( + lst_out.contains("\\lstinputlisting{foo.py}\n"), + "lstinputlisting must stay one atomic command, got:\n{lst_out}" + ); + assert!( + !lst_out.contains("\\lstinputlisting{foo.py} After."), + "lstinputlisting must not join following prose, got:\n{lst_out}" + ); + assert!( + lst_out.contains("After.\nNext."), + "prose after lstinputlisting must still split, got:\n{lst_out}" + ); + + let minted = concat!( + "Before. Next.\n", + "\\inputminted{python}{foo.py}\n", + "After. Next.\n", + ); + let minted_out = format_text(minted, &latex_cfg()).unwrap(); + assert!( + minted_out.contains("\\inputminted{python}{foo.py}\n"), + "inputminted must stay one atomic command, got:\n{minted_out}" + ); + assert!( + !minted_out.contains("\\inputminted{python}{foo.py} After."), + "inputminted must not join following prose, got:\n{minted_out}" + ); + assert!( + minted_out.contains("After.\nNext."), + "prose after inputminted must still split, got:\n{minted_out}" + ); + } + /// Ticket fixture (GitHub #245): minted `\mintinline{lang}|body|` is /// one token; following `Next sentence.` still splits. #[test] diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 66062acb..05f827df 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -200,7 +200,7 @@ pub fn protect_inline_tokens_with( /// `\mintinline{lang}|...|` / `\mint{lang}{...}` / /// `\inputminted{lang}{file}` / `\Verb|...|` / /// `\SaveVerb{name}|...|` / `\piton|...|` / -/// `\lstinputlisting[...]{file}` so inner `.!?%` cannot +/// `\lstinputlisting[...]{file}` / `\tcbinputlisting{keys}` so inner `.!?%` cannot /// split or comment. `\piton{...}` stays on the generic `\cmd{arg}` /// path (piton.sty brace syntax is not verbatim; GitHub #305). fn protect_latex_verbatim( @@ -228,7 +228,7 @@ fn protect_latex_verbatim( /// Byte end of a `\verb` / `\lstinline` / `\spverb` / `\mintinline` / /// `\mint` / `\inputminted` / `\Verb` / `\SaveVerb` / `\piton` / -/// `\lstinputlisting` / +/// `\lstinputlisting` / `\tcbinputlisting` / /// extra-name span starting at `at`. /// /// `\verb` / `\verb*` / `\spverb` / `\spverb*` / `\Verb` / `\Verb*`: next @@ -247,7 +247,10 @@ fn protect_latex_verbatim( /// delimiter so `\piton{...}` stays on the generic `\cmd{arg}` path. /// `\lstinputlisting` / `\lstinputlisting*` (listings.sty leftover; /// GitHub #391) take optional `[...]` then a required `{filename}`; -/// no brace is not a span. Extra names are tokenized like `\verb`. +/// no brace is not a span. `\tcbinputlisting` (tcolorbox leftover; +/// GitHub #400) takes one required `{keyval}` group; no optional +/// `[...]` and no brace is not a span. Extra names are tokenized +/// like `\verb`. /// With no closer, the span runs to end of line so an inner `%` is /// not a comment. pub(crate) fn latex_verb_span_end_with( @@ -282,6 +285,14 @@ pub(crate) fn latex_verb_span_end_with( after_bs + "lstinputlisting".len(), VerbKind::Lstinputlisting, ) + } else if let Some(stripped) = tail.strip_prefix("tcbinputlisting") { + if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { + return None; + } + ( + after_bs + "tcbinputlisting".len(), + VerbKind::Tcbinputlisting, + ) } else if let Some(stripped) = tail.strip_prefix("lstinline") { if stripped.starts_with(|c: char| c.is_ascii_alphabetic()) { return None; @@ -358,6 +369,17 @@ pub(crate) fn latex_verb_span_end_with( return Some(find_unescaped_brace_close(text, i).unwrap_or_else(|| line_end(text, i))); } + // tcolorbox `\tcbinputlisting{keyvals}` is one keyval group. + // No optional `[...]`; a following `[` is not this command. + if kind == VerbKind::Tcbinputlisting { + i = skip_ascii_ws(text, i); + if !text.get(i..).is_some_and(|s| s.starts_with('{')) { + return None; + } + i += 1; + return Some(find_unescaped_brace_close(text, i).unwrap_or_else(|| line_end(text, i))); + } + if kind == VerbKind::Mint { if !text.get(i..).is_some_and(|s| s.starts_with('{')) { return None; @@ -414,6 +436,8 @@ enum VerbKind { Lstinline, /// `\lstinputlisting`: optional `[...]` then required `{filename}`. Lstinputlisting, + /// `\tcbinputlisting`: one required `{keyval}` group (GitHub #400). + Tcbinputlisting, /// `\mintinline` / `\mint` / `\inputminted`: optional `[...]`, /// `{lang}`, then body. Mint, @@ -439,6 +463,7 @@ fn match_extra_verb_command<'a>(tail: &'a str, extras: &'a [String]) -> Option<& || name == "verb" || name == "lstinline" || name == "lstinputlisting" + || name == "tcbinputlisting" || name == "spverb" || name == "mintinline" || name == "inputminted" @@ -2336,6 +2361,51 @@ mod tests { ); } + /// Ticket fixture (GitHub #400): tcolorbox `\tcbinputlisting{keys}` + /// is one token (one keyval group); following `After.` still splits. + #[test] + fn latex_tcbinputlisting_stays_atomic() { + let text = r"See \tcbinputlisting{listing file=foo.py} here. After."; + let (_, placeholders) = protect_inline_tokens(text); + assert!( + placeholders + .iter() + .any(|p| p == r"\tcbinputlisting{listing file=foo.py}"), + "tcbinputlisting span must be protected, got {placeholders:?}" + ); + assert_eq!( + latex_verb_span_end_with(r"\tcbinputlisting{listing file=foo.py}", 0, &[]), + Some(r"\tcbinputlisting{listing file=foo.py}".len()) + ); + assert_eq!( + split(text), + vec![ + r"See \tcbinputlisting{listing file=foo.py} here.".to_string(), + "After.".to_string() + ] + ); + assert_eq!( + latex_verb_span_end_with(r"\tcbinputlisting foo.py", 0, &[]), + None, + "tcbinputlisting without a brace keyval is not a verb span" + ); + assert_eq!( + latex_verb_span_end_with(r"\tcbinputlisting[listing file=foo.py]", 0, &[]), + None, + "tcbinputlisting optional brackets are not a keyval group" + ); + assert_eq!( + latex_verb_span_end_with(r"\lstinputlisting{foo.py}", 0, &[]), + Some(r"\lstinputlisting{foo.py}".len()), + "tcbinputlisting must not steal lstinputlisting" + ); + assert_eq!( + latex_verb_span_end_with(r"\inputminted{python}{foo.py}", 0, &[]), + Some(r"\inputminted{python}{foo.py}".len()), + "tcbinputlisting must not steal inputminted" + ); + } + /// Ticket fixture (GitHub #275): fancyvrb `\SaveVerb{name}|body|` /// is one token; following `After.` still splits. #[test] diff --git a/tests/latex_verbatim_envs.rs b/tests/latex_verbatim_envs.rs index b7e758ec..4413e7ff 100644 --- a/tests/latex_verbatim_envs.rs +++ b/tests/latex_verbatim_envs.rs @@ -11,6 +11,8 @@ //! GitHub #245: minted.sty \\mintinline / \\mint take {lang} then a FancyVerb body. //! GitHub #394: minted.sty \\inputminted is one leftover command; //! following flush prose stays on its own line. +//! GitHub #400: tcolorbox \\tcbinputlisting is one leftover command +//! (one keyval group); following flush prose stays on its own line. //! GitHub #273: minted.sty minted* is the starred twin of minted (same raw body). //! GitHub #275: fancyvrb \\SaveVerb{name}|body| is the same delimiter body as \\Verb. //! GitHub #246: tcolorbox listings tcblisting* is the starred twin of tcblisting. @@ -2963,6 +2965,115 @@ fn inputminted_fixture_does_not_join_following_prose() { ); } +/// Ticket fixture (GitHub #400): tcolorbox `\tcbinputlisting{keys}` +/// stays one atomic command. Following flush `After.` does not join +/// the command line. `After.` / `Next.` still split. tcboutputlisting +/// / lstinputlisting / inputminted unchanged. +#[test] +fn tcbinputlisting_fixture_does_not_join_following_prose() { + let input = concat!( + "Before. Next.\n", + "\\tcbinputlisting{listing file=foo.py}\n", + "After. Next.\n", + ); + let regions = LatexParser::default().parse(input); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Structure(s) if s.contains(r"\tcbinputlisting{listing file=foo.py}") + )), + "tcbinputlisting must stay one Structure command, got: {regions:?}" + ); + assert!( + !regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains(r"\tcbinputlisting{listing file=foo.py}") + )), + "tcbinputlisting must not leak into Prose, got: {regions:?}" + ); + assert!( + regions.iter().any(|r| matches!( + r, + Region::Prose(p) if p.contains("After.") && p.contains("Next.") + )), + "After. / Next. must stay Prose, got: {regions:?}" + ); + let out = format_text(input, &latex_cfg()).unwrap(); + assert!( + out.contains("\\tcbinputlisting{listing file=foo.py}\n"), + "tcbinputlisting must stay one atomic command, got:\n{out}" + ); + assert!( + !out.contains("\\tcbinputlisting{listing file=foo.py} After."), + "following flush prose must not join the command line, got:\n{out}" + ); + assert!( + out.contains("Before.\nNext."), + "prose before tcbinputlisting must still split, got:\n{out}" + ); + assert!( + out.contains("After.\nNext."), + "prose after tcbinputlisting must still split, got:\n{out}" + ); + assert_eq!(format_text(&out, &latex_cfg()).unwrap(), out); + + let tcbout = concat!( + "\\begin{tcboutputlisting}\n", + "First line. Second line.\n", + "\\end{tcboutputlisting}\n", + "After the block. Next.\n", + ); + let tcbout_out = format_text(tcbout, &latex_cfg()).unwrap(); + assert!( + tcbout_out.contains( + "\\begin{tcboutputlisting}\nFirst line. Second line.\n\\end{tcboutputlisting}" + ), + "tcboutputlisting must stay a code env, got:\n{tcbout_out}" + ); + assert!( + tcbout_out.contains("After the block.\nNext."), + "prose after tcboutputlisting must still split, got:\n{tcbout_out}" + ); + + let lst = concat!( + "Before. Next.\n", + "\\lstinputlisting{foo.py}\n", + "After. Next.\n", + ); + let lst_out = format_text(lst, &latex_cfg()).unwrap(); + assert!( + lst_out.contains("\\lstinputlisting{foo.py}\n"), + "lstinputlisting must stay one atomic command, got:\n{lst_out}" + ); + assert!( + !lst_out.contains("\\lstinputlisting{foo.py} After."), + "lstinputlisting must not join following prose, got:\n{lst_out}" + ); + assert!( + lst_out.contains("After.\nNext."), + "prose after lstinputlisting must still split, got:\n{lst_out}" + ); + + let minted = concat!( + "Before. Next.\n", + "\\inputminted{python}{foo.py}\n", + "After. Next.\n", + ); + let minted_cmd = format_text(minted, &latex_cfg()).unwrap(); + assert!( + minted_cmd.contains("\\inputminted{python}{foo.py}\n"), + "inputminted must stay one atomic command, got:\n{minted_cmd}" + ); + assert!( + !minted_cmd.contains("\\inputminted{python}{foo.py} After."), + "inputminted must not join following prose, got:\n{minted_cmd}" + ); + assert!( + minted_cmd.contains("After.\nNext."), + "prose after inputminted must still split, got:\n{minted_cmd}" + ); +} + /// Ticket fixture (GitHub #245): `\mintinline{python}|a.b! c|` is one /// token; following `Next sentence.` still splits. `{lang}{body}` and /// `\mint` are the same class.