diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs
index d7be734..f189894 100644
--- a/crates/tinytools-agent/src/parse/grammar/tagged.rs
+++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs
@@ -230,43 +230,28 @@ impl Grammar for Tagged {
impl Tagged {
/// The next block whose opener is fully present.
fn probe_decided(text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe {
- let Some(opener) = next_opener(text, from) else {
+ let opener = next_opener(text, from);
+ if let Some(noise) = orphan_closer_noise(text, from, opener.as_ref()) {
+ return noise;
+ }
+
+ let Some(opener) = opener else {
return Probe::None;
};
let mut body_start = opener.body_start;
- // How many extra openers a doubled block skipped, so the matching
- // number of extra closers — never an unrelated closing tag such as
- // `` — can be swallowed below. `DeepSeek` V4 doubles both the
- // opener and the closer under a code dialect:
- // `\n\nNAME(...)\n\n`.
- let mut skipped = 0usize;
- let close = match opener.kind {
+ let (close, skipped) = match opener.kind {
OpenerKind::Tag => {
- // Positional pairing means a doubled opener would otherwise
- // close the first tag on an empty body and lose the call. An
- // opener followed by nothing but whitespace is the same
- // block starting again, so the scan moves past it.
- let re = TAG_RE.as_ref();
- loop {
- let after = &text[body_start..];
- let Some(m) = re.and_then(|re| re.find(after)) else {
- break None;
- };
- let is_opener = !is_closing_marker(m.as_str());
- if is_opener && after[..m.start()].trim().is_empty() {
- body_start += m.end();
- skipped += 1;
- continue;
- }
- break Some((m.start(), m.end()));
- }
+ let (close, skipped, new_body_start) =
+ tag_close_skipping_doubled_openers(text, body_start);
+ body_start = new_body_start;
+ (close, skipped)
}
OpenerKind::Invoke => {
let after = &text[body_start..];
- invoke_close(&text[opener.start..body_start], after)
+ (invoke_close(&text[opener.start..body_start], after), 0)
}
- OpenerKind::Fence => fence_close(&text[body_start..]),
+ OpenerKind::Fence => (fence_close(&text[body_start..]), 0),
};
let after = &text[body_start..];
@@ -447,6 +432,65 @@ fn swallow_extra_closers(rest: &str, max: usize, mode: ScanMode) -> Option` opener in its text at all, but sometimes still types a
+/// habitual closer in its narrative; this sweeps it as furniture rather than
+/// leaving literal markup in the visible text.
+fn orphan_closer_noise(text: &str, from: usize, opener: Option<&Opener>) -> Option {
+ let re = TAG_RE.as_ref()?;
+ let hay = &text[from..];
+ let m = re.find(hay)?;
+ if !is_closing_marker(m.as_str()) {
+ return None;
+ }
+ let start = from + m.start();
+ if opener.is_some_and(|o| start >= o.start) {
+ return None;
+ }
+ Some(Probe::Found(Block {
+ start,
+ end: from + m.end(),
+ decoded: Decoded::Noise,
+ }))
+}
+
+/// The closer for a tag opener at `body_start`, skipping past a doubled
+/// opener first, and how many extra openers it skipped — so the matching
+/// number of extra closers, never an unrelated closing tag such as ``,
+/// can be swallowed by the caller. `DeepSeek` V4 doubles both the opener and
+/// the closer under a code dialect:
+/// `\n\nNAME(...)\n\n`.
+///
+/// Positional pairing means a doubled opener would otherwise close the first
+/// tag on an empty body and lose the call. An opener followed by nothing but
+/// whitespace is the same block starting again, so the scan moves past it.
+/// Returns the closer (if any), how many openers it skipped, and the
+/// possibly-advanced body start.
+fn tag_close_skipping_doubled_openers(
+ text: &str,
+ mut body_start: usize,
+) -> (Option<(usize, usize)>, usize, usize) {
+ let re = TAG_RE.as_ref();
+ let mut skipped = 0usize;
+ let close = loop {
+ let after = &text[body_start..];
+ let Some(m) = re.and_then(|re| re.find(after)) else {
+ break None;
+ };
+ let is_opener = !is_closing_marker(m.as_str());
+ if is_opener && after[..m.start()].trim().is_empty() {
+ body_start += m.end();
+ skipped += 1;
+ continue;
+ }
+ break Some((m.start(), m.end()));
+ };
+ (close, skipped, body_start)
+}
+
/// The earliest opener at or after `from`: a non-closing tag-family marker,
/// the bare `` literal, or a fence opener.
fn next_opener(text: &str, from: usize) -> Option {
diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs
index 75294ae..e587eac 100644
--- a/crates/tinytools-agent/src/parse/test/tagged.rs
+++ b/crates/tinytools-agent/src/parse/test/tagged.rs
@@ -465,6 +465,23 @@ fn two_adjacent_blocks_are_still_two_blocks() {
assert_eq!(calls[1].name, "two");
}
+#[test]
+fn an_orphaned_closer_with_no_opener_anywhere_is_removed_not_shown() {
+ // A model that made its actual call over a structured/native channel has
+ // no `` opener anywhere in its text at all, but sometimes still
+ // types a habitual `` in its narrative. `next_opener` never
+ // treats a closing marker as an opener, so nothing pairs with it — it
+ // must not be left in the visible text as literal markup.
+ let (text, calls) = parse(
+ "Let me check the details on the top contenders to find the best one for you.\n\nHere are the results.",
+ );
+ assert_eq!(
+ text,
+ "Let me check the details on the top contenders to find the best one for you.\nHere are the results."
+ );
+ assert!(calls.is_empty());
+}
+
#[test]
fn a_doubled_opener_does_not_swallow_an_unrelated_closing_tag() {
// Only the extra `` a doubled opener leaves behind is