From fdb223f14251dffbd9f163f77da32c031252348a Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 16:33:23 +0530 Subject: [PATCH 1/6] fix(agent): decode a named invoke wrapped in a tool_call tag Told to call tools inside tags, DeepSeek V4 writes its native invoke XML there. The tagged grammar claimed the block, found no JSON body, and dropped the call as malformed, though the same invoke parses bare. When a tag body opens with a named invoke, decode it with invoke_xml. The anchor keeps an invoke quoted inside other body text from executing. Refs tinyhumansai/openhuman#6722 --- .../src/parse/grammar/invoke_xml.rs | 25 ++++++++ .../src/parse/grammar/tagged.rs | 5 ++ .../tinytools-agent/src/parse/test/tagged.rs | 63 +++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs index 9a08b03..f2beabe 100644 --- a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -217,6 +217,31 @@ impl InvokeXml { } } +/// Every call in a tag body that *is* invoke XML — `…`, what `DeepSeek` V4 writes when told to +/// call tools inside `` tags. Empty unless the body opens with a +/// named invoke, so an invoke quoted inside some other body (a JSON string, +/// say) is never executed. +pub(crate) fn decode_body(body: &str) -> Vec { + let body = body.trim_start(); + if OPEN_RE + .as_ref() + .and_then(|re| re.find(body)) + .is_none_or(|m| m.start() != 0) + { + return Vec::new(); + } + let mut calls = Vec::new(); + let mut from = 0; + while let Probe::Found(block) = InvokeXml::probe_decided(body, from, ScanMode::Batch) { + if let Decoded::Calls(found) = block.decoded { + calls.extend(found); + } + from = block.end; + } + calls +} + /// Whether a wrapper tag is a closer or carries a DSML / namespace prefix — /// either is unambiguous protocol furniture even with no invoke in sight. fn is_closer_or_prefixed(tag: &str) -> bool { diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index f189894..bf2154e 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -590,6 +590,11 @@ pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec tags", `DeepSeek` V4 writes its +// native invoke XML there. The tag claimed the block and found no JSON, so +// the call was dropped as malformed even though the same invoke parses bare. + +#[test] +fn a_named_invoke_wrapped_in_a_tool_call_tag_is_decoded() { + let raw = "Searching.\n\n\nrepos\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Searching."); +} + +#[test] +fn a_wrapped_invoke_with_string_attributes_is_decoded() { + let raw = concat!( + "\n\n", + "repos\n", + "5\n", + "\n" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos", "limit": 5}) + ); +} + +/// A `` block, a line of narration, then the wrapped invoke inside a +/// closed bare fence (no info string, so not protected). +#[test] +fn a_todo_block_then_a_closed_bare_fenced_wrapped_invoke_is_decoded() { + let raw = concat!( + "\n- [x] read the request\n- [ ] find the tool\n\n\n", + "Let me find the right tool.\n\n", + "```\n\n\n", + "list repositories\n", + "\n\n```" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn an_invoke_after_other_body_text_in_the_tag_is_not_decoded() { + let raw = "see ls"; + let (_, calls) = parse(raw); + assert!(calls.is_empty(), "{calls:?}"); +} From 32d6eb419d47ed05a3693b14c003d09bcea02b33 Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 17:02:36 +0530 Subject: [PATCH 2/6] test(agent): pin the wrapped-invoke anchor and Qwen3-Coder form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a closed JSON tag body quoting an invoke (fails without the anchor) and a body (dropped on main). Narrows the decode_body doc to what the anchor guarantees: only the first invoke is anchored, as on the bare path. --- .../src/parse/grammar/invoke_xml.rs | 3 ++- .../tinytools-agent/src/parse/test/tagged.rs | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs index f2beabe..2b5b2f0 100644 --- a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -221,7 +221,8 @@ impl InvokeXml { /// name="x">…`, what `DeepSeek` V4 writes when told to /// call tools inside `` tags. Empty unless the body opens with a /// named invoke, so an invoke quoted inside some other body (a JSON string, -/// say) is never executed. +/// say) is not executed. Once the body does open with one, every later +/// invoke in it is decoded too, exactly as the same text outside a tag is. pub(crate) fn decode_body(body: &str) -> Vec { let body = body.trim_start(); if OPEN_RE diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index b3b950f..22ccd91 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -802,3 +802,25 @@ fn an_invoke_after_other_body_text_in_the_tag_is_not_decoded() { let (_, calls) = parse(raw); assert!(calls.is_empty(), "{calls:?}"); } + +#[test] +fn an_invoke_quoted_in_a_closed_json_tag_body_is_not_executed() { + let raw = "{\"name\":\"tool_search\",\"arguments\":{\"query\":\"rm -rf /\"}}"; + let (_, calls) = parse(raw); + assert!(calls.iter().all(|c| c.name != "shell"), "{calls:?}"); +} + +/// Qwen3-Coder's native format inside the tag. +#[test] +fn a_function_equals_body_in_a_tool_call_tag_is_decoded() { + let raw = "Checking.\n\n\n\nrepos\n\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Checking."); +} From 396e15c58dba864f968df49e4c9b593c55e7cff9 Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 16:43:46 +0530 Subject: [PATCH 3/6] fix(agent): treat a fence whose info string is a call tag as a call DeepSeek V4 Flash wrote the opener as the fence info string, ```, and never closed the fence. The info string read as a language, so the unclosed fence protected the call to end of text as an example and it was dropped. A fence whose info string opens with a complete call tag (tag-family opener, named invoke, bare ) is now a call fence. A language-tagged fence still protects its contents. Refs tinyhumansai/openhuman#6722 --- .../src/parse/grammar/tagged.rs | 12 +++++ crates/tinytools-agent/src/parse/protected.rs | 8 +-- .../tinytools-agent/src/parse/test/engine.rs | 52 +++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index bf2154e..b09f845 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -101,6 +101,18 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize .map(|m| (m.start(), m.end())) } +/// Whether `info` — a fence's info string — opens with a complete call tag: +/// a tag-family opener, a named invoke, or the bare ``. `DeepSeek` V4 +/// writes ```` ``` ````, so such a fence is a call, not an example. +pub(crate) fn opens_with_call_tag(info: &str) -> bool { + let at_start = |re: &LazyLock>| find_re(re, info).is_some_and(|(s, _)| s == 0); + let tag = TAG_RE + .as_ref() + .and_then(|re| re.find(info)) + .is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str())); + tag || at_start(&NAMED_INVOKE_OPEN_RE) || at_start(&BARE_INVOKE_OPEN_RE) +} + /// Finds the closer that has the exact prefix and spelling of `opener`. /// /// A bare `` must not be closed by `` embedded in its diff --git a/crates/tinytools-agent/src/parse/protected.rs b/crates/tinytools-agent/src/parse/protected.rs index 6673b9b..2189fdd 100644 --- a/crates/tinytools-agent/src/parse/protected.rs +++ b/crates/tinytools-agent/src/parse/protected.rs @@ -8,8 +8,9 @@ //! //! Two deliberate exceptions keep real calls parseable: //! -//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````) -//! is a call, not an example, and is handled by the tagged grammar; +//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````), +//! or whose info string opens with a call tag (```` ``` ````), +//! is a call, not an example, and is handled by the grammars; //! * a fence with **no** language tag is not protected. Small models wrap a //! genuine call in a bare fence far more often than they quote one, and a //! quoted example almost always carries a language. @@ -71,7 +72,8 @@ fn scan_fences(text: &str) -> (Vec>, Option) { let language = info.split_whitespace().next().unwrap_or(""); let is_tool_call = TOOL_CALL_LANGUAGES .iter() - .any(|lang| lang.eq_ignore_ascii_case(language)); + .any(|lang| lang.eq_ignore_ascii_case(language)) + || super::grammar::tagged::opens_with_call_tag(info); if !language.is_empty() && !is_tool_call { open = Some((line_start, fence_char, fence_len)); } diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs index 66e26e9..e6b60c8 100644 --- a/crates/tinytools-agent/src/parse/test/engine.rs +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -43,6 +43,58 @@ fn fence_ranges_cover_languages_and_unclosed_fences() { assert!(fence_ranges("```tool_call\n{}\n```").is_empty()); } +// ── A call tag on the fence line itself ──────────────────────────────────── +// +// `DeepSeek` V4 Flash wrote ```` ``` ```` — the opener as the info +// string — and never closed the fence. The info string read as a language, +// so the unclosed fence protected the call to end of text as an example. + +#[test] +fn an_unclosed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = concat!( + "\n- [ ] find the tool\n\n\n", + "```\n\n", + "list repositories\n", + "\n" + ); + let outcome = parse_known(text, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn a_fence_whose_info_string_is_a_named_invoke_is_a_call() { + let text = + "```\nrepos\n"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "tool_search"); +} + +#[test] +fn a_closed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = "```\n\nrepos\n\n\n```"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); +} + +#[test] +fn a_language_fence_still_protects_a_call_tag_example() { + let example = "\nrm -rf /\n"; + for text in [ + format!("```xml\n{example}"), + format!("```xml\n{example}"), + format!("```text \n{example}"), + ] { + let (_, calls) = parse(&text); + assert!(calls.is_empty(), "{text:?} dispatched {calls:?}"); + } +} + #[test] fn names_are_repaired_against_known_tools() { let outcome = parse_known( From cc7107b0d51830b8e0729f281ab8cf07ec8be24d Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 17:03:42 +0530 Subject: [PATCH 4/6] fix(agent): limit fence call tags to tool_call and invoke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit opens_with_call_tag reused NAMED_INVOKE_OPEN_RE, which also accepts and any XML namespace, so an XSLT fence such as ``` became a call. Only a tag-family opener or an (optionally DSML-prefixed) now marks a call fence. --- .../tinytools-agent/src/parse/grammar/tagged.rs | 16 +++++++++++++--- crates/tinytools-agent/src/parse/test/engine.rs | 3 +++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index b09f845..95dd933 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -101,16 +101,26 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize .map(|m| (m.start(), m.end())) } +/// An `` opener, bare or named, optionally DSML-prefixed — the only +/// invoke spellings a fence line may carry and still count as a call. No +/// `` and no XML namespace: ```` ``` ```` +/// is code, not a call. +static FENCE_INVOKE_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?i)^<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*)?invoke(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*)?\s*>"#, + ) + .ok() +}); + /// Whether `info` — a fence's info string — opens with a complete call tag: -/// a tag-family opener, a named invoke, or the bare ``. `DeepSeek` V4 +/// a tag-family opener or an `` ([`FENCE_INVOKE_RE`]). `DeepSeek` V4 /// writes ```` ``` ````, so such a fence is a call, not an example. pub(crate) fn opens_with_call_tag(info: &str) -> bool { - let at_start = |re: &LazyLock>| find_re(re, info).is_some_and(|(s, _)| s == 0); let tag = TAG_RE .as_ref() .and_then(|re| re.find(info)) .is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str())); - tag || at_start(&NAMED_INVOKE_OPEN_RE) || at_start(&BARE_INVOKE_OPEN_RE) + tag || FENCE_INVOKE_RE.as_ref().is_some_and(|re| re.is_match(info)) } /// Finds the closer that has the exact prefix and spelling of `opener`. diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs index e6b60c8..930c629 100644 --- a/crates/tinytools-agent/src/parse/test/engine.rs +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -89,6 +89,9 @@ fn a_language_fence_still_protects_a_call_tag_example() { format!("```xml\n{example}"), format!("```xml\n{example}"), format!("```text \n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), ] { let (_, calls) = parse(&text); assert!(calls.is_empty(), "{text:?} dispatched {calls:?}"); From 5f7161d1a6943f39a1e58ade8f1b0321788cda4f Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 17:01:12 +0530 Subject: [PATCH 5/6] fix(agent): decode element-form calls for offered tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeepSeek V4 Flash writes some calls as plain elements under the Python code dialect: […]. No grammar read the form, so the call was dropped and the turn ended on narration. A new registry-gated grammar claims … when NAME is an offered tool with a registry entry and the body is child elements only. It decodes to a call when every child is a parameter and every JSON- looking value parses, and reports a MalformedBlock otherwise. Anything else stays in the text. In a stream, a partial opener of an eligible tool is held back so the markup is not released as text. Refs tinyhumansai/openhuman#6722 --- .../src/parse/grammar/element.rs | 194 ++++++++++++++++++ .../src/parse/grammar/invoke_xml.rs | 2 +- .../tinytools-agent/src/parse/grammar/mod.rs | 2 + .../tinytools-agent/src/parse/test/element.rs | 176 ++++++++++++++++ crates/tinytools-agent/src/parse/test/mod.rs | 1 + crates/tinytools-agent/src/types.rs | 2 + 6 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 crates/tinytools-agent/src/parse/grammar/element.rs create mode 100644 crates/tinytools-agent/src/parse/test/element.rs diff --git a/crates/tinytools-agent/src/parse/grammar/element.rs b/crates/tinytools-agent/src/parse/grammar/element.rs new file mode 100644 index 0000000..f1e1479 --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/element.rs @@ -0,0 +1,194 @@ +//! `value` — a call written as plain XML +//! elements: the tool name as the tag, each parameter as a child. +//! +//! `DeepSeek` V4 Flash writes `todo` calls this way under the Python code +//! dialect, `\n\n[{…}]\n\n`, alongside proper +//! `` blocks. Any tag could be a tool name, so the grammar is +//! gated hard to keep prose markup from dispatching: +//! +//! * the tag is an offered tool **with a registry entry** — the registry is +//! the only place parameter names are known; +//! * the matching `` is present; +//! * the body is child elements and whitespace, nothing else. +//! +//! A block passing all three is claimed. It decodes to a call when every +//! child is a parameter of the tool and every `[`/`{` value is valid JSON; +//! otherwise it is malformed, so the caller hears about the dropped call. A +//! block failing the gate is left in the text untouched. +//! +//! ponytail: registry-gated, so the Xml dialect (no registry) never sees an +//! element call; accept known-tool + child-only there if its models start +//! writing the form. + +use std::sync::LazyLock; + +use regex::Regex; + +use super::{Block, Decoded, Grammar, Probe, ScanMode, prefer_pending}; +use crate::pformat::PFormatRegistry; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The element grammar. +#[derive(Debug)] +pub(crate) struct Element; + +/// An attribute-less opening tag. +static OPEN_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<([A-Za-z_][\w.-]*)>").ok()); + +/// Tag names other grammars own; never element calls. +const RESERVED: &[&str] = &[ + "tool_call", + "toolcall", + "tool-call", + "tool_calls", + "function_calls", + "calls", + "invoke", + "function", + "parameter", +]; + +impl Grammar for Element { + fn source(&self) -> CallSource { + CallSource::Element + } + + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let (Some(open_re), Some(registry)) = (OPEN_RE.as_ref(), options.registry) else { + return Probe::None; + }; + let eligible = |name: &str| { + registry.contains_key(name) && options.knows(name) && !RESERVED.contains(&name) + }; + prefer_pending( + Self::probe_decided(text, from, mode, open_re, registry, &eligible), + (mode == ScanMode::Stream) + .then(|| partial_opener(text, from, registry, &eligible)) + .flatten(), + ) + } + + fn openers(&self) -> &'static [&'static str] { + &[] + } +} + +impl Element { + /// The next claimed block at or after `from`. + fn probe_decided( + text: &str, + from: usize, + mode: ScanMode, + open_re: &Regex, + registry: &PFormatRegistry, + eligible: &dyn Fn(&str) -> bool, + ) -> Probe { + for open in open_re.captures_iter(&text[from..]) { + let (Some(tag), Some(name)) = (open.get(0), open.get(1)) else { + continue; + }; + let name = name.as_str(); + if !eligible(name) { + continue; + } + let Some(params) = registry.get(name) else { + continue; + }; + let start = from + tag.start(); + let body_start = from + tag.end(); + let closer = format!(""); + let Some(body_len) = text[body_start..].find(&closer) else { + if mode == ScanMode::Stream { + return Probe::Pending { start }; + } + continue; + }; + let body = &text[body_start..body_start + body_len]; + let Some(children) = children(body) else { + continue; + }; + let decoded = decode(name, ¶ms.names, &children).map_or( + Decoded::Malformed { + body_chars: body.chars().count(), + }, + |call| Decoded::Calls(vec![call]), + ); + return Probe::Found(Block { + start, + end: body_start + body_len + closer.len(), + decoded, + }); + } + Probe::None + } +} + +/// In a stream, a trailing `` arrived. +fn partial_opener( + text: &str, + from: usize, + registry: &PFormatRegistry, + eligible: &dyn Fn(&str) -> bool, +) -> Option { + let start = from + text[from..].rfind('<')?; + let partial = &text[start + 1..]; + if partial.contains('>') { + return None; + } + registry + .keys() + .any(|name| name.starts_with(partial) && eligible(name)) + .then_some(start) +} + +/// `(name, raw value)` for each child element, or `None` when the body holds +/// anything else — prose, an unclosed child — or no child at all. +fn children(body: &str) -> Option> { + let mut out = Vec::new(); + let mut rest = body.trim_start(); + while !rest.is_empty() { + let inner = rest.strip_prefix('<')?; + let name_end = inner.find('>')?; + let name = &inner[..name_end]; + if name.is_empty() + || !name + .chars() + .all(|c| c.is_alphanumeric() || "_.-".contains(c)) + { + return None; + } + let after = &inner[name_end + 1..]; + let closer = format!(""); + let value_end = after.find(&closer)?; + out.push((name, &after[..value_end])); + rest = after[value_end + closer.len()..].trim_start(); + } + (!out.is_empty()).then_some(out) +} + +/// The call, or `None` when a child is not a parameter or a JSON-looking +/// value does not parse. +fn decode(name: &str, params: &[String], children: &[(&str, &str)]) -> Option { + let mut arguments = serde_json::Map::new(); + for (key, raw) in children { + if !params.iter().any(|param| param == key) { + return None; + } + let trimmed = raw.trim(); + let value = if trimmed.starts_with(['[', '{']) { + serde_json::from_str(trimmed).ok()? + } else { + super::invoke_xml::scalar_value(trimmed) + }; + arguments.insert((*key).to_string(), value); + } + Some(ParsedToolCall::new( + name, + serde_json::Value::Object(arguments), + CallSource::Element, + )) +} diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs index 2b5b2f0..8445e1d 100644 --- a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -297,7 +297,7 @@ fn decode_arguments(body: &str) -> serde_json::Value { /// A parameter value: JSON when it parses as a number, bool, null, array or /// object; otherwise the trimmed text. -fn scalar_value(raw: &str) -> serde_json::Value { +pub(crate) fn scalar_value(raw: &str) -> serde_json::Value { let trimmed = raw.trim(); match serde_json::from_str::(trimmed) { Ok( diff --git a/crates/tinytools-agent/src/parse/grammar/mod.rs b/crates/tinytools-agent/src/parse/grammar/mod.rs index 3b996cb..0d37d01 100644 --- a/crates/tinytools-agent/src/parse/grammar/mod.rs +++ b/crates/tinytools-agent/src/parse/grammar/mod.rs @@ -12,6 +12,7 @@ //! at the same byte; the engine otherwise takes the earliest opener. pub(crate) mod bare_json; +pub(crate) mod element; pub(crate) mod glm; pub(crate) mod harmony; pub(crate) mod invoke_xml; @@ -97,6 +98,7 @@ pub(crate) static GRAMMARS: &[&dyn Grammar] = &[ &harmony::Harmony, &mistral::Mistral, &tagged::Tagged, + &element::Element, ]; /// Every opener prefix across all scan grammars. diff --git a/crates/tinytools-agent/src/parse/test/element.rs b/crates/tinytools-agent/src/parse/test/element.rs new file mode 100644 index 0000000..5026d1f --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/element.rs @@ -0,0 +1,176 @@ +//! `value` element calls. + +use crate::stream::StreamScrubber; +use crate::types::{CallSource, ParseDiagnostic, ParseOptions, ParseOutcome}; +use crate::{PFormatRegistry, build_registry}; +use std::sync::Arc; + +const TODO: &str = "\n\n[{\"status\": \"in_progress\", \"description\": \"step one\"}, {\"status\": \"pending\", \"description\": \"step two\"}]\n\n"; + +fn registry() -> PFormatRegistry { + build_registry([ + ( + "todo", + serde_json::json!({"type": "object", "properties": {"todos": {"type": "array"}}}), + ), + ( + "tool_search", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + ), + ]) +} + +fn parse_with(text: &str, known: &[&str], registry: Option<&PFormatRegistry>) -> ParseOutcome { + let known: Vec = known.iter().map(ToString::to_string).collect(); + let mut options = ParseOptions::new().with_known_tools(&known); + if let Some(registry) = registry { + options = options.with_registry(registry); + } + crate::parse::parse_text(text, &options) +} + +fn parse(text: &str) -> ParseOutcome { + parse_with(text, &["todo", "tool_search"], Some(®istry())) +} + +fn malformed(outcome: &ParseOutcome) -> usize { + outcome + .diagnostics + .iter() + .filter(|d| { + matches!( + d, + ParseDiagnostic::MalformedBlock { + source: CallSource::Element, + .. + } + ) + }) + .count() +} + +#[test] +fn a_todo_element_call_is_decoded() { + let outcome = parse(&format!("Planning.\n{TODO}")); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "todo"); + assert_eq!(outcome.calls[0].source, CallSource::Element); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"todos": [ + {"status": "in_progress", "description": "step one"}, + {"status": "pending", "description": "step two"} + ]}) + ); + assert_eq!(outcome.text, "Planning."); +} + +#[test] +fn a_todo_element_then_a_fenced_wrapped_invoke_yields_both_in_order() { + let text = format!( + "{TODO}\n```\n\nrepos\n\n" + ); + let outcome = parse(&text); + let names: Vec<&str> = outcome.calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["todo", "tool_search"]); +} + +#[test] +fn an_element_for_an_unoffered_tool_is_left_alone() { + let outcome = parse_with(TODO, &["tool_search"], Some(®istry())); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(outcome.text, TODO); +} + +#[test] +fn an_element_without_a_registry_is_left_alone() { + let outcome = parse_with(TODO, &["todo"], None); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(outcome.text, TODO); +} + +#[test] +fn ordinary_markup_is_not_a_call() { + let outcome = parse("

x

"); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert!(outcome.diagnostics.is_empty()); + assert_eq!(outcome.text, "

x

"); +} + +#[test] +fn prose_inside_a_known_tool_tag_is_left_alone() { + let text = "remember to ship it"; + let outcome = parse(text); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert!(outcome.diagnostics.is_empty()); + assert_eq!(outcome.text, text); +} + +#[test] +fn a_language_fence_protects_an_element_example() { + let outcome = parse(&format!("```xml\n{TODO}\n```")); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); +} + +#[test] +fn a_child_that_is_not_a_parameter_is_malformed() { + let outcome = parse("ship"); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(malformed(&outcome), 1, "{:?}", outcome.diagnostics); +} + +#[test] +fn undecodable_json_in_a_claimed_element_is_malformed_in_batch_and_stream() { + let text = "\n\n[{\"status\": \"pending\", \n\n"; + let outcome = parse(text); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(malformed(&outcome), 1, "{:?}", outcome.diagnostics); + + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut diagnostics, mut shown) = (Vec::new(), Vec::new(), String::new()); + for chunk in text.as_bytes().chunks(7) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + diagnostics.extend(step.diagnostics); + shown.push_str(&step.text); + } + let step = scrubber.flush(); + calls.extend(step.calls); + diagnostics.extend(step.diagnostics); + shown.push_str(&step.text); + assert!(calls.is_empty(), "{calls:?}"); + let stream_malformed = diagnostics + .iter() + .filter(|d| { + matches!( + d, + ParseDiagnostic::MalformedBlock { + source: CallSource::Element, + .. + } + ) + }) + .count(); + assert_eq!(stream_malformed, 1, "{diagnostics:?}"); + assert!(!shown.contains(""), "{shown:?}"); +} + +#[test] +fn a_streamed_todo_element_yields_one_call_and_no_markup() { + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut shown) = (Vec::new(), String::new()); + for chunk in TODO.as_bytes().chunks(5) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + shown.push_str(&step.text); + } + let step = scrubber.flush(); + calls.extend(step.calls); + shown.push_str(&step.text); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert!(!shown.contains("todos"), "{shown:?}"); +} diff --git a/crates/tinytools-agent/src/parse/test/mod.rs b/crates/tinytools-agent/src/parse/test/mod.rs index da1751d..747510c 100644 --- a/crates/tinytools-agent/src/parse/test/mod.rs +++ b/crates/tinytools-agent/src/parse/test/mod.rs @@ -2,6 +2,7 @@ #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] mod bare_json; +mod element; mod engine; mod glm; mod harmony_mistral; diff --git a/crates/tinytools-agent/src/types.rs b/crates/tinytools-agent/src/types.rs index 7d61500..124b5e4 100644 --- a/crates/tinytools-agent/src/types.rs +++ b/crates/tinytools-agent/src/types.rs @@ -36,6 +36,8 @@ pub enum CallSource { PFormat, /// A code-style call `name(arg="value")` inside a tag, registry-gated. Code, + /// `value` elements, registry-gated. + Element, } /// One model-requested tool invocation recovered from text or structured data. From 10622c4fd911503a5ff61baa76411e608d8eb917 Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 28 Sep 2026 17:18:50 +0530 Subject: [PATCH 6/6] fix(agent): don't stall the stream on an unclaimable element MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An element opener with no closer held the whole rest of the stream until flush. Hold only while the body is still a viable child-only prefix, so a mis-closed … releases text live. Reserved names (tool_call, invoke, …) are no longer parameter children, so a malformed element wrapping a real leaves that call to its own grammar instead of swallowing it. --- .../src/parse/grammar/element.rs | 51 ++++++++++++++++--- .../tinytools-agent/src/parse/test/element.rs | 36 +++++++++++++ 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/element.rs b/crates/tinytools-agent/src/parse/grammar/element.rs index f1e1479..6108c6e 100644 --- a/crates/tinytools-agent/src/parse/grammar/element.rs +++ b/crates/tinytools-agent/src/parse/grammar/element.rs @@ -99,7 +99,10 @@ impl Element { let body_start = from + tag.end(); let closer = format!(""); let Some(body_len) = text[body_start..].find(&closer) else { - if mode == ScanMode::Stream { + // Hold only while more input could still complete a claimable + // block; a body that already cannot be one (a mis-closed + // ``, prose) must not stall the stream. + if mode == ScanMode::Stream && viable_prefix(&text[body_start..], &closer) { return Probe::Pending { start }; } continue; @@ -145,6 +148,46 @@ fn partial_opener( .then_some(start) } +/// Whether `name` can be a parameter child. A reserved name (`tool_call`, +/// `invoke`, …) is another grammar's block: claiming it would swallow a real +/// call nested inside a malformed element. +fn is_child_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_alphanumeric() || "_.-".contains(c)) + && !RESERVED + .iter() + .any(|reserved| reserved.eq_ignore_ascii_case(name)) +} + +/// Whether `rest` — a body whose outer `closer` has not arrived — can still +/// grow into a claimable one: complete children, then at most one partial +/// child or a partial `closer`. +fn viable_prefix(rest: &str, closer: &str) -> bool { + let mut rest = rest.trim_start(); + loop { + let Some(inner) = rest.strip_prefix('<') else { + return rest.is_empty(); + }; + if inner.starts_with('/') { + return closer[1..].starts_with(inner); + } + let Some(name_end) = inner.find('>') else { + return inner.is_empty() || is_child_name(inner); + }; + let name = &inner[..name_end]; + if !is_child_name(name) { + return false; + } + let after = &inner[name_end + 1..]; + let Some(value_end) = after.find(&format!("")) else { + return true; + }; + rest = after[value_end + name.len() + 3..].trim_start(); + } +} + /// `(name, raw value)` for each child element, or `None` when the body holds /// anything else — prose, an unclosed child — or no child at all. fn children(body: &str) -> Option> { @@ -154,11 +197,7 @@ fn children(body: &str) -> Option> { let inner = rest.strip_prefix('<')?; let name_end = inner.find('>')?; let name = &inner[..name_end]; - if name.is_empty() - || !name - .chars() - .all(|c| c.is_alphanumeric() || "_.-".contains(c)) - { + if !is_child_name(name) { return None; } let after = &inner[name_end + 1..]; diff --git a/crates/tinytools-agent/src/parse/test/element.rs b/crates/tinytools-agent/src/parse/test/element.rs index 5026d1f..3e20005 100644 --- a/crates/tinytools-agent/src/parse/test/element.rs +++ b/crates/tinytools-agent/src/parse/test/element.rs @@ -174,3 +174,39 @@ fn a_streamed_todo_element_yields_one_call_and_no_markup() { assert_eq!(calls.len(), 1, "{calls:?}"); assert!(!shown.contains("todos"), "{shown:?}"); } + +/// Sanitized shape of a real turn: a todo element the model closed with +/// `` instead of ``, narration, then a fenced +/// wrapped invoke. The element is not claimable, so the stream must not +/// hold everything after `` until flush. +#[test] +fn a_mis_closed_todo_element_does_not_stall_the_stream() { + let text = concat!( + "\n\n[{\"status\": \"pending\", \"description\": \"step one\"}]\n", + "\n\nNow searching the repositories.\n", + "```\n\n", + "repos\n\n" + ); + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into(), "tool_search".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut live) = (Vec::new(), String::new()); + for chunk in text.as_bytes().chunks(7) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + live.push_str(&step.text); + } + assert!(live.contains("Now searching"), "released live: {live:?}"); + calls.extend(scrubber.flush().calls); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["tool_search"]); +} + +#[test] +fn a_call_inside_a_malformed_element_survives() { + let text = "\n\nrepos\n\n"; + let outcome = parse(text); + let names: Vec<&str> = outcome.calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["tool_search"], "{:?}", outcome.diagnostics); + assert_eq!(malformed(&outcome), 0, "{:?}", outcome.diagnostics); +}