diff --git a/crates/tinytools-agent/src/parse/grammar/element.rs b/crates/tinytools-agent/src/parse/grammar/element.rs new file mode 100644 index 0000000..6108c6e --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/element.rs @@ -0,0 +1,233 @@ +//! `value` — a call written as plain XML +//! elements: the tool name as the tag, each parameter as a child. +//! +//! `DeepSeek` V4 Flash writes `todo` calls this way under the Python code +//! dialect, `\n\n[{…}]\n\n`, alongside proper +//! `` blocks. Any tag could be a tool name, so the grammar is +//! gated hard to keep prose markup from dispatching: +//! +//! * the tag is an offered tool **with a registry entry** — the registry is +//! the only place parameter names are known; +//! * the matching `` is present; +//! * the body is child elements and whitespace, nothing else. +//! +//! A block passing all three is claimed. It decodes to a call when every +//! child is a parameter of the tool and every `[`/`{` value is valid JSON; +//! otherwise it is malformed, so the caller hears about the dropped call. A +//! block failing the gate is left in the text untouched. +//! +//! ponytail: registry-gated, so the Xml dialect (no registry) never sees an +//! element call; accept known-tool + child-only there if its models start +//! writing the form. + +use std::sync::LazyLock; + +use regex::Regex; + +use super::{Block, Decoded, Grammar, Probe, ScanMode, prefer_pending}; +use crate::pformat::PFormatRegistry; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The element grammar. +#[derive(Debug)] +pub(crate) struct Element; + +/// An attribute-less opening tag. +static OPEN_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<([A-Za-z_][\w.-]*)>").ok()); + +/// Tag names other grammars own; never element calls. +const RESERVED: &[&str] = &[ + "tool_call", + "toolcall", + "tool-call", + "tool_calls", + "function_calls", + "calls", + "invoke", + "function", + "parameter", +]; + +impl Grammar for Element { + fn source(&self) -> CallSource { + CallSource::Element + } + + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let (Some(open_re), Some(registry)) = (OPEN_RE.as_ref(), options.registry) else { + return Probe::None; + }; + let eligible = |name: &str| { + registry.contains_key(name) && options.knows(name) && !RESERVED.contains(&name) + }; + prefer_pending( + Self::probe_decided(text, from, mode, open_re, registry, &eligible), + (mode == ScanMode::Stream) + .then(|| partial_opener(text, from, registry, &eligible)) + .flatten(), + ) + } + + fn openers(&self) -> &'static [&'static str] { + &[] + } +} + +impl Element { + /// The next claimed block at or after `from`. + fn probe_decided( + text: &str, + from: usize, + mode: ScanMode, + open_re: &Regex, + registry: &PFormatRegistry, + eligible: &dyn Fn(&str) -> bool, + ) -> Probe { + for open in open_re.captures_iter(&text[from..]) { + let (Some(tag), Some(name)) = (open.get(0), open.get(1)) else { + continue; + }; + let name = name.as_str(); + if !eligible(name) { + continue; + } + let Some(params) = registry.get(name) else { + continue; + }; + let start = from + tag.start(); + let body_start = from + tag.end(); + let closer = format!(""); + let Some(body_len) = text[body_start..].find(&closer) else { + // Hold only while more input could still complete a claimable + // block; a body that already cannot be one (a mis-closed + // ``, prose) must not stall the stream. + if mode == ScanMode::Stream && viable_prefix(&text[body_start..], &closer) { + return Probe::Pending { start }; + } + continue; + }; + let body = &text[body_start..body_start + body_len]; + let Some(children) = children(body) else { + continue; + }; + let decoded = decode(name, ¶ms.names, &children).map_or( + Decoded::Malformed { + body_chars: body.chars().count(), + }, + |call| Decoded::Calls(vec![call]), + ); + return Probe::Found(Block { + start, + end: body_start + body_len + closer.len(), + decoded, + }); + } + Probe::None + } +} + +/// In a stream, a trailing `` arrived. +fn partial_opener( + text: &str, + from: usize, + registry: &PFormatRegistry, + eligible: &dyn Fn(&str) -> bool, +) -> Option { + let start = from + text[from..].rfind('<')?; + let partial = &text[start + 1..]; + if partial.contains('>') { + return None; + } + registry + .keys() + .any(|name| name.starts_with(partial) && eligible(name)) + .then_some(start) +} + +/// Whether `name` can be a parameter child. A reserved name (`tool_call`, +/// `invoke`, …) is another grammar's block: claiming it would swallow a real +/// call nested inside a malformed element. +fn is_child_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_alphanumeric() || "_.-".contains(c)) + && !RESERVED + .iter() + .any(|reserved| reserved.eq_ignore_ascii_case(name)) +} + +/// Whether `rest` — a body whose outer `closer` has not arrived — can still +/// grow into a claimable one: complete children, then at most one partial +/// child or a partial `closer`. +fn viable_prefix(rest: &str, closer: &str) -> bool { + let mut rest = rest.trim_start(); + loop { + let Some(inner) = rest.strip_prefix('<') else { + return rest.is_empty(); + }; + if inner.starts_with('/') { + return closer[1..].starts_with(inner); + } + let Some(name_end) = inner.find('>') else { + return inner.is_empty() || is_child_name(inner); + }; + let name = &inner[..name_end]; + if !is_child_name(name) { + return false; + } + let after = &inner[name_end + 1..]; + let Some(value_end) = after.find(&format!("")) else { + return true; + }; + rest = after[value_end + name.len() + 3..].trim_start(); + } +} + +/// `(name, raw value)` for each child element, or `None` when the body holds +/// anything else — prose, an unclosed child — or no child at all. +fn children(body: &str) -> Option> { + let mut out = Vec::new(); + let mut rest = body.trim_start(); + while !rest.is_empty() { + let inner = rest.strip_prefix('<')?; + let name_end = inner.find('>')?; + let name = &inner[..name_end]; + if !is_child_name(name) { + return None; + } + let after = &inner[name_end + 1..]; + let closer = format!(""); + let value_end = after.find(&closer)?; + out.push((name, &after[..value_end])); + rest = after[value_end + closer.len()..].trim_start(); + } + (!out.is_empty()).then_some(out) +} + +/// The call, or `None` when a child is not a parameter or a JSON-looking +/// value does not parse. +fn decode(name: &str, params: &[String], children: &[(&str, &str)]) -> Option { + let mut arguments = serde_json::Map::new(); + for (key, raw) in children { + if !params.iter().any(|param| param == key) { + return None; + } + let trimmed = raw.trim(); + let value = if trimmed.starts_with(['[', '{']) { + serde_json::from_str(trimmed).ok()? + } else { + super::invoke_xml::scalar_value(trimmed) + }; + arguments.insert((*key).to_string(), value); + } + Some(ParsedToolCall::new( + name, + serde_json::Value::Object(arguments), + CallSource::Element, + )) +} diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs index 9a08b03..8445e1d 100644 --- a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -217,6 +217,32 @@ impl InvokeXml { } } +/// Every call in a tag body that *is* invoke XML — `…`, what `DeepSeek` V4 writes when told to +/// call tools inside `` tags. Empty unless the body opens with a +/// named invoke, so an invoke quoted inside some other body (a JSON string, +/// say) is not executed. Once the body does open with one, every later +/// invoke in it is decoded too, exactly as the same text outside a tag is. +pub(crate) fn decode_body(body: &str) -> Vec { + let body = body.trim_start(); + if OPEN_RE + .as_ref() + .and_then(|re| re.find(body)) + .is_none_or(|m| m.start() != 0) + { + return Vec::new(); + } + let mut calls = Vec::new(); + let mut from = 0; + while let Probe::Found(block) = InvokeXml::probe_decided(body, from, ScanMode::Batch) { + if let Decoded::Calls(found) = block.decoded { + calls.extend(found); + } + from = block.end; + } + calls +} + /// Whether a wrapper tag is a closer or carries a DSML / namespace prefix — /// either is unambiguous protocol furniture even with no invoke in sight. fn is_closer_or_prefixed(tag: &str) -> bool { @@ -271,7 +297,7 @@ fn decode_arguments(body: &str) -> serde_json::Value { /// A parameter value: JSON when it parses as a number, bool, null, array or /// object; otherwise the trimmed text. -fn scalar_value(raw: &str) -> serde_json::Value { +pub(crate) fn scalar_value(raw: &str) -> serde_json::Value { let trimmed = raw.trim(); match serde_json::from_str::(trimmed) { Ok( diff --git a/crates/tinytools-agent/src/parse/grammar/mod.rs b/crates/tinytools-agent/src/parse/grammar/mod.rs index 3b996cb..0d37d01 100644 --- a/crates/tinytools-agent/src/parse/grammar/mod.rs +++ b/crates/tinytools-agent/src/parse/grammar/mod.rs @@ -12,6 +12,7 @@ //! at the same byte; the engine otherwise takes the earliest opener. pub(crate) mod bare_json; +pub(crate) mod element; pub(crate) mod glm; pub(crate) mod harmony; pub(crate) mod invoke_xml; @@ -97,6 +98,7 @@ pub(crate) static GRAMMARS: &[&dyn Grammar] = &[ &harmony::Harmony, &mistral::Mistral, &tagged::Tagged, + &element::Element, ]; /// Every opener prefix across all scan grammars. diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index f189894..95dd933 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -101,6 +101,28 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize .map(|m| (m.start(), m.end())) } +/// An `` opener, bare or named, optionally DSML-prefixed — the only +/// invoke spellings a fence line may carry and still count as a call. No +/// `` and no XML namespace: ```` ``` ```` +/// is code, not a call. +static FENCE_INVOKE_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?i)^<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*)?invoke(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*)?\s*>"#, + ) + .ok() +}); + +/// Whether `info` — a fence's info string — opens with a complete call tag: +/// a tag-family opener or an `` ([`FENCE_INVOKE_RE`]). `DeepSeek` V4 +/// writes ```` ``` ````, so such a fence is a call, not an example. +pub(crate) fn opens_with_call_tag(info: &str) -> bool { + let tag = TAG_RE + .as_ref() + .and_then(|re| re.find(info)) + .is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str())); + tag || FENCE_INVOKE_RE.as_ref().is_some_and(|re| re.is_match(info)) +} + /// Finds the closer that has the exact prefix and spelling of `opener`. /// /// A bare `` must not be closed by `` embedded in its @@ -590,6 +612,11 @@ pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec ````), +//! is a call, not an example, and is handled by the grammars; //! * a fence with **no** language tag is not protected. Small models wrap a //! genuine call in a bare fence far more often than they quote one, and a //! quoted example almost always carries a language. @@ -71,7 +72,8 @@ fn scan_fences(text: &str) -> (Vec>, Option) { let language = info.split_whitespace().next().unwrap_or(""); let is_tool_call = TOOL_CALL_LANGUAGES .iter() - .any(|lang| lang.eq_ignore_ascii_case(language)); + .any(|lang| lang.eq_ignore_ascii_case(language)) + || super::grammar::tagged::opens_with_call_tag(info); if !language.is_empty() && !is_tool_call { open = Some((line_start, fence_char, fence_len)); } diff --git a/crates/tinytools-agent/src/parse/test/element.rs b/crates/tinytools-agent/src/parse/test/element.rs new file mode 100644 index 0000000..3e20005 --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/element.rs @@ -0,0 +1,212 @@ +//! `value` element calls. + +use crate::stream::StreamScrubber; +use crate::types::{CallSource, ParseDiagnostic, ParseOptions, ParseOutcome}; +use crate::{PFormatRegistry, build_registry}; +use std::sync::Arc; + +const TODO: &str = "\n\n[{\"status\": \"in_progress\", \"description\": \"step one\"}, {\"status\": \"pending\", \"description\": \"step two\"}]\n\n"; + +fn registry() -> PFormatRegistry { + build_registry([ + ( + "todo", + serde_json::json!({"type": "object", "properties": {"todos": {"type": "array"}}}), + ), + ( + "tool_search", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + ), + ]) +} + +fn parse_with(text: &str, known: &[&str], registry: Option<&PFormatRegistry>) -> ParseOutcome { + let known: Vec = known.iter().map(ToString::to_string).collect(); + let mut options = ParseOptions::new().with_known_tools(&known); + if let Some(registry) = registry { + options = options.with_registry(registry); + } + crate::parse::parse_text(text, &options) +} + +fn parse(text: &str) -> ParseOutcome { + parse_with(text, &["todo", "tool_search"], Some(®istry())) +} + +fn malformed(outcome: &ParseOutcome) -> usize { + outcome + .diagnostics + .iter() + .filter(|d| { + matches!( + d, + ParseDiagnostic::MalformedBlock { + source: CallSource::Element, + .. + } + ) + }) + .count() +} + +#[test] +fn a_todo_element_call_is_decoded() { + let outcome = parse(&format!("Planning.\n{TODO}")); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "todo"); + assert_eq!(outcome.calls[0].source, CallSource::Element); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"todos": [ + {"status": "in_progress", "description": "step one"}, + {"status": "pending", "description": "step two"} + ]}) + ); + assert_eq!(outcome.text, "Planning."); +} + +#[test] +fn a_todo_element_then_a_fenced_wrapped_invoke_yields_both_in_order() { + let text = format!( + "{TODO}\n```\n\nrepos\n\n" + ); + let outcome = parse(&text); + let names: Vec<&str> = outcome.calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["todo", "tool_search"]); +} + +#[test] +fn an_element_for_an_unoffered_tool_is_left_alone() { + let outcome = parse_with(TODO, &["tool_search"], Some(®istry())); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(outcome.text, TODO); +} + +#[test] +fn an_element_without_a_registry_is_left_alone() { + let outcome = parse_with(TODO, &["todo"], None); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(outcome.text, TODO); +} + +#[test] +fn ordinary_markup_is_not_a_call() { + let outcome = parse("

x

"); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert!(outcome.diagnostics.is_empty()); + assert_eq!(outcome.text, "

x

"); +} + +#[test] +fn prose_inside_a_known_tool_tag_is_left_alone() { + let text = "remember to ship it"; + let outcome = parse(text); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert!(outcome.diagnostics.is_empty()); + assert_eq!(outcome.text, text); +} + +#[test] +fn a_language_fence_protects_an_element_example() { + let outcome = parse(&format!("```xml\n{TODO}\n```")); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); +} + +#[test] +fn a_child_that_is_not_a_parameter_is_malformed() { + let outcome = parse("ship"); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(malformed(&outcome), 1, "{:?}", outcome.diagnostics); +} + +#[test] +fn undecodable_json_in_a_claimed_element_is_malformed_in_batch_and_stream() { + let text = "\n\n[{\"status\": \"pending\", \n\n"; + let outcome = parse(text); + assert!(outcome.calls.is_empty(), "{:?}", outcome.calls); + assert_eq!(malformed(&outcome), 1, "{:?}", outcome.diagnostics); + + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut diagnostics, mut shown) = (Vec::new(), Vec::new(), String::new()); + for chunk in text.as_bytes().chunks(7) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + diagnostics.extend(step.diagnostics); + shown.push_str(&step.text); + } + let step = scrubber.flush(); + calls.extend(step.calls); + diagnostics.extend(step.diagnostics); + shown.push_str(&step.text); + assert!(calls.is_empty(), "{calls:?}"); + let stream_malformed = diagnostics + .iter() + .filter(|d| { + matches!( + d, + ParseDiagnostic::MalformedBlock { + source: CallSource::Element, + .. + } + ) + }) + .count(); + assert_eq!(stream_malformed, 1, "{diagnostics:?}"); + assert!(!shown.contains(""), "{shown:?}"); +} + +#[test] +fn a_streamed_todo_element_yields_one_call_and_no_markup() { + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut shown) = (Vec::new(), String::new()); + for chunk in TODO.as_bytes().chunks(5) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + shown.push_str(&step.text); + } + let step = scrubber.flush(); + calls.extend(step.calls); + shown.push_str(&step.text); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert!(!shown.contains("todos"), "{shown:?}"); +} + +/// Sanitized shape of a real turn: a todo element the model closed with +/// `
` instead of ``, narration, then a fenced +/// wrapped invoke. The element is not claimable, so the stream must not +/// hold everything after `` until flush. +#[test] +fn a_mis_closed_todo_element_does_not_stall_the_stream() { + let text = concat!( + "\n\n[{\"status\": \"pending\", \"description\": \"step one\"}]\n", + "\n
\nNow searching the repositories.\n", + "```\n\n", + "repos\n\n" + ); + let mut scrubber = StreamScrubber::new() + .with_known_tools(vec!["todo".into(), "tool_search".into()]) + .with_registry(Arc::new(registry())); + let (mut calls, mut live) = (Vec::new(), String::new()); + for chunk in text.as_bytes().chunks(7) { + let step = scrubber.feed(std::str::from_utf8(chunk).unwrap_or_default()); + calls.extend(step.calls); + live.push_str(&step.text); + } + assert!(live.contains("Now searching"), "released live: {live:?}"); + calls.extend(scrubber.flush().calls); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["tool_search"]); +} + +#[test] +fn a_call_inside_a_malformed_element_survives() { + let text = "\n\nrepos\n\n"; + let outcome = parse(text); + let names: Vec<&str> = outcome.calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["tool_search"], "{:?}", outcome.diagnostics); + assert_eq!(malformed(&outcome), 0, "{:?}", outcome.diagnostics); +} diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs index 66e26e9..930c629 100644 --- a/crates/tinytools-agent/src/parse/test/engine.rs +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -43,6 +43,61 @@ fn fence_ranges_cover_languages_and_unclosed_fences() { assert!(fence_ranges("```tool_call\n{}\n```").is_empty()); } +// ── A call tag on the fence line itself ──────────────────────────────────── +// +// `DeepSeek` V4 Flash wrote ```` ``` ```` — the opener as the info +// string — and never closed the fence. The info string read as a language, +// so the unclosed fence protected the call to end of text as an example. + +#[test] +fn an_unclosed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = concat!( + "\n- [ ] find the tool\n\n\n", + "```\n\n", + "list repositories\n", + "\n" + ); + let outcome = parse_known(text, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn a_fence_whose_info_string_is_a_named_invoke_is_a_call() { + let text = + "```\nrepos\n"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "tool_search"); +} + +#[test] +fn a_closed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = "```\n\nrepos\n\n\n```"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); +} + +#[test] +fn a_language_fence_still_protects_a_call_tag_example() { + let example = "\nrm -rf /\n"; + for text in [ + format!("```xml\n{example}"), + format!("```xml\n{example}"), + format!("```text \n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), + ] { + let (_, calls) = parse(&text); + assert!(calls.is_empty(), "{text:?} dispatched {calls:?}"); + } +} + #[test] fn names_are_repaired_against_known_tools() { let outcome = parse_known( diff --git a/crates/tinytools-agent/src/parse/test/mod.rs b/crates/tinytools-agent/src/parse/test/mod.rs index da1751d..747510c 100644 --- a/crates/tinytools-agent/src/parse/test/mod.rs +++ b/crates/tinytools-agent/src/parse/test/mod.rs @@ -2,6 +2,7 @@ #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] mod bare_json; +mod element; mod engine; mod glm; mod harmony_mistral; diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index e587eac..22ccd91 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -739,3 +739,88 @@ fn recovery_does_not_execute_a_named_invoke_inside_malformed_json() { let (_, calls) = parse(raw); assert!(calls.is_empty(), "{calls:?}"); } + +// ── Invoke XML inside the tag ─────────────────────────────────────────────── +// +// Told to call tools "inside tags", `DeepSeek` V4 writes its +// native invoke XML there. The tag claimed the block and found no JSON, so +// the call was dropped as malformed even though the same invoke parses bare. + +#[test] +fn a_named_invoke_wrapped_in_a_tool_call_tag_is_decoded() { + let raw = "Searching.\n\n\nrepos\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Searching."); +} + +#[test] +fn a_wrapped_invoke_with_string_attributes_is_decoded() { + let raw = concat!( + "\n\n", + "repos\n", + "5\n", + "\n" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos", "limit": 5}) + ); +} + +/// A `` block, a line of narration, then the wrapped invoke inside a +/// closed bare fence (no info string, so not protected). +#[test] +fn a_todo_block_then_a_closed_bare_fenced_wrapped_invoke_is_decoded() { + let raw = concat!( + "\n- [x] read the request\n- [ ] find the tool\n\n\n", + "Let me find the right tool.\n\n", + "```\n\n\n", + "list repositories\n", + "\n\n```" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn an_invoke_after_other_body_text_in_the_tag_is_not_decoded() { + let raw = "see ls"; + let (_, calls) = parse(raw); + assert!(calls.is_empty(), "{calls:?}"); +} + +#[test] +fn an_invoke_quoted_in_a_closed_json_tag_body_is_not_executed() { + let raw = "{\"name\":\"tool_search\",\"arguments\":{\"query\":\"rm -rf /\"}}"; + let (_, calls) = parse(raw); + assert!(calls.iter().all(|c| c.name != "shell"), "{calls:?}"); +} + +/// Qwen3-Coder's native format inside the tag. +#[test] +fn a_function_equals_body_in_a_tool_call_tag_is_decoded() { + let raw = "Checking.\n\n\n\nrepos\n\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Checking."); +} diff --git a/crates/tinytools-agent/src/types.rs b/crates/tinytools-agent/src/types.rs index 7d61500..124b5e4 100644 --- a/crates/tinytools-agent/src/types.rs +++ b/crates/tinytools-agent/src/types.rs @@ -36,6 +36,8 @@ pub enum CallSource { PFormat, /// A code-style call `name(arg="value")` inside a tag, registry-gated. Code, + /// `value` elements, registry-gated. + Element, } /// One model-requested tool invocation recovered from text or structured data.