diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs index 9a08b03..2b5b2f0 100644 --- a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -217,6 +217,32 @@ impl InvokeXml { } } +/// Every call in a tag body that *is* invoke XML — `…`, what `DeepSeek` V4 writes when told to +/// call tools inside `` tags. Empty unless the body opens with a +/// named invoke, so an invoke quoted inside some other body (a JSON string, +/// say) is not executed. Once the body does open with one, every later +/// invoke in it is decoded too, exactly as the same text outside a tag is. +pub(crate) fn decode_body(body: &str) -> Vec { + let body = body.trim_start(); + if OPEN_RE + .as_ref() + .and_then(|re| re.find(body)) + .is_none_or(|m| m.start() != 0) + { + return Vec::new(); + } + let mut calls = Vec::new(); + let mut from = 0; + while let Probe::Found(block) = InvokeXml::probe_decided(body, from, ScanMode::Batch) { + if let Decoded::Calls(found) = block.decoded { + calls.extend(found); + } + from = block.end; + } + calls +} + /// Whether a wrapper tag is a closer or carries a DSML / namespace prefix — /// either is unambiguous protocol furniture even with no invoke in sight. fn is_closer_or_prefixed(tag: &str) -> bool { diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index f189894..95dd933 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -101,6 +101,28 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize .map(|m| (m.start(), m.end())) } +/// An `` opener, bare or named, optionally DSML-prefixed — the only +/// invoke spellings a fence line may carry and still count as a call. No +/// `` and no XML namespace: ```` ``` ```` +/// is code, not a call. +static FENCE_INVOKE_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?i)^<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*)?invoke(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*)?\s*>"#, + ) + .ok() +}); + +/// Whether `info` — a fence's info string — opens with a complete call tag: +/// a tag-family opener or an `` ([`FENCE_INVOKE_RE`]). `DeepSeek` V4 +/// writes ```` ``` ````, so such a fence is a call, not an example. +pub(crate) fn opens_with_call_tag(info: &str) -> bool { + let tag = TAG_RE + .as_ref() + .and_then(|re| re.find(info)) + .is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str())); + tag || FENCE_INVOKE_RE.as_ref().is_some_and(|re| re.is_match(info)) +} + /// Finds the closer that has the exact prefix and spelling of `opener`. /// /// A bare `` must not be closed by `` embedded in its @@ -590,6 +612,11 @@ pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec ````), +//! is a call, not an example, and is handled by the grammars; //! * a fence with **no** language tag is not protected. Small models wrap a //! genuine call in a bare fence far more often than they quote one, and a //! quoted example almost always carries a language. @@ -71,7 +72,8 @@ fn scan_fences(text: &str) -> (Vec>, Option) { let language = info.split_whitespace().next().unwrap_or(""); let is_tool_call = TOOL_CALL_LANGUAGES .iter() - .any(|lang| lang.eq_ignore_ascii_case(language)); + .any(|lang| lang.eq_ignore_ascii_case(language)) + || super::grammar::tagged::opens_with_call_tag(info); if !language.is_empty() && !is_tool_call { open = Some((line_start, fence_char, fence_len)); } diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs index 66e26e9..930c629 100644 --- a/crates/tinytools-agent/src/parse/test/engine.rs +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -43,6 +43,61 @@ fn fence_ranges_cover_languages_and_unclosed_fences() { assert!(fence_ranges("```tool_call\n{}\n```").is_empty()); } +// ── A call tag on the fence line itself ──────────────────────────────────── +// +// `DeepSeek` V4 Flash wrote ```` ``` ```` — the opener as the info +// string — and never closed the fence. The info string read as a language, +// so the unclosed fence protected the call to end of text as an example. + +#[test] +fn an_unclosed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = concat!( + "\n- [ ] find the tool\n\n\n", + "```\n\n", + "list repositories\n", + "\n" + ); + let outcome = parse_known(text, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn a_fence_whose_info_string_is_a_named_invoke_is_a_call() { + let text = + "```\nrepos\n"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!(calls[0].name, "tool_search"); +} + +#[test] +fn a_closed_fence_whose_info_string_is_a_call_tag_is_a_call() { + let text = "```\n\nrepos\n\n\n```"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1, "{calls:?}"); +} + +#[test] +fn a_language_fence_still_protects_a_call_tag_example() { + let example = "\nrm -rf /\n"; + for text in [ + format!("```xml\n{example}"), + format!("```xml\n{example}"), + format!("```text \n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), + format!("```\n{example}"), + ] { + let (_, calls) = parse(&text); + assert!(calls.is_empty(), "{text:?} dispatched {calls:?}"); + } +} + #[test] fn names_are_repaired_against_known_tools() { let outcome = parse_known( diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index e587eac..22ccd91 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -739,3 +739,88 @@ fn recovery_does_not_execute_a_named_invoke_inside_malformed_json() { let (_, calls) = parse(raw); assert!(calls.is_empty(), "{calls:?}"); } + +// ── Invoke XML inside the tag ─────────────────────────────────────────────── +// +// Told to call tools "inside tags", `DeepSeek` V4 writes its +// native invoke XML there. The tag claimed the block and found no JSON, so +// the call was dropped as malformed even though the same invoke parses bare. + +#[test] +fn a_named_invoke_wrapped_in_a_tool_call_tag_is_decoded() { + let raw = "Searching.\n\n\nrepos\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Searching."); +} + +#[test] +fn a_wrapped_invoke_with_string_attributes_is_decoded() { + let raw = concat!( + "\n\n", + "repos\n", + "5\n", + "\n" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos", "limit": 5}) + ); +} + +/// A `` block, a line of narration, then the wrapped invoke inside a +/// closed bare fence (no info string, so not protected). +#[test] +fn a_todo_block_then_a_closed_bare_fenced_wrapped_invoke_is_decoded() { + let raw = concat!( + "\n- [x] read the request\n- [ ] find the tool\n\n\n", + "Let me find the right tool.\n\n", + "```\n\n\n", + "list repositories\n", + "\n\n```" + ); + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "list repositories"}) + ); +} + +#[test] +fn an_invoke_after_other_body_text_in_the_tag_is_not_decoded() { + let raw = "see ls"; + let (_, calls) = parse(raw); + assert!(calls.is_empty(), "{calls:?}"); +} + +#[test] +fn an_invoke_quoted_in_a_closed_json_tag_body_is_not_executed() { + let raw = "{\"name\":\"tool_search\",\"arguments\":{\"query\":\"rm -rf /\"}}"; + let (_, calls) = parse(raw); + assert!(calls.iter().all(|c| c.name != "shell"), "{calls:?}"); +} + +/// Qwen3-Coder's native format inside the tag. +#[test] +fn a_function_equals_body_in_a_tool_call_tag_is_decoded() { + let raw = "Checking.\n\n\n\nrepos\n\n\n"; + let outcome = super::parse_known(raw, &["tool_search"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "tool_search"); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({"query": "repos"}) + ); + assert_eq!(outcome.calls[0].source, CallSource::InvokeXml); + assert_eq!(outcome.text, "Checking."); +}