Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions crates/tinytools-agent/src/parse/grammar/invoke_xml.rs
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,32 @@ impl InvokeXml {
}
}

/// Every call in a tag body that *is* invoke XML — `<tool_call><invoke
/// name="x">…</invoke></tool_call>`, what `DeepSeek` V4 writes when told to
/// call tools inside `<tool_call>` tags. Empty unless the body opens with a
/// named invoke, so an invoke quoted inside some other body (a JSON string,
/// say) is not executed. Once the body does open with one, every later
/// invoke in it is decoded too, exactly as the same text outside a tag is.
pub(crate) fn decode_body(body: &str) -> Vec<ParsedToolCall> {
let body = body.trim_start();
if OPEN_RE
.as_ref()
.and_then(|re| re.find(body))
.is_none_or(|m| m.start() != 0)
{
return Vec::new();
}
let mut calls = Vec::new();
let mut from = 0;
while let Probe::Found(block) = InvokeXml::probe_decided(body, from, ScanMode::Batch) {
if let Decoded::Calls(found) = block.decoded {
calls.extend(found);
}
from = block.end;
}
calls
}

/// Whether a wrapper tag is a closer or carries a DSML / namespace prefix —
/// either is unambiguous protocol furniture even with no invoke in sight.
fn is_closer_or_prefixed(tag: &str) -> bool {
Expand Down
27 changes: 27 additions & 0 deletions crates/tinytools-agent/src/parse/grammar/tagged.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,28 @@ fn find_re(re: &LazyLock<Option<Regex>>, haystack: &str) -> Option<(usize, usize
.map(|m| (m.start(), m.end()))
}

/// An `<invoke>` opener, bare or named, optionally DSML-prefixed — the only
/// invoke spellings a fence line may carry and still count as a call. No
/// `<function …>` and no XML namespace: ```` ```<xsl:function name="f"> ````
/// is code, not a call.
static FENCE_INVOKE_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
Regex::new(
r#"(?i)^<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*)?invoke(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*)?\s*>"#,
)
.ok()
});

/// Whether `info` — a fence's info string — opens with a complete call tag:
/// a tag-family opener or an `<invoke>` ([`FENCE_INVOKE_RE`]). `DeepSeek` V4
/// writes ```` ```<tool_call> ````, so such a fence is a call, not an example.
pub(crate) fn opens_with_call_tag(info: &str) -> bool {
let tag = TAG_RE
.as_ref()
.and_then(|re| re.find(info))
.is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str()));
tag || FENCE_INVOKE_RE.as_ref().is_some_and(|re| re.is_match(info))
}

/// Finds the closer that has the exact prefix and spelling of `opener`.
///
/// A bare `<invoke>` must not be closed by `</atem:invoke>` embedded in its
Expand Down Expand Up @@ -590,6 +612,11 @@ pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec<ParsedT
let body = strip_call_prefix(body);
let is_known = |name: &str| options.knows(name);

let calls = super::invoke_xml::decode_body(strip_code_fence(body));
if !calls.is_empty() {
return calls;
}

if let Some(registry) = options.registry {
if let Some((name, arguments)) = crate::pformat::parse_call(body, registry) {
return vec![ParsedToolCall::new(name, arguments, CallSource::PFormat)];
Expand Down
8 changes: 5 additions & 3 deletions crates/tinytools-agent/src/parse/protected.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
//!
//! Two deliberate exceptions keep real calls parseable:
//!
//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````)
//! is a call, not an example, and is handled by the tagged grammar;
//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````),
//! or whose info string opens with a call tag (```` ```<tool_call> ````),
//! is a call, not an example, and is handled by the grammars;
//! * a fence with **no** language tag is not protected. Small models wrap a
//! genuine call in a bare fence far more often than they quote one, and a
//! quoted example almost always carries a language.
Expand Down Expand Up @@ -71,7 +72,8 @@ fn scan_fences(text: &str) -> (Vec<Range<usize>>, Option<usize>) {
let language = info.split_whitespace().next().unwrap_or("");
let is_tool_call = TOOL_CALL_LANGUAGES
.iter()
.any(|lang| lang.eq_ignore_ascii_case(language));
.any(|lang| lang.eq_ignore_ascii_case(language))
|| super::grammar::tagged::opens_with_call_tag(info);
if !language.is_empty() && !is_tool_call {
open = Some((line_start, fence_char, fence_len));
}
Expand Down
55 changes: 55 additions & 0 deletions crates/tinytools-agent/src/parse/test/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,61 @@ fn fence_ranges_cover_languages_and_unclosed_fences() {
assert!(fence_ranges("```tool_call\n{}\n```").is_empty());
}

// ── A call tag on the fence line itself ────────────────────────────────────
//
// `DeepSeek` V4 Flash wrote ```` ```<tool_call> ```` — the opener as the info
// string — and never closed the fence. The info string read as a language,
// so the unclosed fence protected the call to end of text as an example.

#[test]
fn an_unclosed_fence_whose_info_string_is_a_call_tag_is_a_call() {
let text = concat!(
"<todos>\n- [ ] find the tool\n</todos>\n</tool_call>\n",
"```<tool_call>\n<invoke name=\"tool_search\">\n",
"<parameter name=\"query\" string=\"true\">list repositories</parameter>\n",
"</invoke>\n</tool_call>"
);
let outcome = parse_known(text, &["tool_search"]);
assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls);
assert_eq!(outcome.calls[0].name, "tool_search");
assert_eq!(
outcome.calls[0].arguments,
serde_json::json!({"query": "list repositories"})
);
}

#[test]
fn a_fence_whose_info_string_is_a_named_invoke_is_a_call() {
let text =
"```<invoke name=\"tool_search\">\n<parameter name=\"query\">repos</parameter>\n</invoke>";
let (_, calls) = parse(text);
assert_eq!(calls.len(), 1, "{calls:?}");
assert_eq!(calls[0].name, "tool_search");
}

#[test]
fn a_closed_fence_whose_info_string_is_a_call_tag_is_a_call() {
let text = "```<tool_call>\n<invoke name=\"tool_search\">\n<parameter name=\"query\">repos</parameter>\n</invoke>\n</tool_call>\n```";
let (_, calls) = parse(text);
assert_eq!(calls.len(), 1, "{calls:?}");
}

#[test]
fn a_language_fence_still_protects_a_call_tag_example() {
let example = "<tool_call>\n<invoke name=\"shell\"><parameter name=\"command\">rm -rf /</parameter></invoke>\n</tool_call>";
for text in [
format!("```xml\n{example}"),
format!("```xml<tool_call>\n{example}"),
format!("```text <tool_call>\n{example}"),
format!("```<function name=\"f\">\n{example}"),
format!("```<xsl:function name=\"f\">\n{example}"),
format!("```<function=shell>\n{example}"),
] {
let (_, calls) = parse(&text);
assert!(calls.is_empty(), "{text:?} dispatched {calls:?}");
}
}

#[test]
fn names_are_repaired_against_known_tools() {
let outcome = parse_known(
Expand Down
85 changes: 85 additions & 0 deletions crates/tinytools-agent/src/parse/test/tagged.rs
Original file line number Diff line number Diff line change
Expand Up @@ -739,3 +739,88 @@ fn recovery_does_not_execute_a_named_invoke_inside_malformed_json() {
let (_, calls) = parse(raw);
assert!(calls.is_empty(), "{calls:?}");
}

// ── Invoke XML inside the tag ───────────────────────────────────────────────
//
// Told to call tools "inside <tool_call> tags", `DeepSeek` V4 writes its
// native invoke XML there. The tag claimed the block and found no JSON, so
// the call was dropped as malformed even though the same invoke parses bare.

#[test]
fn a_named_invoke_wrapped_in_a_tool_call_tag_is_decoded() {
let raw = "Searching.\n<tool_call>\n<invoke name=\"tool_search\">\n<parameter name=\"query\">repos</parameter>\n</invoke>\n</tool_call>";
let outcome = super::parse_known(raw, &["tool_search"]);
assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls);
assert_eq!(outcome.calls[0].name, "tool_search");
assert_eq!(
outcome.calls[0].arguments,
serde_json::json!({"query": "repos"})
);
assert_eq!(outcome.calls[0].source, CallSource::InvokeXml);
assert_eq!(outcome.text, "Searching.");
}

#[test]
fn a_wrapped_invoke_with_string_attributes_is_decoded() {
let raw = concat!(
"<tool_call>\n<invoke name=\"tool_search\">\n",
"<parameter name=\"query\" string=\"true\">repos</parameter>\n",
"<parameter name=\"limit\" string=\"false\">5</parameter>\n",
"</invoke>\n</tool_call>"
);
let outcome = super::parse_known(raw, &["tool_search"]);
assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls);
assert_eq!(
outcome.calls[0].arguments,
serde_json::json!({"query": "repos", "limit": 5})
);
}

/// A `<todo>` block, a line of narration, then the wrapped invoke inside a
/// closed bare fence (no info string, so not protected).
#[test]
fn a_todo_block_then_a_closed_bare_fenced_wrapped_invoke_is_decoded() {
let raw = concat!(
"<todo>\n- [x] read the request\n- [ ] find the tool\n</todo>\n\n",
"Let me find the right tool.\n\n",
"```\n<tool_call>\n<invoke name=\"tool_search\">\n",
"<parameter name=\"query\" string=\"true\">list repositories</parameter>\n",
"</invoke>\n</tool_call>\n```"
);
let outcome = super::parse_known(raw, &["tool_search"]);
assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls);
assert_eq!(outcome.calls[0].name, "tool_search");
assert_eq!(
outcome.calls[0].arguments,
serde_json::json!({"query": "list repositories"})
);
}

#[test]
fn an_invoke_after_other_body_text_in_the_tag_is_not_decoded() {
let raw = "<tool_call>see <invoke name=\"shell\"><parameter name=\"command\">ls</parameter></invoke></tool_call>";
let (_, calls) = parse(raw);
assert!(calls.is_empty(), "{calls:?}");
}

#[test]
fn an_invoke_quoted_in_a_closed_json_tag_body_is_not_executed() {
let raw = "<tool_call>{\"name\":\"tool_search\",\"arguments\":{\"query\":\"<invoke name=\"shell\"><parameter name=\"command\">rm -rf /</parameter></invoke>\"}}</tool_call>";
let (_, calls) = parse(raw);
assert!(calls.iter().all(|c| c.name != "shell"), "{calls:?}");
}

/// Qwen3-Coder's native format inside the tag.
#[test]
fn a_function_equals_body_in_a_tool_call_tag_is_decoded() {
let raw = "Checking.\n<tool_call>\n<function=tool_search>\n<parameter=query>\nrepos\n</parameter>\n</function>\n</tool_call>";
let outcome = super::parse_known(raw, &["tool_search"]);
assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls);
assert_eq!(outcome.calls[0].name, "tool_search");
assert_eq!(
outcome.calls[0].arguments,
serde_json::json!({"query": "repos"})
);
assert_eq!(outcome.calls[0].source, CallSource::InvokeXml);
assert_eq!(outcome.text, "Checking.");
}
Loading