fix(agent): decode element-form calls for offered tools

crates/tinytools-agent/src/parse/grammar/element.rs

@@ -0,0 +1,233 @@
+//! `<NAME><param>value</param></NAME>` — a call written as plain XML
+//! elements: the tool name as the tag, each parameter as a child.
+//!
+//! `DeepSeek` V4 Flash writes `todo` calls this way under the Python code
+//! dialect, `<todo>\n<todos>\n[{…}]\n</todos>\n</todo>`, alongside proper
+//! `<tool_call>` blocks. Any tag could be a tool name, so the grammar is
+//! gated hard to keep prose markup from dispatching:
+//!
+//! * the tag is an offered tool **with a registry entry** — the registry is
+//!   the only place parameter names are known;
+//! * the matching `</NAME>` is present;
+//! * the body is child elements and whitespace, nothing else.
+//!
+//! A block passing all three is claimed. It decodes to a call when every
+//! child is a parameter of the tool and every `[`/`{` value is valid JSON;
+//! otherwise it is malformed, so the caller hears about the dropped call. A
+//! block failing the gate is left in the text untouched.
+//!
+//! ponytail: registry-gated, so the Xml dialect (no registry) never sees an
+//! element call; accept known-tool + child-only there if its models start
+//! writing the form.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use super::{Block, Decoded, Grammar, Probe, ScanMode, prefer_pending};
+use crate::pformat::PFormatRegistry;
+use crate::types::{CallSource, ParseOptions, ParsedToolCall};
+
+/// The element grammar.
+#[derive(Debug)]
+pub(crate) struct Element;
+
+/// An attribute-less opening tag.
+static OPEN_RE: LazyLock<Option<Regex>> =
+    LazyLock::new(|| Regex::new(r"<([A-Za-z_][\w.-]*)>").ok());
+
+/// Tag names other grammars own; never element calls.
+const RESERVED: &[&str] = &[
+    "tool_call",
+    "toolcall",
+    "tool-call",
+    "tool_calls",
+    "function_calls",
+    "calls",
+    "invoke",
+    "function",
+    "parameter",
+];
+
+impl Grammar for Element {
+    fn source(&self) -> CallSource {
+        CallSource::Element
+    }
+
+    fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe {
+        let (Some(open_re), Some(registry)) = (OPEN_RE.as_ref(), options.registry) else {
+            return Probe::None;
+        };
+        let eligible = |name: &str| {
+            registry.contains_key(name) && options.knows(name) && !RESERVED.contains(&name)
+        };
+        prefer_pending(
+            Self::probe_decided(text, from, mode, open_re, registry, &eligible),
+            (mode == ScanMode::Stream)
+                .then(|| partial_opener(text, from, registry, &eligible))
+                .flatten(),
+        )
+    }
+
+    fn openers(&self) -> &'static [&'static str] {
+        &[]
+    }
+}
+
+impl Element {
+    /// The next claimed block at or after `from`.
+    fn probe_decided(
+        text: &str,
+        from: usize,
+        mode: ScanMode,
+        open_re: &Regex,
+        registry: &PFormatRegistry,
+        eligible: &dyn Fn(&str) -> bool,
+    ) -> Probe {
+        for open in open_re.captures_iter(&text[from..]) {
+            let (Some(tag), Some(name)) = (open.get(0), open.get(1)) else {
+                continue;
+            };
+            let name = name.as_str();
+            if !eligible(name) {
+                continue;
+            }
+            let Some(params) = registry.get(name) else {
+                continue;
+            };
+            let start = from + tag.start();
+            let body_start = from + tag.end();
+            let closer = format!("</{name}>");
+            let Some(body_len) = text[body_start..].find(&closer) else {
+                // Hold only while more input could still complete a claimable
+                // block; a body that already cannot be one (a mis-closed
+                // `</todos></tool_call>`, prose) must not stall the stream.
+                if mode == ScanMode::Stream && viable_prefix(&text[body_start..], &closer) {
+                    return Probe::Pending { start };
+                }
+                continue;
+            };
+            let body = &text[body_start..body_start + body_len];
+            let Some(children) = children(body) else {
+                continue;
+            };
+            let decoded = decode(name, &params.names, &children).map_or(
+                Decoded::Malformed {
+                    body_chars: body.chars().count(),
+                },
+                |call| Decoded::Calls(vec![call]),
+            );
+            return Probe::Found(Block {
+                start,
+                end: body_start + body_len + closer.len(),
+                decoded,
+            });
+        }
+        Probe::None
+    }
+}
+
+/// In a stream, a trailing `<na` that could still grow into an eligible
+/// tool's opener. The shared scrubber holds back only the static openers
+/// grammars list, and a tool name is not one, so without this the fragment
+/// would be released as text before its `me>` arrived.
+fn partial_opener(
+    text: &str,
+    from: usize,
+    registry: &PFormatRegistry,
+    eligible: &dyn Fn(&str) -> bool,
+) -> Option<usize> {
+    let start = from + text[from..].rfind('<')?;
+    let partial = &text[start + 1..];
+    if partial.contains('>') {
+        return None;
+    }
+    registry
+        .keys()
+        .any(|name| name.starts_with(partial) && eligible(name))
+        .then_some(start)
+}
+
+/// Whether `name` can be a parameter child. A reserved name (`tool_call`,
+/// `invoke`, …) is another grammar's block: claiming it would swallow a real
+/// call nested inside a malformed element.
+fn is_child_name(name: &str) -> bool {
+    !name.is_empty()
+        && name
+            .chars()
+            .all(|c| c.is_alphanumeric() || "_.-".contains(c))
+        && !RESERVED
+            .iter()
+            .any(|reserved| reserved.eq_ignore_ascii_case(name))
+}
+
+/// Whether `rest` — a body whose outer `closer` has not arrived — can still
+/// grow into a claimable one: complete children, then at most one partial
+/// child or a partial `closer`.
+fn viable_prefix(rest: &str, closer: &str) -> bool {
+    let mut rest = rest.trim_start();
+    loop {
+        let Some(inner) = rest.strip_prefix('<') else {
+            return rest.is_empty();
+        };
+        if inner.starts_with('/') {
+            return closer[1..].starts_with(inner);
+        }
+        let Some(name_end) = inner.find('>') else {
+            return inner.is_empty() || is_child_name(inner);
+        };
+        let name = &inner[..name_end];
+        if !is_child_name(name) {
+            return false;
+        }
+        let after = &inner[name_end + 1..];
+        let Some(value_end) = after.find(&format!("</{name}>")) else {
+            return true;
+        };
+        rest = after[value_end + name.len() + 3..].trim_start();
+    }
+}
+
+/// `(name, raw value)` for each child element, or `None` when the body holds
+/// anything else — prose, an unclosed child — or no child at all.
+fn children(body: &str) -> Option<Vec<(&str, &str)>> {
+    let mut out = Vec::new();
+    let mut rest = body.trim_start();
+    while !rest.is_empty() {
+        let inner = rest.strip_prefix('<')?;
+        let name_end = inner.find('>')?;
+        let name = &inner[..name_end];
+        if !is_child_name(name) {
+            return None;
+        }
+        let after = &inner[name_end + 1..];
+        let closer = format!("</{name}>");
+        let value_end = after.find(&closer)?;
+        out.push((name, &after[..value_end]));
+        rest = after[value_end + closer.len()..].trim_start();
+    }
+    (!out.is_empty()).then_some(out)
+}
+
+/// The call, or `None` when a child is not a parameter or a JSON-looking
+/// value does not parse.
+fn decode(name: &str, params: &[String], children: &[(&str, &str)]) -> Option<ParsedToolCall> {
+    let mut arguments = serde_json::Map::new();
+    for (key, raw) in children {
+        if !params.iter().any(|param| param == key) {
+            return None;
+        }
+        let trimmed = raw.trim();
+        let value = if trimmed.starts_with(['[', '{']) {
+            serde_json::from_str(trimmed).ok()?
+        } else {
+            super::invoke_xml::scalar_value(trimmed)
+        };
+        arguments.insert((*key).to_string(), value);
+    }
+    Some(ParsedToolCall::new(
+        name,
+        serde_json::Value::Object(arguments),
+        CallSource::Element,
+    ))
+}

crates/tinytools-agent/src/parse/grammar/invoke_xml.rs

@@ -217,6 +217,32 @@ impl InvokeXml {
     }
 }
 
+/// Every call in a tag body that *is* invoke XML — `<tool_call><invoke
+/// name="x">…</invoke></tool_call>`, what `DeepSeek` V4 writes when told to
+/// call tools inside `<tool_call>` tags. Empty unless the body opens with a
+/// named invoke, so an invoke quoted inside some other body (a JSON string,
+/// say) is not executed. Once the body does open with one, every later
+/// invoke in it is decoded too, exactly as the same text outside a tag is.
+pub(crate) fn decode_body(body: &str) -> Vec<ParsedToolCall> {
+    let body = body.trim_start();
+    if OPEN_RE
+        .as_ref()
+        .and_then(|re| re.find(body))
+        .is_none_or(|m| m.start() != 0)
+    {
+        return Vec::new();
+    }
+    let mut calls = Vec::new();
+    let mut from = 0;
+    while let Probe::Found(block) = InvokeXml::probe_decided(body, from, ScanMode::Batch) {
+        if let Decoded::Calls(found) = block.decoded {
+            calls.extend(found);
+        }
+        from = block.end;
+    }
+    calls
+}
+
 /// Whether a wrapper tag is a closer or carries a DSML / namespace prefix —
 /// either is unambiguous protocol furniture even with no invoke in sight.
 fn is_closer_or_prefixed(tag: &str) -> bool {
@@ -271,7 +297,7 @@ fn decode_arguments(body: &str) -> serde_json::Value {
 
 /// A parameter value: JSON when it parses as a number, bool, null, array or
 /// object; otherwise the trimmed text.
-fn scalar_value(raw: &str) -> serde_json::Value {
+pub(crate) fn scalar_value(raw: &str) -> serde_json::Value {
     let trimmed = raw.trim();
     match serde_json::from_str::<serde_json::Value>(trimmed) {
         Ok(

crates/tinytools-agent/src/parse/grammar/mod.rs

@@ -12,6 +12,7 @@
 //! at the same byte; the engine otherwise takes the earliest opener.
 
 pub(crate) mod bare_json;
+pub(crate) mod element;
 pub(crate) mod glm;
 pub(crate) mod harmony;
 pub(crate) mod invoke_xml;
@@ -97,6 +98,7 @@ pub(crate) static GRAMMARS: &[&dyn Grammar] = &[
     &harmony::Harmony,
     &mistral::Mistral,
     &tagged::Tagged,
+    &element::Element,
 ];
 
 /// Every opener prefix across all scan grammars.

crates/tinytools-agent/src/parse/grammar/tagged.rs

@@ -101,6 +101,28 @@ fn find_re(re: &LazyLock<Option<Regex>>, haystack: &str) -> Option<(usize, usize
         .map(|m| (m.start(), m.end()))
 }
 
+/// An `<invoke>` opener, bare or named, optionally DSML-prefixed — the only
+/// invoke spellings a fence line may carry and still count as a call. No
+/// `<function …>` and no XML namespace: ```` ```<xsl:function name="f"> ````
+/// is code, not a call.
+static FENCE_INVOKE_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
+    Regex::new(
+        r#"(?i)^<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*)?invoke(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*)?\s*>"#,
+    )
+    .ok()
+});
+
+/// Whether `info` — a fence's info string — opens with a complete call tag:
+/// a tag-family opener or an `<invoke>` ([`FENCE_INVOKE_RE`]). `DeepSeek` V4
+/// writes ```` ```<tool_call> ````, so such a fence is a call, not an example.
+pub(crate) fn opens_with_call_tag(info: &str) -> bool {
+    let tag = TAG_RE
+        .as_ref()
+        .and_then(|re| re.find(info))
+        .is_some_and(|m| m.start() == 0 && !is_closing_marker(m.as_str()));
+    tag || FENCE_INVOKE_RE.as_ref().is_some_and(|re| re.is_match(info))
+}
+
 /// Finds the closer that has the exact prefix and spelling of `opener`.
 ///
 /// A bare `<invoke>` must not be closed by `</atem:invoke>` embedded in its
@@ -590,6 +612,11 @@ pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec<ParsedT
     let body = strip_call_prefix(body);
     let is_known = |name: &str| options.knows(name);
 
+    let calls = super::invoke_xml::decode_body(strip_code_fence(body));
+    if !calls.is_empty() {
+        return calls;
+    }
+
     if let Some(registry) = options.registry {
         if let Some((name, arguments)) = crate::pformat::parse_call(body, registry) {
             return vec![ParsedToolCall::new(name, arguments, CallSource::PFormat)];

crates/tinytools-agent/src/parse/protected.rs

@@ -8,8 +8,9 @@
 //!
 //! Two deliberate exceptions keep real calls parseable:
 //!
-//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````)
-//!   is a call, not an example, and is handled by the tagged grammar;
+//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````),
+//!   or whose info string opens with a call tag (```` ```<tool_call> ````),
+//!   is a call, not an example, and is handled by the grammars;
 //! * a fence with **no** language tag is not protected. Small models wrap a
 //!   genuine call in a bare fence far more often than they quote one, and a
 //!   quoted example almost always carries a language.
@@ -71,7 +72,8 @@ fn scan_fences(text: &str) -> (Vec<Range<usize>>, Option<usize>) {
                 let language = info.split_whitespace().next().unwrap_or("");
                 let is_tool_call = TOOL_CALL_LANGUAGES
                     .iter()
-                    .any(|lang| lang.eq_ignore_ascii_case(language));
+                    .any(|lang| lang.eq_ignore_ascii_case(language))
+                    || super::grammar::tagged::opens_with_call_tag(info);
                 if !language.is_empty() && !is_tool_call {
                     open = Some((line_start, fence_char, fence_len));
                 }