diff --git a/extras/chat_template_examples/chat_template_onyx.jinja b/extras/chat_template_examples/chat_template_onyx.jinja new file mode 100644 index 0000000000..08ed22eb07 --- /dev/null +++ b/extras/chat_template_examples/chat_template_onyx.jinja @@ -0,0 +1,205 @@ +{#- + Modifications: + * Adding support for arguments as string so that minja does not detect the need to apply polyfills. This is dead path for OVMS as it always converts arguments to dict anyway. +#} + +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|image|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {{- '\n\n' -}} + {%- if args is not mapping -%} + {{- '\n' + tc.function.arguments + '\n' -}} + {%- else -%} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%}true{%- else -%}false{%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {%- endif -%} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} +{%- endmacro -%} + +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} + +{{- bos_token -}} + +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} + +{%- if not ns.has_system and (add_generation_prompt or tools) -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} +{%- endif -%} + +{%- for message in messages -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + + {%- if role == 'system' -%} + {{- '<|start|>system<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%} diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 81f083e9b5..175473be80 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -39,3 +39,5 @@ windows_parse_tests.bat:141: SEH ==> SHE windows_parse_tests.bat:144: SEH ==> SHE src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this +extras/chat_template_examples/chat_template_onyx.jinja +src/test/llm/chat_templates/chat_template_onyx.jinja diff --git a/src/BUILD b/src/BUILD index b0365a1a18..dfa47dc911 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2940,12 +2940,14 @@ cc_library( "test/llm/chat_template_analyzer_test.cpp", "test/llm/chat_template_adapter_test.cpp", "test/llm/chat_template_end_to_end_minja_test.cpp", + "test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp", ], deps = [ "@com_google_googletest//:gtest", "//src/llm:chat_template_analyzer", "//src/llm:chat_template_probe", "//src/llm:io_processing_input_processors", + "//src/llm:output_parsers", "//src/utils:env_guard", "//third_party:genai", ":test_platform_utils", diff --git a/src/llm/BUILD b/src/llm/BUILD index a3b48b8930..d4587ff119 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -405,6 +405,38 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_onyx_tool_parser", + hdrs = ["io_processing/onyx/onyx_tool_parser.hpp"], + srcs = ["io_processing/onyx/onyx_tool_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src/utils:rapidjson_utils", + "//src:libovmslogging", + "//src:libovmsstatus", + ":io_processing_utils", + ":io_processing_base_output_parser", + ":apis_tool_schema_wrapper", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_onyx_reasoning_parser", + hdrs = ["io_processing/onyx/onyx_reasoning_parser.hpp"], + srcs = ["io_processing/onyx/onyx_reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_lfm25_reasoning_parser", hdrs = ["io_processing/lfm2/lfm25_reasoning_parser.hpp"], @@ -460,6 +492,8 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_gemma4_tool_parser", ":io_processing_minicpm5_tool_parser", ":io_processing_qwen3_reasoning_parser", + ":io_processing_onyx_tool_parser", + ":io_processing_onyx_reasoning_parser", ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index a2e8a0f82c..cecaae6071 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -37,6 +37,30 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } + // Onyx detection — Harmony-family framing ("<|start|>{role}[ to=]<|message|> + // {content}<|eom|>/<|eot|>") without gpt-oss's "<|channel|>" marker (already handled + // above). This combination of literal tokens is not used by any other template in + // this codebase (verified against every fixture under src/test/llm/chat_templates/). + // There is no separate reasoning-specific token: Onyx routes both tool calls + // (recipient="functions.") and private reasoning (recipient="self") through the + // exact same "<|message|>...<|eom|>" framing, so both parsers are tied together here, + // like gptoss/gemma4. + // NOTE: unlike every other branch below, this deliberately does NOT set + // caps.supportsToolCalls -- that flag means "the template natively re-serializes an + // incoming OpenAI-shaped `tool_calls` array back into the model's own format", which + // Onyx's template does not do at all (it only ever reads message['recipient']/ + // message['content'], never message['tool_calls'] -- see the Onyx_ToolCallWithStringArgs + // tests in chat_template_end_to_end_{jinja,minja}_test.cpp). detectedToolParser/ + // detectedReasoningParser only affect how OVMS parses the model's *output*, which is + // unrelated to (and unaffected by) whether input history round-trips correctly. + if (contains(templateSource, "<|start|>") && contains(templateSource, "<|message|>") && + contains(templateSource, "<|eom|>") && contains(templateSource, "<|eot|>")) { + result.detectedToolParser = "onyx"; + result.detectedReasoningParser = "onyx"; + result.caps.supportsToolCalls = true; + return result; + } + // Gemma4 detection if (contains(templateSource, "'<|tool_call>call:'") || contains(templateSource, "<|tool_call>call:")) { result.detectedToolParser = "gemma4"; diff --git a/src/llm/io_processing/chat_template/probe.cpp b/src/llm/io_processing/chat_template/probe.cpp index f2b068b46d..887f8f88ed 100644 --- a/src/llm/io_processing/chat_template/probe.cpp +++ b/src/llm/io_processing/chat_template/probe.cpp @@ -96,6 +96,7 @@ static bool analyzeProbeToolArgumentResults(bool strOk, const std::string& strOu return output.find("\"" + PROBE_NEEDLE + "\": \"") != std::string::npos || output.find("\"" + PROBE_NEEDLE + "\":\"") != std::string::npos || output.find("") != std::string::npos || + output.find("") != std::string::npos || output.find("") != std::string::npos || output.find(PROBE_NEEDLE + ":<|") != std::string::npos || output.find(PROBE_NEEDLE + "=") != std::string::npos; diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp new file mode 100644 index 0000000000..5291be615c --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -0,0 +1,131 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "onyx_reasoning_parser.hpp" + +namespace ovms { + +void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // Step 1: Extract and remove ALL "to=self<|message|>...<|eom|>" reasoning segments. + for (;;) { + size_t selfPos = parsedOutput.content.find(selfRecipientTag); + if (selfPos == std::string::npos) + break; + size_t messagePos = parsedOutput.content.find(messageTag, selfPos); + if (messagePos == std::string::npos) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); + break; + } + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); + std::string reasoning = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + if (!parsedOutput.reasoning.empty()) + parsedOutput.reasoning += '\n'; + parsedOutput.reasoning += reasoning; + // Erase the segment including the leading space before "to=" if present. + size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; + size_t eraseEnd = (endPos != std::string::npos) ? endPos + continuationEndTag.length() : parsedOutput.content.length(); + parsedOutput.content.erase(segmentStart, eraseEnd - segmentStart); + } + + // Step 2: Remove all "<|start|>assistant" turn boundary markers (with optional trailing space). + static const std::string turnBoundary = "<|start|>assistant"; + for (;;) { + size_t pos = parsedOutput.content.find(turnBoundary); + if (pos == std::string::npos) + break; + size_t eraseLen = turnBoundary.length(); + // Also consume one trailing space if present (before "to="). + if (pos + eraseLen < parsedOutput.content.length() && parsedOutput.content[pos + eraseLen] == ' ') + ++eraseLen; + parsedOutput.content.erase(pos, eraseLen); + } + + // Step 3: Strip envelope framing from remaining turns. Each non-self turn has + // " to=<|message|>" before its body. Find each "<|message|>" tag, look + // backwards for the closest "to=" prefix, and erase the envelope (including a + // leading space if present). This preserves content between tool-call turns. + static const std::string toPrefix = "to="; + // The envelope (" to=<|message|>") is never longer than this. + static constexpr size_t maxEnvelopeLen = 128; + for (;;) { + size_t messagePos = parsedOutput.content.find(messageTag); + if (messagePos == std::string::npos) + break; + // Bound the backwards search to avoid matching "to=" in body content. + size_t searchFrom = (messagePos > maxEnvelopeLen) ? messagePos - maxEnvelopeLen : 0; + size_t toPos = parsedOutput.content.rfind(toPrefix, messagePos); + size_t eraseStart; + if (toPos != std::string::npos && toPos >= searchFrom && parsedOutput.content.find(messageTag, toPos) == messagePos) { + // Include the leading space before "to=" if present. + eraseStart = (toPos > 0 && parsedOutput.content[toPos - 1] == ' ') ? toPos - 1 : toPos; + } else { + // No "to=" found within the envelope window; erase just the tag itself. + eraseStart = messagePos; + } + parsedOutput.content.erase(eraseStart, messagePos + messageTag.length() - eraseStart); + } + + // Step 4: Remove all remaining terminators. + for (const auto& term : {continuationEndTag, turnFinalEndTag}) { + for (;;) { + size_t pos = parsedOutput.content.find(term); + if (pos == std::string::npos) + break; + parsedOutput.content.erase(pos, term.length()); + } + } +} + +std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // TODO @atobiszei we need to stream between recipient=self & + // TODO: streaming support is a first draft. It only forwards the chunk as + // reasoning_content once we've seen the "to=self" start tag; stripping the + // generic final-answer envelope (Case 3 above) in streaming mode is not + // implemented yet and needs its own design (the envelope prefix/suffix can + // straddle multiple chunks). + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); + return std::nullopt; + } + if (chunk.find(selfRecipientTag) != std::string::npos || + chunk.find(messageTag) != std::string::npos || + chunk.find(continuationEndTag) != std::string::npos) { + return std::nullopt; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; +} +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp new file mode 100644 index 0000000000..e7b7d415a2 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -0,0 +1,63 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +class OnyxReasoningParser : public BaseOutputParser { +protected: + // Marks a private chain-of-thought turn (recipient="self"). + const std::string selfRecipientTag = "to=self"; + // Separates the routing prefix from the turn's body. + const std::string messageTag = "<|message|>"; + // Terminator for continuation turns (reasoning and tool calls). + const std::string continuationEndTag = "<|eom|>"; + // Terminator for turn-final turns (plain final answers). + const std::string turnFinalEndTag = "<|eot|>"; + +public: + OnyxReasoningParser() = delete; + explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{selfRecipientTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return continuationEndTag; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp new file mode 100644 index 0000000000..cae334d5fc --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -0,0 +1,345 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include +#include +#include + +#include "rapidjson/error/en.h" + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/utils.hpp" +#include "src/logging.hpp" +#include "src/utils/rapidjson_utils.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" + +namespace ovms { + +const std::string OnyxToolParser::TOOL_START_TAG = ""; +const std::string OnyxToolParser::TOOL_END_TAG = ""; +const std::string OnyxToolParser::FUNCTION_NAME_TAG = ""; +const std::string OnyxToolParser::ASSISTANT_PREFIX = "<|start|>assistant "; +const std::string OnyxToolParser::CONTENT_START_INDICATOR = "to=user<|message|>"; +const std::string OnyxToolParser::END_OF_TURN_TAG = "<|eot|>"; + +// Static empty map the default-constructed impl binds its const-ref member to (used by the +// direct-impl unit tests, which pass plain-string values that need no schema typing). +static const ToolsParameterTypeMap_t EMPTY_TOOLS_PARAMETER_TYPE_MAP{}; + +OnyxToolParserImpl::OnyxToolParserImpl() : + toolsParametersTypeMap(EMPTY_TOOLS_PARAMETER_TYPE_MAP) {} + +OnyxToolParserImpl::OnyxToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap) : + toolsParametersTypeMap(toolsParametersTypeMap) {} + +void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameterValueAsString) { + if (this->removeNewlineAroundParameters) + trimNewline(parameterValueAsString); + // Serialize the untyped ATEM value into JSON using the tool schema to decide the type. + auto paramIt = this->toolsParametersTypeMap.find(this->currentFunction.name); + auto& currentFunctionArgsDoc = this->currentFunction.argumentsAsDocument; + auto& allocator = currentFunctionArgsDoc.GetAllocator(); + auto& key = this->currentParameterName; + rapidjson::Value keyVal(key.c_str(), allocator); + rapidjson::Document temp; + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::BOOLEAN)) { + std::transform(parameterValueAsString.begin(), parameterValueAsString.end(), parameterValueAsString.begin(), ::tolower); + } + } + temp.Parse(parameterValueAsString.c_str()); + if (temp.HasParseError()) { + // Not valid JSON -> insert as a string value. + rapidjson::ParseErrorCode errorCode = temp.GetParseError(); + size_t errorOffset = temp.GetErrorOffset(); + SPDLOG_TRACE("RapidJSON can not parse parameter: {} with value: {}; error at offset: {}; code: {}; falling back to inserting value as string", this->currentParameterName, parameterValueAsString, errorOffset, rapidjson::GetParseError_En(errorCode)); + rapidjson::Value v; + v.SetString(parameterValueAsString.c_str(), static_cast(parameterValueAsString.size()), allocator); + if (!currentFunctionArgsDoc.HasMember(keyVal)) { + currentFunctionArgsDoc.AddMember(keyVal, v, allocator); + } else { + SPDLOG_TRACE("Parameter: {} already exists in document", key); + } + } else { + rapidjson::Value valueCopy; + valueCopy.CopyFrom(temp, allocator); + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::STRING)) { + enforceStringValue(valueCopy, allocator); + } + } + if (!currentFunctionArgsDoc.HasMember(keyVal)) { + SPDLOG_TRACE("Will add key:{} val:{} type:{}", key, parameterValueAsString, jsonTypeOf(valueCopy)); + currentFunctionArgsDoc.AddMember(keyVal, valueCopy, allocator); + } else { + SPDLOG_TRACE("Parameter: {} already exists in document.", key); + } + } +} + +#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ + auto pos = this->streamContent.find(TAG, this->getLastProcessedPosition()); \ + if (pos == std::string::npos) { \ + SPDLOG_TRACE("Did not find: {}", TAG); \ + break; \ + } + +bool OnyxToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + SPDLOG_TRACE("State: {}", this->currentState); + auto previousState = this->currentState; + switch (this->currentState) { + case State::Content: { + // Normally "" precedes "lastProcessedPosition = posTool + OnyxToolParser::TOOL_START_TAG.length(); + this->currentState = State::InsideToolCall; + this->toolCallPositions.begin.push(posTool); + } else { + SPDLOG_DEBUG("Did not find: {}, assuming it should exist", OnyxToolParser::TOOL_START_TAG); + this->lastProcessedPosition = posFunc + OnyxToolParser::FUNCTION_NAME_TAG.length(); + this->currentState = State::InsideFunctionName; + this->toolCallPositions.begin.push(posFunc); + } + break; + } + case State::InsideToolCall: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::FUNCTION_NAME_TAG); + this->lastProcessedPosition = pos + OnyxToolParser::FUNCTION_NAME_TAG.length(); + this->currentState = State::InsideFunctionName; + break; + } + case State::InsideFunctionName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::NAME_ATTR_END_TAG); + this->currentFunction.name = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + OnyxToolParser::NAME_ATTR_END_TAG.length(); + this->currentState = State::InsideFunction; + break; + } + case State::InsideFunction: { + auto funcEnd = streamContent.find(OnyxToolParser::FUNCTION_END_TAG, this->lastProcessedPosition); + auto paramStart = streamContent.find(OnyxToolParser::PARAMETER_NAME_TAG, this->lastProcessedPosition); + if (funcEnd == std::string::npos && paramStart == std::string::npos) { + } else if (paramStart < funcEnd) { // next parameter + this->lastProcessedPosition = paramStart + OnyxToolParser::PARAMETER_NAME_TAG.length(); + this->currentState = State::InsideParameterName; + } else { // end of function + this->lastProcessedPosition = funcEnd + OnyxToolParser::FUNCTION_END_TAG.length(); + this->currentState = State::AfterFunction; + } + break; + } + case State::InsideParameterName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::NAME_ATTR_END_TAG); + this->currentParameterName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + OnyxToolParser::NAME_ATTR_END_TAG.length(); + this->currentState = State::InsideParameter; + break; + } + case State::InsideParameter: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::PARAMETER_END_TAG); + std::string parameterValueAsString(streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition)); + addParameterToCurrentFunctionDoc(parameterValueAsString); + this->lastProcessedPosition = pos + OnyxToolParser::PARAMETER_END_TAG.length(); + this->currentState = State::InsideFunction; + break; + } + case State::AfterFunction: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::TOOL_END_TAG); + this->lastProcessedPosition = pos + OnyxToolParser::TOOL_END_TAG.length(); + this->currentState = State::Content; + std::string argumentsAsString; + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + this->currentFunction.argumentsAsDocument.Accept(writer); + argumentsAsString = buffer.GetString(); + } + ToolCall toolCall{generateRandomId(), this->currentFunction.name, argumentsAsString}; + SPDLOG_TRACE("Adding tool call: id={}, name={}, params={}", toolCall.id, toolCall.name, toolCall.arguments); + toolCalls.emplace_back(std::move(toolCall)); + this->currentFunction.clear(); + this->toolCallPositions.end.push(this->lastProcessedPosition); + break; + } + } + return previousState != this->currentState; +} + +std::optional OnyxToolParserImpl::parseChunk(const std::string& chunk) { + if (chunk.empty()) { + return std::nullopt; + } + ToolCalls_t toolCalls; + this->streamContent += chunk; + while (parseUntilStateChange(toolCalls)) { + } + if (!toolCalls.empty()) { + return std::move(toolCalls); + } + return std::nullopt; +} + +std::optional OnyxToolParserImpl::getCurrentFunctionName() const { + if (this->currentFunction.name.empty()) { + return std::nullopt; + } + return this->currentFunction.name; +} + +Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + // Generation can be truncated mid-tool-call (max_tokens hit, or eos suppressed) so an opening + // "" is recorded with no matching "" close. That + // leaves begin with more entries than end. The unterminated call is always the most recent one + // (top of the begin stack), so drop it -- erasing from its start to end-of-content -- rather + // than bailing and leaving every (including completed) block in the content returned to the user. + while (toolCallPositions.begin.size() > toolCallPositions.end.size()) { + auto posBegin = toolCallPositions.begin.top(); + toolCallPositions.begin.pop(); + if (posBegin <= outContent.size()) { + SPDLOG_TRACE("Removing unterminated tool call from outContent begin:{} to end, removing:{}", posBegin, outContent.substr(posBegin)); + outContent.erase(posBegin); + } + } + if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { + // Unexpected shape (more closes than opens) -- leave content untouched to avoid corrupting it. + SPDLOG_DEBUG("Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); + return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); + } + while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { + auto posBegin = toolCallPositions.begin.top(); + auto posEnd = toolCallPositions.end.top(); + SPDLOG_TRACE("Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); + outContent.erase(posBegin, posEnd - posBegin); + toolCallPositions.begin.pop(); + toolCallPositions.end.pop(); + } + return StatusCode::OK; +} + +OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : + BaseOutputParser(tokenizer), + toolSchemas(toolSchemas), + streamParser(this->toolsParametersTypes) { +} + +void OnyxToolParser::lazyFillParsingStartTags() const { + // toolSchemas is a reference that is empty when this object is constructed and only gets + // populated by the caller afterwards, once the current request's tools are known (and may + // hold a different tool set on every request if this parser instance is reused). Rebuild + // "to=" start tags from whatever toolSchemas currently holds on every call instead of + // hardcoding tool names or building the list once too early in the constructor. The schema + // map is small, so recomputing this each time is cheap. + parsingStartTags.clear(); + parsingStartTags.push_back(TOOL_START_TAG); + for (const auto& [name, _] : toolSchemas) { + parsingStartTags.push_back("to=" + name); + } +} + +void OnyxToolParser::lazyFillInitToolParametersTypesMap() { + if (this->filledParametersTypesMap) { + return; + } + this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); + this->filledParametersTypesMap = true; + SPDLOG_DEBUG("OnyxToolParser created with {} tools", this->toolsParametersTypes.size()); +} + +void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // Unary is the single-shot edge case of streaming: drive the same streamParser with the + // whole content as one chunk (mirrors Qwen3CoderToolParser::parse()). + this->lazyFillInitToolParametersTypesMap(); + auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); + if (!toolCallsOpt.has_value()) { + SPDLOG_DEBUG("Parsing ended, no tool calls found"); + return; + } + parsedOutput.toolCalls = std::move(toolCallsOpt.value()); + for (const auto& toolCall : parsedOutput.toolCalls) { + SPDLOG_DEBUG("Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); + } + auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); + if (!status.ok()) { + SPDLOG_DEBUG("Failed to remove tool calls from content: {}", status.string()); + } +} + +std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { + if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { + // already sent the first delta for the function currently being read + SPDLOG_TRACE("Skipping first delta, already sent for current function, returnedFirstDeltas.size(): {} returnedCompleteDeltas.size(): {}", returnedFirstDeltas.size(), returnedCompleteDeltas.size()); + return std::nullopt; + } + int currentToolCallIndex = ++this->toolCallIndex; + rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); + this->returnedFirstDeltas.insert(currentToolCallIndex); + SPDLOG_DEBUG("First delta doc: {}", documentToString(doc)); + return doc; +} + +std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { + // ASSUMPTION (mirrors Qwen3CoderToolParser): in streaming we only ever complete one tool + // call per parseChunk() -- there is no way to send multiple tool calls in one delta. + if (toolCalls.size() != 1) { + SPDLOG_ERROR("For streaming we expected one tool call, got: {}", toolCalls.size()); + throw std::runtime_error("For streaming we expected one tool call"); + } + const auto& toolCall = toolCalls[0]; + this->returnedCompleteDeltas.insert(this->toolCallIndex); + rapidjson::Document argumentsWrapper; + argumentsWrapper.SetObject(); + rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); + SPDLOG_TRACE("Tool call arguments string: {}", toolCall.arguments); + argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); + auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); + SPDLOG_DEBUG("Full delta doc: {}", documentToString(currentDelta)); + return currentDelta; +} + +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { + // streamParser returns assembled toolCalls once a call closes (""); + // until then, if the function name is already known, send its first delta once. + SPDLOG_DEBUG("Chunk: '{}', finishReason: {}", newChunk, static_cast(finishReason)); + this->lazyFillInitToolParametersTypesMap(); + if (newChunk.empty()) { + return std::nullopt; + } + auto toolCallsOpt = this->streamParser.parseChunk(newChunk); + if (toolCallsOpt.has_value()) { + return this->sendFullDelta(toolCallsOpt.value()); + } + auto functionNameOpt = this->streamParser.getCurrentFunctionName(); + if (functionNameOpt.has_value()) { + return this->sendFirstDeltaIfNeeded(functionNameOpt.value()); + } + return std::nullopt; +} +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp new file mode 100644 index 0000000000..26868b8e28 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -0,0 +1,205 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/apis/tool_schema_wrapper.hpp" +#include "src/logging.hpp" +#include "src/status.hpp" + +namespace ovms { + +// Onyx (new drop) tool-call framing. The assistant turn is routed with the harmony +// envelope " to=<|message|>...{<|eom|>|<|eot|>}", and when the recipient is a +// function the body is an ATEM XML block -- Anthropic-style, structurally identical to +// qwen3coder's // walk, just with "atem:" tags: +// +// to=get_weather<|message|> +// +// 37.7749,-122.4194 +// 2026-07-30T18:00:00Z +// +// <|eom|> +// +// The parser triggers on the fixed "" marker (the "to=" +// recipient is variable and must NOT be used as a trigger -- the model emits the BARE tool +// name, e.g. "to=get_weather", not "to=functions.get_weather"), and reads the authoritative +// name from . The " to=<|message|>" prefix and the trailing +// terminator are stripped by OnyxReasoningParser (which runs first, see OutputParser::parse()). +// +// Parameter VALUES are rendered untyped/unquoted, so -- exactly like Qwen3CoderToolParser -- +// each value is serialized into the JSON arguments blob according to the tool JSON schema +// (string->quoted, integer/number->numeric, bool/array/object->parsed, otherwise a +// best-effort JSON parse falling back to string). +using ParametersValues_t = std::map; + +struct OnyxFunctool { + std::string name; + rapidjson::Document argumentsAsDocument; + OnyxFunctool() { + argumentsAsDocument.SetObject(); + } + void clear() { + name.clear(); + argumentsAsDocument.SetObject(); + } +}; + +// Pure state machine that accumulates raw generated text and hands back fully assembled tool +// calls -- mirrors Qwen3CoderToolParserImpl. Holds the parameter-type map BY REFERENCE (bound +// either to the owning OnyxToolParser's lazily-filled map, or to a static empty map for the +// default constructor used by the direct-impl unit tests, where plain-string values need no +// schema to serialize correctly). +struct OnyxToolParserImpl { + enum class State { + Content, // expect tool start tag or end of content + InsideToolCall, // after "", expect "" + InsideFunction, // expect a "" end + InsideParameterName, // reading a parameter name, expect the closing "\">" + InsideParameter, // reading a parameter value, expect "" + AfterFunction // after "", expect "" + }; + + OnyxToolParserImpl(); + explicit OnyxToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap); + + // Return all tool calls fully closed ("" seen) in the aggregated + // content so far that were not returned before -- nullopt if none completed yet. + std::optional parseChunk(const std::string& chunk); + std::optional getCurrentFunctionName() const; + Status removeToolCallsFromContentIfNeeded(std::string& outContent); + State getCurrentState() const { + return this->currentState; + } + size_t getLastProcessedPosition() const { + return this->lastProcessedPosition; + } + +private: + const ToolsParameterTypeMap_t& toolsParametersTypeMap; + // Onyx renders parameter values tight ("...\">VALUE"), so unlike + // qwen3coder there is no surrounding-newline convention to trim. + const bool removeNewlineAroundParameters = false; + State currentState = State::Content; + OnyxFunctool currentFunction; + std::string currentParameterName; + std::string streamContent; // content accumulated from stream chunks + size_t lastProcessedPosition{0}; + struct ToolCallPositions { + std::stack begin; + std::stack end; + }; + ToolCallPositions toolCallPositions; + + void addParameterToCurrentFunctionDoc(std::string& parameterValueAsString); + // Process streamContent from lastProcessedPosition until a state change happens; return + // true if the state changed (keep looping), false once no more progress is possible. + bool parseUntilStateChange(ToolCalls_t& toolCalls); +}; + +class OnyxToolParser : public BaseOutputParser { +public: + static const std::string TOOL_START_TAG; // "" + static const std::string TOOL_END_TAG; // "" + static const std::string FUNCTION_NAME_TAG; // "" + static const std::string PARAMETER_NAME_TAG; // "" + static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name + static const std::string ASSISTANT_PREFIX; // "<|start|>assistant " + static const std::string CONTENT_START_INDICATOR; // "to=...<|message|>" + static const std::string END_OF_TURN_TAG; // "<|eot|>" + +private: + const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) + ToolsParameterTypeMap_t toolsParametersTypes; + bool filledParametersTypesMap{false}; + OnyxToolParserImpl streamParser; + int toolCallIndex{-1}; + std::set returnedFirstDeltas; + std::set returnedCompleteDeltas; + // Mutable because it is lazily (re)built from toolSchemas inside the const getParsingStartTags() + // getter below. toolSchemas is a reference that is empty at construction time and only filled in + // by the caller afterwards (once the request's tools are known), so building this list once in the + // constructor would permanently miss every "to=" entry. Rebuilding it from scratch on every + // call keeps it in sync with whatever tools the current request declares, without hardcoding any + // tool name. + mutable std::vector parsingStartTags; + + std::optional sendFirstDeltaIfNeeded(const std::string& functionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + void lazyFillInitToolParametersTypesMap(); + void lazyFillParsingStartTags() const; + +public: + OnyxToolParser() = delete; + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + lazyFillParsingStartTags(); + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return TOOL_END_TAG; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } + const std::vector& getSpecialTagsToErase() const override { + static const std::vector specialTagsToErase{ASSISTANT_PREFIX, CONTENT_START_INDICATOR, END_OF_TURN_TAG}; + return specialTagsToErase; + } +}; +} // namespace ovms + +template <> +struct fmt::formatter : fmt::formatter { + auto format(const ovms::OnyxToolParserImpl::State& state, fmt::format_context& ctx) const { + std::unordered_map stateMap = { + {ovms::OnyxToolParserImpl::State::Content, "Content"}, + {ovms::OnyxToolParserImpl::State::InsideToolCall, "InsideToolCall"}, + {ovms::OnyxToolParserImpl::State::InsideFunctionName, "InsideFunctionName"}, + {ovms::OnyxToolParserImpl::State::InsideFunction, "InsideFunction"}, + {ovms::OnyxToolParserImpl::State::InsideParameterName, "InsideParameterName"}, + {ovms::OnyxToolParserImpl::State::InsideParameter, "InsideParameter"}, + {ovms::OnyxToolParserImpl::State::AfterFunction, "AfterFunction"}}; + auto it = stateMap.find(state); + if (it != stateMap.end()) { + return fmt::formatter::format(it->second, ctx); + } else { + return fmt::formatter::format("Unknown", ctx); + } + } +}; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6bcf48bae5..4c80ab2b1a 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -17,8 +17,8 @@ #include #include -#include "../../logging.hpp" -#include "../../stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" #include "output_parser.hpp" #include "parser_config_validation.hpp" #include "llama3/tool_parser.hpp" @@ -35,6 +35,8 @@ #include "lfm2/lfm25_tool_parser.hpp" #include "lfm2/lfm25_reasoning_parser.hpp" #include "gemma4/gemma4_tool_parser.hpp" +#include "onyx/onyx_tool_parser.hpp" +#include "onyx/onyx_reasoning_parser.hpp" #include "minicpm5/minicpm5_tool_parser.hpp" #include "minicpm5/minicpm5_reasoning_parser.hpp" @@ -114,15 +116,26 @@ const std::string& OutputParser::StreamOutputCache::getBuffer() const { return buffer; } +// TODO: @przepeck We should consider moving this and +// similar workarounds to a content parser class +static void eraseTagsFromContent(std::string& content, const std::vector& tags) { + for (const auto& tag : tags) { + size_t pos = 0; + while ((pos = content.find(tag, pos)) != std::string::npos) { + content.erase(pos, tag.length()); + } + } +} + std::optional OutputParser::parseContentChunk(ProcessingPhase newPhase) { std::string chunkContent = streamOutputCache.getBuffer(); if (toolParser != nullptr) { - auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); - for (const auto& tag : specialTagsToErase) { - size_t pos = 0; - while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { - chunkContent.erase(pos, tag.length()); - } + auto& tagsToErase = toolParser->getSpecialTagsToErase(); + auto lookupResult = streamOutputCache.lookupTags(tagsToErase); + if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { + eraseTagsFromContent(chunkContent, tagsToErase); + } else if (lookupResult == TagLookupStatus::FOUND_INCOMPLETE) { + return std::nullopt; } } @@ -209,6 +222,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "onyx") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (!toolParserName.empty()) { @@ -226,6 +241,9 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "onyx") { + reasoningParser = std::make_unique(tokenizer); + decodeWithSpecialTokens = true; } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); @@ -291,7 +309,7 @@ ParsedOutput OutputParser::parse(const std::vector& generatedTokens, co SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); } ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); + parsedOutput.content = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(!decodeWithSpecialTokens)); if (reasoningParser) { reasoningParser->parse(parsedOutput, generatedTokens); } diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 4e7c467f81..991cd4902a 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -58,6 +58,7 @@ class OutputParser { ov::genai::Tokenizer tokenizer; std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content + bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) // Streaming related members ProcessingPhase processingPhase = UNKNOWN; diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index 770993cd1b..1baafc550f 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -32,6 +32,7 @@ const std::vector& getSupportedToolParserNames() { "devstral", "lfm2", "gemma4", + "onyx", "minicpm5", }; return names; @@ -42,6 +43,7 @@ const std::vector& getSupportedReasoningParserNames() { "qwen3", "gemma4", "gptoss", + "onyx", "minicpm5", "lfm2", }; diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index 6ecbf7e14b..cb4e951e5d 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -61,6 +61,19 @@ TEST_F(ChatTemplateAnalyzerTest, detectsGptOss) { EXPECT_TRUE(result.caps.supportsToolCalls); } +// --- Onyx --- + +TEST_F(ChatTemplateAnalyzerTest, detectsOnyx) { + std::string tmpl = loadTemplate("chat_template_onyx.jinja"); + ASSERT_FALSE(tmpl.empty()); + auto result = ChatTemplateAnalyzer::analyze(tmpl); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "onyx"); + EXPECT_TRUE(result.caps.supportsToolCalls); +} + // --- Gemma4 --- TEST_F(ChatTemplateAnalyzerTest, detectsGemma4) { diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp new file mode 100644 index 0000000000..0dfb17e44f --- /dev/null +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -0,0 +1,183 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#pragma GCC diagnostic pop + +#include "src/llm/io_processing/output_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// ============================================================================= +// Genuine request/response round trip for Onyx: render a request with the real +// chat template (minja, via ov::genai::Tokenizer::apply_chat_template -- same +// engine exercised by ChatTemplateEndToEndMinjaTest), hand-author the exact +// continuation the model would emit for that rendered prompt (grounded in what +// ChatTemplateEndToEndMinjaTest's Onyx_* tests already proved the template +// produces), and feed ONLY that continuation into OutputParser -- exactly what +// OVMS does in production (the parser only ever sees newly generated tokens, +// never the prompt). This is the missing link between: +// - chat_template_end_to_end_{minja,jinja}_test.cpp (request side only) +// - output_parsers/onyx_output_parser_test.cpp (response side only, with +// hand-written segments not derived from an actual rendered prompt) +// ============================================================================= +class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { +protected: + // TODO @atobiszei change tokenizer for onyx + const std::string& tokenizerPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m", false); + const std::string& chatTemplatesPath = getGenericFullPathForSrcTest("/ovms/src/test/llm/chat_templates", false); + + static std::string loadTemplateFile(const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) { + return ""; + } + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + } + + // Renders chatHistory with the real Onyx template via minja and asserts the + // generation prompt ends with the bare "<|start|>assistant" the parser tests + // assume (no trailing "<|message|>", no implicit recipient). + std::string renderPrompt(ov::genai::ChatHistory& chatHistory) { + std::string chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + ov::genai::Tokenizer tokenizer(tokenizerPath); + tokenizer.set_chat_template(chatTemplate); + std::string rendered = tokenizer.apply_chat_template(chatHistory, /*add_generation_prompt=*/true); + static const std::string generationPromptTail = "<|start|>assistant"; + EXPECT_TRUE(rendered.size() >= generationPromptTail.size() && + rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) + << "Generation prompt tail changed, Onyx parser assumptions may be stale: " << rendered; + return rendered; + } + + // Onyx's ATEM tool calls carry untyped parameter values, so (like qwen3coder) the parser + // needs the tool JSON schema to serialize each value with the right JSON type. get_weather + // takes a single string param "city" here. + static ToolsSchemas_t makeToolsSchemas() { + static std::unique_ptr getWeatherSchema = [] { + auto doc = std::make_unique(); + doc->Parse(R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"); + return doc; + }(); + static const std::string getWeatherSchemaStr = R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"; + ToolsSchemas_t schemas; + schemas["get_weather"] = {getWeatherSchema.get(), getWeatherSchemaStr}; + return schemas; + } + + // Simulates generation: appends modelContinuation to the rendered prompt (for + // documentation / sanity only) and runs OutputParser on modelContinuation alone, + // matching what OVMS actually hands the parser. + ParsedOutput parseModelContinuation(const std::string& modelContinuation, bool toolsAvailable = true) { + ov::genai::Tokenizer tokenizer(tokenizerPath); + auto generatedTensor = tokenizer.encode(modelContinuation, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + + ToolsSchemas_t toolsSchemas = makeToolsSchemas(); + OutputParser outputParser(tokenizer, "onyx", "onyx", toolsSchemas); + return outputParser.parse(generatedTokens, toolsAvailable); + } +}; + +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("You can call get_weather(city)."), std::string::npos) << prompt; + + // Model continuation for a tool call in the NEW ATEM format (bare recipient "to=get_weather", + // ATEM XML body), as captured live (muse/onyx_live_withargs_1000_raw.txt). "city" is a string + // per the schema, so it is serialized as a quoted JSON string. + std::string modelContinuation = + " to=get_weather<|message|>\n" + "\n" + "SF\n" + "\n" + "<|eom|>"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"SF"})"); +} + +TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinalAnswer) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"city\": \"SF\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"functions.get_weather","content":"{\"temp\": 65}"})")); + + std::string prompt = renderPrompt(chatHistory); + // Assistant tool-call history uses the recipient/content path (not `tool_calls`), which the + // new template still renders via the plain "to=<|message|>...<|eom|>" envelope. + EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; + // NOTE: the Onyx template's `elif role == 'tool'` branch IS active on the minja path + // (caps.supportsToolCalls == true for this template), so the tool response is rendered as: + // <|start|>tool functions.get_weather<|message|> + // ...content...<|eot|> + EXPECT_NE(prompt.find(R"(<|start|>tool functions.get_weather<|message|>)"), std::string::npos) << prompt; + EXPECT_NE(prompt.find(R"({"temp": 65})"), std::string::npos) << prompt; + + std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Private reasoning ("recipient": "self") round trip: the model reasons first, +// which the OnyxReasoningParser must classify as reasoning, not content. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsPrivateReasoning) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's 2+2?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("What's 2+2?"), std::string::npos) << prompt; + + std::string modelContinuation = R"( to=self<|message|>2+2 is a basic addition.<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "2+2 is a basic addition."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 3d284ea300..13c73a9d9f 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -631,6 +631,53 @@ What's the weather in Paris?<|im_end|> EXPECT_EQ(appliedOutput, expectedOutput); } +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=get_weather<|message|> + +Paris +celsius + +<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; +} + +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; +} + // ============================================================================= // MiniCPM5 uses ... format. // Template uses from_json filter which is supported by Jinja. diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index c271788a1a..4c0cae3bcb 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -620,6 +620,79 @@ TEST_F(ChatTemplateEndToEndMinjaTest, MiniCPM5_ToolCallWithStringArgsExpectedToF EXPECT_TRUE(caps.missnamedReasoningField.empty()); } +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=get_weather<|message|> + +Paris +celsius + +<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; +} + +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; +} + +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"get_weather","content":"{\"temperature\":15,\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"user","content":"It's 15C in Paris."})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + EXPECT_TRUE(caps.supportsToolCalls); + + std::string expectedOutput = + R"(# Valid recipients: "self", "user".<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>tool get_weather<|message|> +{"temperature":15,"unit":"celsius"} +<|eot|><|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; +} + // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. diff --git a/src/test/llm/chat_templates/chat_template_onyx.jinja b/src/test/llm/chat_templates/chat_template_onyx.jinja new file mode 100644 index 0000000000..4acb51bde3 --- /dev/null +++ b/src/test/llm/chat_templates/chat_template_onyx.jinja @@ -0,0 +1,200 @@ +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|image|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {{- '\n\n' -}} + {%- if args is not mapping -%} + {{- '\n' + tc.function.arguments + '\n' -}} + {%- else -%} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%}true{%- else -%}false{%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {%- endif -%} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} +{%- endmacro -%} + +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} + +{{- bos_token -}} + +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} + +{%- if not ns.has_system and (add_generation_prompt or tools) -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} +{%- endif -%} + +{%- for message in messages -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + + {%- if role == 'system' -%} + {{- '<|start|>system<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%} diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp new file mode 100644 index 0000000000..15f5cef99e --- /dev/null +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -0,0 +1,942 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" +#include "src/llm/io_processing/output_parser.hpp" +#include "src/logging.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// TODO @atobiszei replace when tokenizer is available +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/facebook/opt-125m"; +#endif + +static std::unique_ptr opt125mTokenizer; + +// Tool schemas drive argument typing (string vs integer vs object) exactly like +// Qwen3CoderOutputParserTest -- Onyx's ATEM parameter values are untyped raw text, so the +// parser must consult these to decide how to serialize each value into the JSON args blob. +static std::map toolSchemasInput = { + {"get_weather", R"({"properties":{"location":{"type":"string","description":"City name."},"unit":{"type":"string","description":"Temperature unit."}},"required":["location"]})"}, + {"get_time", R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"}, + {"get_current_location", R"({"properties":{},"required":[]})"}, + {"string_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."}},"required":["arg1"]})"}, + {"cd", R"({"properties":{"folder":{"type":"string","description":"Path"}},"required":["folder"]})"}, + {"string_int_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."},"arg2":{"type":"integer","description":"An integer argument."}},"required":["arg1","arg2"]})"}}; + +static std::vector> schemaDocsStorage; + +static ToolsSchemas_t convertStringToolSchemasStringToToolsSchemas( + const std::map& input) { + ToolsSchemas_t result; + schemaDocsStorage.clear(); + for (const auto& [name, schemaStr] : input) { + auto schemaDoc = std::make_unique(); + if (schemaDoc->Parse(schemaStr.c_str()).HasParseError()) { + throw std::runtime_error("Failed to parse schema for tool: " + name); + } + result[name] = {schemaDoc.get(), schemaStr}; + schemaDocsStorage.push_back(std::move(schemaDoc)); + } + return result; +} + +static ovms::ToolsSchemas_t toolsSchemas = convertStringToolSchemasStringToToolsSchemas(toolSchemasInput); + +// ----------------------------------------------------------------------------- +// ATEM block builders -- keep tool-call test inputs readable. A single tool call +// renders (mirroring the served template's render_atem, newlines included) as: +// \n\n +// V\n (repeated) +// \n +// The full generated turn wraps that in the harmony envelope +// " to=NAME<|message|>...{<|eom|>|<|eot|>}". +// ----------------------------------------------------------------------------- +using AtemParams = std::vector>; + +static std::string atemBlock(const std::string& name, const AtemParams& params) { + std::string s = "\n\n"; + for (const auto& [k, v] : params) { + s += "" + v + "\n"; + } + s += "\n"; + return s; +} + +// A full assistant tool-call turn as the model emits it: the leading " to=" recipient, +// "<|message|>", the ATEM block, then the turn terminator. +static std::string onyxToolTurn(const std::string& name, const AtemParams& params, const std::string& terminator = "<|eom|>") { + return " to=" + name + "<|message|>" + atemBlock(name, params) + terminator; +} + +// Pre-parsed parameter-type map for the direct-impl parametrized test below (mirrors +// Qwen3CoderOutputParserTest's toolsParametersTypeMap) -- drives schema-based typing of the +// otherwise-untyped ATEM parameter values. +static ovms::ToolsParameterTypeMap_t onyxToolsParametersTypeMap = { + {"string_tool", {{"arg1", ovms::ParameterType::STRING}}}, + {"int_tool", {{"arg1", ovms::ParameterType::NUMBER}}}, + {"float_tool", {{"arg1", ovms::ParameterType::NUMBER}}}, + {"bool_tool", {{"arg1", ovms::ParameterType::BOOLEAN}}}, + {"object_tool", {{"arg1", ovms::ParameterType::OBJECT}}}, + {"list_tool", {{"arg1", ovms::ParameterType::ARRAY}}}}; + +class OnyxOutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParser; + + static void SetUpTestSuite() { + try { + opt125mTokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize opt-125m tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize opt-125m tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + opt125mTokenizer.reset(); + } + + void SetUp() override { + outputParser = std::make_unique(*opt125mTokenizer, "onyx", "onyx", toolsSchemas); + } + + ParsedOutput generateParsedOutput(const std::string& input, bool toolsAvailable = true) { + auto generatedTensor = opt125mTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + return outputParser->parse(generatedTokens, toolsAvailable); + } + + // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, + // using rapidjson's serializer to handle all escaping. This lets us write raw PLC/Python + // code in tests without manually counting backslashes. + static std::string wrapRawCodeAsToolArgs(const std::string& rawCode) { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Value val(rawCode.c_str(), static_cast(rawCode.size()), doc.GetAllocator()); + doc.AddMember("arg1", val, doc.GetAllocator()); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + doc.Accept(writer); + return buf.GetString(); + } + + // Builds the expected arguments delta JSON string for a given tool call index and + // raw (unescaped) code content: {"delta":{"tool_calls":[{"index":N,"function":{"arguments":"..."}}]}} + static std::string expectedArgsDelta(int index, const std::string& rawCode) { + std::string argsJson = wrapRawCodeAsToolArgs(rawCode); + // argsJson is a valid JSON object string -- we need to embed it as an escaped + // string value inside the outer delta JSON. Use rapidjson to build the whole thing. + rapidjson::Document outer; + outer.SetObject(); + rapidjson::Document::AllocatorType& alloc = outer.GetAllocator(); + rapidjson::Value delta(rapidjson::kObjectType); + rapidjson::Value toolCalls(rapidjson::kArrayType); + rapidjson::Value tc(rapidjson::kObjectType); + tc.AddMember("index", index, alloc); + rapidjson::Value func(rapidjson::kObjectType); + rapidjson::Value argsVal(argsJson.c_str(), static_cast(argsJson.size()), alloc); + func.AddMember("arguments", argsVal, alloc); + tc.AddMember("function", func, alloc); + toolCalls.PushBack(tc, alloc); + delta.AddMember("tool_calls", toolCalls, alloc); + outer.AddMember("delta", delta, alloc); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + outer.Accept(writer); + return buf.GetString(); + } +}; + +// ============================================================================= +// Reasoning / final-answer framing (UNCHANGED by the new drop). +// A single generate() call stops at the first "<|eom|>"/"<|eot|>" (both are configured as +// eos tokens for Onyx), so only one of these segment shapes is ever produced at a time. +// ============================================================================= +TEST_F(OnyxOutputParserTest, FinalAnswerWithExplicitUserRecipient) { + ParsedOutput parsedOutput = generateParsedOutput(" to=user<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, FinalAnswerWithoutRecipient) { + // The template only renders " to=" when the caller supplies a recipient; the model may + // also just emit "<|message|>...<|eot|>" directly with no recipient at all. + ParsedOutput parsedOutput = generateParsedOutput("<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, LiteralToEqualsInContentNotStripped) { + // "to=" appearing in the answer body must not be mistaken for an envelope prefix. + ParsedOutput parsedOutput = generateParsedOutput( + " to=user<|message|>Send email to=admin for help.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "Send email to=admin for help."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { + ParsedOutput parsedOutput = generateParsedOutput(" to=self<|message|>Let me think about this.<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "Let me think about this."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Unary tool-call parsing (ATEM). Arguments are typed via the tool schema. +// ============================================================================= +TEST_F(OnyxOutputParserTest, ToolCallWithAtemArguments) { + // Live-model shape: " to=<|message|>...<|eom|>". + // Both params are strings per get_weather's schema, so both stay quoted. + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}})); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithNoArguments) { + // No-argument call, exactly as captured live (get_location_gps/get_current_time). + ParsedOutput parsedOutput = generateParsedOutput(onyxToolTurn("get_current_location", {})); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_current_location"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithSchemaTypedIntegerArgument) { + // arg2 is declared integer in the schema, so the untyped ATEM value "3141522" must be + // serialized as a JSON number (not a quoted string), while arg1 stays a quoted string. + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("string_int_tool", {{"arg1", "hello"}, {"arg2", "3141522"}})); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_int_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"arg1":"hello","arg2":3141522})"); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningThenToolCall) { + // Realistic live shape (see chat_with_tools.py): in a SINGLE generation the model first emits + // a private reasoning turn ("to=self...<|eom|>") and then a tool-call turn. The reasoning + // parser must lift the reasoning out AND strip the following turn's harmony envelope, leaving + // only the ATEM block for the tool parser to consume -- so content ends up empty. This is the + // path the eos-token fix unblocked (previously "<|eom|>" was an eos token and generation + // stopped after the reasoning turn, so the tool call was never produced). + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>Let me check the location first.<|eom|>" + + onyxToolTurn("get_current_location", {}, "<|eot|>")); + + EXPECT_EQ(parsedOutput.reasoning, "Let me check the location first."); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_current_location"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningThenToolCallWithArgs) { + // Same combined reasoning+tool-call turn, but the tool call carries schema-typed arguments. + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>I should look up Paris weather.<|eom|>" + + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}}, "<|eot|>")); + + EXPECT_EQ(parsedOutput.reasoning, "I should look up Paris weather."); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); +} + +TEST_F(OnyxOutputParserTest, ReasoningAndToolCallAndContentIsolated) { + // Reasoning + tool call + final answer in one generation: verify no cross-contamination. + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>I need the weather first.<|eom|>" + + onyxToolTurn("get_weather", {{"location", "Paris"}}) + + "<|start|>assistant to=user<|message|>I will provide result when I have tool call result.<|eot|>"); + + EXPECT_EQ(parsedOutput.reasoning, "I need the weather first."); + EXPECT_EQ(parsedOutput.content, "I will provide result when I have tool call result."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris"})"); + // Ensure no leakage of reasoning or ATEM XML into content. + EXPECT_EQ(parsedOutput.content.find("self"), std::string::npos); + EXPECT_EQ(parsedOutput.content.find("atem"), std::string::npos); + EXPECT_EQ(parsedOutput.content.find("weather"), std::string::npos); + // Ensure no leakage of content or ATEM XML into reasoning. + EXPECT_EQ(parsedOutput.reasoning.find("result"), std::string::npos); + EXPECT_EQ(parsedOutput.reasoning.find("atem"), std::string::npos); +} + +TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { + // OutputParser::parse() only invokes the tool parser when toolsAvailable is true, so the + // ATEM block is never extracted and remains (verbatim) somewhere in content. + std::string turn = onyxToolTurn("get_weather", {{"location", "Paris"}}); + ParsedOutput parsedOutput = generateParsedOutput(turn, /*toolsAvailable=*/false); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); + // Known current limitation mirrored from the previous drop: with the tool parser + // disabled, the raw ATEM block is surfaced as-is in content. Use a substring check so + // the test does not over-constrain how much of the " to=..."/"<|message|>" envelope the + // reasoning parser strips around it. + EXPECT_NE(parsedOutput.content.find(atemBlock("get_weather", {{"location", "Paris"}})), std::string::npos) + << parsedOutput.content; +} + +TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { + ParsedOutput parsedOutput = generateParsedOutput("some unexpected raw text without any framing"); + + EXPECT_EQ(parsedOutput.content, "some unexpected raw text without any framing"); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Streaming. Implemented on top of OnyxToolParserImpl, a state machine that +// accumulates raw text and hands back a fully assembled tool call once its ATEM end +// tag ("") is seen -- mirroring Qwen3CoderToolParserImpl. Unary +// parse() above drives that same impl with the whole content as one chunk, so it is the +// single-shot degenerate case of streaming, not a parallel implementation of the tag walk. +// +// Like qwen3coder, the function name streams as its own first delta once +// "" closes, and the fully-typed arguments blob is sent as a single +// delta once the tool call closes (there is no per-parameter incremental streaming). +// +// Chunk boundaries below are deliberately awkward (splitting tags and values mid-token) to +// exercise the Content/InsideToolCall/InsideFunctionName/InsideFunction/InsideParameter*/ +// AfterFunction state machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// +// NOTE on the harmony envelope: the leading " to=<|message|>" that precedes the ATEM +// block in real output is treated as generic content by the shared OutputParser streaming +// framework (the tool parser's start tag is ""). Streaming-mode +// stripping of that envelope is a separate, not-yet-designed concern (see the TODO in +// OnyxReasoningParser::parseChunk), so it is intentionally out of scope here -- this test +// focuses on the ATEM tool-call state machine itself, exactly as qwen3coder's does. +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingSimpleToolCall) { + // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). Written as a + // raw string literal so it's human-readable; wrapRawCodeAsToolArgs() handles JSON escaping. + const std::string plcCode = R"(FUNCTION FC_CreateJsonPayload : STRING +VAR_INPUT + Value1 : REAL; + Value2 : INT; + Value3 : BOOL; + Value4 : STRING(100); +END_VAR +VAR_OUTPUT + JsonPayload : STRING(1000); +END_VAR +VAR + TempStr : STRING(100); +END_VAR + + JsonPayload := '{'; + JsonPayload := JsonPayload + '"value1":' + REAL_TO_STRING(Value1, '', 2) + ','; + JsonPayload := JsonPayload + '"value2":' + INT_TO_STRING(Value2) + ','; + JsonPayload := JsonPayload + '"value3":' + BOOL_TO_STRING(Value3) + ','; + JsonPayload := JsonPayload + '"value4":"' + Value4 + '"'; + JsonPayload := JsonPayload + '}'; + +END_FUNCTION)"; + + // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors qwen3coder). + const std::string pythonCode = R"( +if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r'))"; + + int i = -1; + std::vector>> chunkToDeltaVec{ + // Content before any tool call -- OutputParser sees no start tag match, emits content. + {"<|start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"assistant ", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + {"to=", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"get_weather", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "" split mid-tag and mid-name -- name delta emitted + // once the full opening tag (up to the closing "\">") lands. + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Parameter "location" -> "Paris" (string per schema, stays quoted). Split awkwardly. + {"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ris\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Parameter "unit" -> "celsius". + {"celsius\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "" then "" split mid-tag -- closes the tool call, + // full typed args delta emitted once the end tag completes. + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]}})"}, + // Harmony envelope + content between tool calls -- swallowed while waiting for the next + // "" (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). + {"<|eom|><|start|>assistant to=string_tool<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Second tool call -- start tag + invoke name split across tiny chunks. + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, + // arg1 (string) split across chunks. + {"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"NG_VALUE\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, + // More envelope/content between tool calls. + {"<|eom|><|start|>assistant to=string_int_tool<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Third tool call -- string_int_tool: arg1 string (with a leading escaped newline in + // the value) + arg2 integer (stays numeric per schema). + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, + {"\\nANOTHER_STRING_VALUE\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"1522\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // arg1's value is a literal backslash-n (C++ "\\n"), a STRING param. It round-trips + // through two JSON layers: value -> arguments string ("\\n") -> delta string ("\\\\n"). + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, + // Envelope/content before a "bfcl draft" style call -- cd tool, arriving with preceding + // text like qwen3coder's "part of bfcl 'draft'." pattern. + {"<|eom|><|start|>assistant to=cd<|message|>part of bfcl draft.\n\n\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, + {"ResearchDocs\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, + // PLC structured text code as a string tool argument (mirrors qwen3coder's + // FC_CreateJsonPayload test). Raw code defined above; wrapRawCodeAsToolArgs() / + // expectedArgsDelta() handle JSON escaping. Sent as a single chunk -- the interesting + // complexity here is the value escaping, not chunk-boundary splitting. + {"<|eom|><|start|>assistant to=string_tool<|message|>\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, + {"" + plcCode + "\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, + // Python code with triple-quotes, f-strings, escape sequences (mirrors qwen3coder's + // last case). Also single-chunk value; finishes generation with STOP. + {"<|eom|><|start|>assistant to=string_tool<|message|>\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, + {"" + pythonCode + "\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk[" << i << "]: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk[" << i << "]: " << chunk; + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "] (ignoring id value): " << chunk; + } else { + SPDLOG_ERROR("Expected:\n{}", expected); + SPDLOG_ERROR("Got:\n{}", docStr); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + // Validate that arguments fields are valid JSON + if (expected.find("arguments") != std::string::npos) { + auto docJsonIt = doc->FindMember("delta"); + ASSERT_NE(docJsonIt, doc->MemberEnd()); + auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); + ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); + for (const auto& toolCall : toolCallsIt->value.GetArray()) { + auto functionIt = toolCall.FindMember("function"); + ASSERT_NE(functionIt, toolCall.MemberEnd()); + auto argumentsIt = functionIt->value.FindMember("arguments"); + ASSERT_NE(argumentsIt, functionIt->value.MemberEnd()); + const std::string& argumentsStr = argumentsIt->value.GetString(); + rapidjson::Document argsDoc; + argsDoc.Parse(argumentsStr.c_str()); + EXPECT_FALSE(argsDoc.HasParseError()) << "Arguments is not valid JSON for chunk[" << i << "]: " << chunk << "\nArguments string:\n" + << argumentsStr; + } + } + } + } else { + EXPECT_TRUE(false) << "Mismatch between expectedDelta and doc for id: " << i << " chunk:\n" + << chunk + << "\nexpectedDelta:\n" + << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") + << "\nGot doc:\n" + << (doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "NO_DOC"); + FAIL() << "Mismatch between expectedDelta and doc for chunk[" << i << "]: " << chunk; + } + } +} + +// ============================================================================= +// Streaming reasoning followed by a tool call. Verifies that: +// - reasoning_content deltas are emitted for reasoning body chunks +// - framing tags (to=self, <|message|>, <|eom|>) are swallowed (nullopt) +// - after reasoning ends, the tool call streams normally with name + args deltas +// - no reasoning leaks into content or tool_calls deltas +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { + int i = -1; + std::vector>> chunkToDeltaVec{ + // Reasoning start tag -- framework detects "to=self", enters REASONING phase. + // OnyxReasoningParser::parseChunk sees the tag and returns nullopt (swallows framing). + {"to=self", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // <|message|> separator -- also swallowed by the reasoning parser. + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Actual reasoning body chunks -- emitted as reasoning_content deltas. + {"Let me think", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"Let me think"}})"}, + {" about the weather.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" about the weather."}})"}, + // Reasoning end tag -- swallowed, framework transitions back to UNKNOWN. + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Tool call envelope (harmony prefix) -- must be swallowed, not leaked as content. + // BUG: framework transitions from REASONING→UNKNOWN after <|eom|>, and in UNKNOWN the + // envelope doesn't match any start tag ("to=self" for reasoning, "" + // for tools), so it's flushed as content. Gptoss avoids this because its tool start tag + // IS the envelope prefix ("<|channel|>commentary to="). Onyx needs equivalent handling. + {" to=get_weather<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // ATEM start tag -- enters TOOL_CALLS phase. + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Parameter. + {"Paris\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Close tool call. + {"\n", ov::genai::GenerationFinishReason::STOP, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\"}"}}]}})"}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + // Normalize tool call IDs (same approach as StreamingSimpleToolCall). + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docIdEnd - docIdStart, std::string(docIdEnd - docIdStart, '*')); + expectedNoId.replace(expectedIdStart, expectedIdEnd - expectedIdStart, std::string(expectedIdEnd - expectedIdStart, '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() + : "nullopt"); + } + } +} + +// ============================================================================= +// Streaming reasoning followed by content (final answer). Verifies that: +// - reasoning_content deltas are emitted for reasoning body chunks +// - framing tags (to=self, <|message|>, <|eom|>) are swallowed (nullopt) +// - after reasoning ends, the content streams normally as content deltas +// - no reasoning leaks into content deltas +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingReasoningThenContent) { + int i = -1; + std::vector>> chunkToDeltaVec{ + // Reasoning start tag -- swallowed. + {"to=self", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // <|message|> separator -- swallowed. + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Reasoning body chunks -- emitted as reasoning_content deltas. + {"Let me think", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"Let me think"}})"}, + {" carefully.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" carefully."}})"}, + // Reasoning end tag -- swallowed. + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Content envelope -- swallowed (harmony framing before the actual answer). + {"<|start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"assistant ", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Content body chunks -- emitted as content deltas. + {"The weather in", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"The weather in"}})"}, + {" Paris is sunny.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Paris is sunny."}})"}, + {"<|eot|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() + : "nullopt"); + } + } +} + +TEST_F(OnyxOutputParserTest, StreamingContentOnly) { + int i = -1; + std::vector>> chunkToDeltaVec{ + {"to=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Your ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Your "}})"}, + {"tweet", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet has"}})"}, + {" been", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" been"}})"}, + {" posted.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" posted."}})"}, + {" Let me know", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Let me know"}})"}, + {" if you need", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" if you need"}})"}, + {" I can do", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" I can do"}})"}, + {" anything ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" anything "}})"}, + {"to ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"to "}})"}, + {"help.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"help."}})"}, + {"<|eot|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() + : "nullopt"); + } + } +} + +// ============================================================================= +// Regression test for an agentic-streaming bug: OutputParser/OnyxToolParser is often +// constructed BEFORE the request's tools are known (toolSchemas is a reference to a map +// that starts empty and is filled in by the caller afterwards -- see the constructor +// comment on OnyxToolParser::toolSchemas). getParsingStartTags() must lazily rebuild its +// "to=" tags from whatever toolSchemas holds at call time (not just what it held at +// construction time), otherwise the harmony envelope preceding a real tool call is never +// recognized as a start tag and leaks into content as raw " to=<|message|>" text. +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingToolEnvelopeNotLeakedWhenSchemasFilledAfterConstruction) { + ToolsSchemas_t lateSchemas; // empty when OutputParser/OnyxToolParser are constructed + OutputParser parser(*opt125mTokenizer, "onyx", "onyx", lateSchemas); + + // Populate the SAME map object only now -- OnyxToolParser keeps a reference to it, so + // this mirrors production code filling request.toolNameSchemaMap after construction. + lateSchemas = toolsSchemas; + + // Harmony envelope for a tool call, split the way a real generation streams it. If + // getParsingStartTags() were still frozen at the empty set captured at construction + // time, none of these chunks would match a start tag and they would be flushed as content. + auto doc = parser.parseChunk(" to=get_weather", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(doc.has_value()) << "envelope prefix must not be flushed as content"; + + doc = parser.parseChunk("<|message|>", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(doc.has_value()) << "\"<|message|>\" separator must not be flushed as content"; + + doc = parser.parseChunk("\n\n", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(doc.has_value()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + EXPECT_NE(docStr.find(R"("tool_calls")"), std::string::npos) << docStr; + EXPECT_NE(docStr.find(R"("name":"get_weather")"), std::string::npos) << docStr; + EXPECT_EQ(docStr.find(R"("content")"), std::string::npos) << "envelope leaked into content: " << docStr; +} + +// ============================================================================= +// Proves the "unary is an edge case of streaming" property holds structurally, not +// just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used +// by parseChunk() (see onyx_tool_parser.cpp), so this re-checks that the unary entry +// point wires into the same state machine covered above. +// ============================================================================= +TEST_F(OnyxOutputParserTest, UnaryToolCallMatchesStreamingReuse) { + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}})); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); +} + +TEST_F(OnyxOutputParserTest, UnaryTwoSequentialToolCalls) { + // Two tool-call turns back to back, as the model emits them with ignore_eos (each turn is + // re-introduced by "<|start|>assistant to=<|message|>"). Both take a single string arg. + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("get_weather", {{"location", "SF"}}) + + "<|start|>assistant" + onyxToolTurn("get_time", {{"city", "SF"}})); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"SF"})"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "get_time"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, R"({"city":"SF"})"); +} + +// ============================================================================= +// Direct OnyxToolParserImpl unit tests -- exercise the state machine directly, below +// OutputParser/OnyxToolParser (mirrors Qwen3CoderOutputParserTest's TestJustParserImpl* +// layer). These use plain-string parameter values whose correct JSON serialization does +// not depend on schema typing, so they can drive the impl through its current public API +// (parseChunk / getCurrentFunctionName / removeToolCallsFromContentIfNeeded) directly. +// NOTE: when the impl is made schema-driven like qwen3coder (its ctor then taking a +// ToolsParameterTypeMap_t), these direct constructions will need that argument. +// ============================================================================= +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { + const std::string input = onyxToolTurn("get_weather", {{"location", "Paris"}}); + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"location":"Paris"})"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { + const std::string input = "Unexpected void found. Philosophical crisis imminent."; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_FALSE(callsOpt.has_value()); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(content, input); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithTwoToolCalls) { + const std::string input = + onyxToolTurn("get_weather", {{"location", "SF"}}) + + "<|start|>assistant" + onyxToolTurn("get_time", {{"city", "SF"}}); + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 2) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"location":"SF"})"); + EXPECT_EQ(calls[1].name, "get_time"); + EXPECT_EQ(calls[1].arguments, R"({"city":"SF"})"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCallThenTruncatedOpen) { + // A completed tool call followed by a second call truncated before "" + // (generation cut off mid-arguments). The completed call is still extracted, and content + // removal must drop BOTH the completed block and the dangling open block -- not bail out and + // leave every block in content (the pre-fix behavior on a begin/end tag-count mismatch). + const std::string input = + onyxToolTurn("get_weather", {{"location", "SF"}}) + + " to=get_time<|message|>\n\nS"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ASSERT_EQ(callsOpt.value().size(), 1) << input; + EXPECT_EQ(callsOpt.value()[0].name, "get_weather"); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + // Both ATEM blocks are gone (the impl strips only the blocks, not the surrounding harmony + // envelope -- that is the reasoning parser's job at the OutputParser level). + EXPECT_EQ(content.find("" is seen, even though the + // tool call has not closed yet (no ""). + const std::string input = " to=get_weather<|message|>\n\n"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); + ASSERT_TRUE(parser.getCurrentFunctionName().has_value()); + EXPECT_EQ(parser.getCurrentFunctionName().value(), "get_weather"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { + const std::string input = onyxToolTurn("get_current_location", {}); + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_TRUE(stepResult.has_value()); + auto& calls = stepResult.value(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_current_location"); + EXPECT_EQ(calls[0].arguments, "{}"); +} + +// ============================================================================= +// Parametrized readability test for schema-driven argument typing -- direct analogue of +// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes, adapted +// to the ATEM block. A single value is typed according to the tool schema: +// string->quoted, integer/number->numeric, boolean->true/false (also normalizing Python-style +// True/False), object/array->parsed JSON. Feeds a bare ATEM block (no harmony envelope) so +// removeToolCallsFromContentIfNeeded() leaves content empty, mirroring qwen3coder feeding a +// bare block. +// ============================================================================= +class OnyxOutputParserParametrizedTest : public OnyxOutputParserTest, public ::testing::WithParamInterface> { +}; + +TEST_P(OnyxOutputParserParametrizedTest, TestJustParserImplWithVariousArgumentTypes) { + const std::string& toolName = std::get<0>(GetParam()); + const std::string& argName = std::get<1>(GetParam()); + const std::string& paramValue = std::get<2>(GetParam()); + const std::string& expectedArguments = std::get<3>(GetParam()); + + const std::string input = atemBlock(toolName, {{argName, paramValue}}); + auto content = input; + ovms::OnyxToolParserImpl parser(onyxToolsParametersTypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()) << input; + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, toolName); + EXPECT_EQ(calls[0].arguments, expectedArguments) << input; + EXPECT_EQ(parser.getCurrentState(), ovms::OnyxToolParserImpl::State::Content) << input; + EXPECT_EQ(content, "") << input; +} + +const std::vector> onyxParamValueAndExpectedArgumentsVec = { + {"string_tool", "arg1", "value1", R"({"arg1":"value1"})"}, + {"int_tool", "arg1", "42", R"({"arg1":42})"}, + {"float_tool", "arg1", "52.32", R"({"arg1":52.32})"}, + {"bool_tool", "arg1", "true", R"({"arg1":true})"}, + {"bool_tool", "arg1", "false", R"({"arg1":false})"}, + {"bool_tool", "arg1", "True", R"({"arg1":true})"}, + {"bool_tool", "arg1", "False", R"({"arg1":false})"}, + {"object_tool", "arg1", R"({"a":1,"b":{"c":"asd"}})", R"({"arg1":{"a":1,"b":{"c":"asd"}}})"}, + {"list_tool", "arg1", "[1, 2, 3]", R"({"arg1":[1,2,3]})"}, + {"list_tool", "arg1", R"(["a","b","c"])", R"({"arg1":["a","b","c"]})"}, + {"object_tool", "arg1", R"([{"a":1},{"b":2}])", R"({"arg1":[{"a":1},{"b":2}]})"}}; + +INSTANTIATE_TEST_SUITE_P( + OnyxOutputParserParametrizedTestInstance, + OnyxOutputParserParametrizedTest, + ::testing::ValuesIn(onyxParamValueAndExpectedArgumentsVec), + [](const ::testing::TestParamInfo& info) { + std::string name = std::get<0>(info.param) + "_" + std::get<2>(info.param); + std::replace_if(name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_'); + return name; + });