From 4d9b42482ea8ce19152fdebe71ca293e3a22598f Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Mon, 27 Jul 2026 16:55:29 +0200 Subject: [PATCH 01/38] Model enablement WIP --- src/BUILD | 2 + src/llm/BUILD | 32 + .../io_processing/chat_template/analyzer.cpp | 23 + .../text_content_normalization_processor.cpp | 13 + .../text_content_normalization_processor.hpp | 12 +- .../onyx/onyx_reasoning_parser.cpp | 102 +++ .../onyx/onyx_reasoning_parser.hpp | 81 +++ .../io_processing/onyx/onyx_tool_parser.cpp | 183 ++++++ .../io_processing/onyx/onyx_tool_parser.hpp | 120 ++++ src/llm/io_processing/output_parser.cpp | 10 +- .../parser_config_validation.cpp | 2 + src/test/llm/chat_template_analyzer_test.cpp | 15 + ...emplate_and_parser_onyx_roundtrip_test.cpp | 178 +++++ .../chat_template_end_to_end_jinja_test.cpp | 62 ++ .../chat_template_end_to_end_minja_test.cpp | 139 ++++ .../chat_templates/chat_template_onyx.jinja | 41 ++ ...t_content_normalization_processor_test.cpp | 20 + .../onyx_output_parser_test.cpp | 619 ++++++++++++++++++ 18 files changed, 1648 insertions(+), 6 deletions(-) create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.hpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.hpp create mode 100644 src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp create mode 100644 src/test/llm/chat_templates/chat_template_onyx.jinja create mode 100644 src/test/llm/output_parsers/onyx_output_parser_test.cpp diff --git a/src/BUILD b/src/BUILD index 2f271b28c2..3e2e58f40d 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2909,12 +2909,14 @@ cc_library( "test/llm/chat_template_analyzer_test.cpp", "test/llm/chat_template_adapter_test.cpp", "test/llm/chat_template_end_to_end_minja_test.cpp", + "test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp", ], deps = [ "@com_google_googletest//:gtest", "//src/llm:chat_template_analyzer", "//src/llm:chat_template_probe", "//src/llm:io_processing_input_processors", + "//src/llm:output_parsers", "//src/utils:env_guard", "//third_party:genai", ":test_platform_utils", diff --git a/src/llm/BUILD b/src/llm/BUILD index d6b91852b1..0f83815b70 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -353,6 +353,36 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_onyx_tool_parser", + hdrs = ["io_processing/onyx/onyx_tool_parser.hpp"], + srcs = ["io_processing/onyx/onyx_tool_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstatus", + ":io_processing_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_onyx_reasoning_parser", + hdrs = ["io_processing/onyx/onyx_reasoning_parser.hpp"], + srcs = ["io_processing/onyx/onyx_reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( # TODO split further so we don't have to recompile everything when changing one parser ... name = "output_parsers", hdrs = [ @@ -390,6 +420,8 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_lfm2_tool_parser", ":io_processing_gemma4_tool_parser", ":io_processing_qwen3_reasoning_parser", + ":io_processing_onyx_tool_parser", + ":io_processing_onyx_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", ], diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index c3d20aca09..2248109192 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -37,6 +37,29 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } + // Onyx detection — Harmony-family framing ("<|start|>{role}[ to=]<|message|> + // {content}<|eom|>/<|eot|>") without gpt-oss's "<|channel|>" marker (already handled + // above). This combination of literal tokens is not used by any other template in + // this codebase (verified against every fixture under src/test/llm/chat_templates/). + // There is no separate reasoning-specific token: Onyx routes both tool calls + // (recipient="functions.") and private reasoning (recipient="self") through the + // exact same "<|message|>...<|eom|>" framing, so both parsers are tied together here, + // like gptoss/gemma4. + // NOTE: unlike every other branch below, this deliberately does NOT set + // caps.supportsToolCalls -- that flag means "the template natively re-serializes an + // incoming OpenAI-shaped `tool_calls` array back into the model's own format", which + // Onyx's template does not do at all (it only ever reads message['recipient']/ + // message['content'], never message['tool_calls'] -- see the Onyx_ToolCallWithStringArgs + // tests in chat_template_end_to_end_{jinja,minja}_test.cpp). detectedToolParser/ + // detectedReasoningParser only affect how OVMS parses the model's *output*, which is + // unrelated to (and unaffected by) whether input history round-trips correctly. + if (contains(templateSource, "<|start|>") && contains(templateSource, "<|message|>") && + contains(templateSource, "<|eom|>") && contains(templateSource, "<|eot|>")) { + result.detectedToolParser = "onyx"; + result.detectedReasoningParser = "onyx"; + return result; + } + // Gemma4 detection if (contains(templateSource, "'<|tool_call>call:'") || contains(templateSource, "<|tool_call>call:")) { result.detectedToolParser = "gemma4"; diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 2180d5faae..871a0f5f06 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -29,6 +29,19 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { ov::genai::ChatHistory& chatHistory = std::get(req.input); for (size_t i = 0; i < chatHistory.size(); i++) { const auto content = chatHistory[i]["content"]; + if (content.is_null()) { + // TODO @atobiszei to check if really needed when we have IR + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls" sets "content": null (openai_completions.cpp stores this + // verbatim -- only a *missing* content field is defaulted to ""). Some + // chat templates (e.g. Onyx's) unconditionally render content for every + // message regardless of role/tool_calls and are not written to expect + // null there, which raises a template error instead of just omitting + // the text. Normalize null the same way a missing field is already + // defaulted, so every template sees a plain string as before. + chatHistory[i]["content"] = std::string(""); + continue; + } if (!content.is_array()) { continue; } diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index 7280214920..b6292cb302 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,10 +19,14 @@ namespace ovms { -// Flattens text-only content arrays in ChatHistory messages to plain strings. -// Parts are joined with "\n" for backward compatibility with chat templates. -// Runs for both LM and VLM chat paths: arrays that contain images (or other -// non-text modalities) are left untouched for ImageDecodingProcessor. +// Flattens text-only content arrays in ChatHistory messages to plain strings, and +// normalizes an explicit "content": null (the standard OpenAI shape for e.g. an +// assistant message that only carries tool_calls) to "" -- some chat templates +// (e.g. Onyx's) unconditionally render content for every message and are not +// written to expect null there. Parts/null are joined/replaced for backward +// compatibility with chat templates. Runs for both LM and VLM chat paths: arrays +// that contain images (or other non-text modalities) are left untouched for +// ImageDecodingProcessor. // Must run before ChatTemplateProcessor. class TextContentNormalizationProcessor : public BaseInputProcessor { public: diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp new file mode 100644 index 0000000000..43b45a1e8f --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -0,0 +1,102 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "onyx_reasoning_parser.hpp" + +namespace ovms { + +void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // TODO @atobiszei overcomplicated I think? We just need t find recipient self & them cut that part out. + // Case 1: private chain-of-thought turn (recipient="self") -> extract reasoning, + // consume the whole segment (nothing meaningful is expected to follow it within the + // same generate() call, see class comment). + size_t selfPos = parsedOutput.content.find(selfRecipientTag); + if (selfPos != std::string::npos) { + size_t messagePos = parsedOutput.content.find(messageTag, selfPos); + if (messagePos != std::string::npos) { + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.reasoning = body; + // Drop the leading " " before "to=" (rendered by the chat template) too, if present. + size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; + parsedOutput.content.erase(segmentStart); + return; + } + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); + return; + } + + // Case 2: tool-call turn (recipient="functions.") -> leave untouched, OnyxToolParser + // (which runs after this parser, see OutputParser::parse()) is responsible for it. + if (parsedOutput.content.find(functionsRecipientTag) != std::string::npos) { + return; + } + + // Case 3: plain final answer (recipient="user" or absent) -> strip the generic + // " to=user"? + "<|message|>" + "<|eot|>" envelope, leaving just the clean text. + size_t messagePos = parsedOutput.content.find(messageTag); + if (messagePos == std::string::npos) { + // No framing found at all -- unexpected/malformed output, leave content as-is. + return; + } + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(turnFinalEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.content = body; +} + +std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // TODO @atobiszei we need to stream between recipient=self & + // TODO: streaming support is a first draft. It only forwards the chunk as + // reasoning_content once we've seen the "to=self" start tag; stripping the + // generic final-answer envelope (Case 3 above) in streaming mode is not + // implemented yet and needs its own design (the envelope prefix/suffix can + // straddle multiple chunks). + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); + return std::nullopt; + } + if (chunk.find(selfRecipientTag) != std::string::npos || + chunk.find(messageTag) != std::string::npos || + chunk.find(continuationEndTag) != std::string::npos) { + return std::nullopt; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; +} +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp new file mode 100644 index 0000000000..27218d8414 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -0,0 +1,81 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +// Onyx (early preview model) framing: +// TODO @atobiszei simplify comment. tag naming convention. no need to define all tags here +// <|start|>assistant[ to=]<|message|>{content}{<|eom|>|<|eot|>} +// The chat template never emits a ""-style dedicated reasoning tag: private +// chain-of-thought is just an assistant turn routed with recipient="self", ending in +// the continuation marker "<|eom|>" (never "<|eot|>", which is reserved for turns that +// end the whole assistant turn -- i.e. the final answer). +// +// Because generation stops at the first "<|eom|>"/"<|eot|>"/"<|end_of_text|>" (see +// generation_config.json's eos_token_id list in the Onyx HF conversion script), a single +// generate() call only ever produces ONE such framed segment. This parser is therefore +// also responsible for stripping the generic " to="+"<|message|>"+terminator +// envelope from plain final-answer turns (recipient="user" or absent) -- this class runs +// before the tool parser (see OutputParser::parse()), so it must NOT touch content when +// the envelope routes to a function call (recipient="functions."); it leaves that +// segment untouched so OnyxToolParser can find and parse it afterwards. +class OnyxReasoningParser : public BaseOutputParser { +protected: + // Marks a private chain-of-thought turn (recipient="self"). + const std::string selfRecipientTag = "to=self"; + // Marks a tool-call turn (recipient="functions.") -- left untouched here. + const std::string functionsRecipientTag = "to=functions."; + // Separates the routing prefix from the turn's body. + const std::string messageTag = "<|message|>"; + // Terminator for continuation turns (reasoning and tool calls). + const std::string continuationEndTag = "<|eom|>"; + // Terminator for turn-final turns (plain final answers). + const std::string turnFinalEndTag = "<|eot|>"; + +public: + OnyxReasoningParser() = delete; + explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{selfRecipientTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return continuationEndTag; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp new file mode 100644 index 0000000000..76b6a65e49 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -0,0 +1,183 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "src/llm/io_processing/utils.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" + +namespace ovms { + +const std::string OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG = "to=functions."; +const std::string OnyxToolParserImpl::MESSAGE_TAG = "<|message|>"; +const std::string OnyxToolParserImpl::END_TAG = "<|eom|>"; + +#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ + auto pos = this->streamContent.find(TAG, this->lastProcessedPosition); \ + if (pos == std::string::npos) { \ + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Did not find: {}", TAG); \ + break; \ + } + +bool OnyxToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + auto previousState = this->currentState; + switch (this->currentState) { + case State::Content: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(FUNCTIONS_RECIPIENT_TAG); + this->toolCallPositions.begin.push(pos); + this->lastProcessedPosition = pos + FUNCTIONS_RECIPIENT_TAG.length(); + this->currentState = State::InsideName; + break; + } + case State::InsideName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(MESSAGE_TAG); + this->currentFunctionName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + MESSAGE_TAG.length(); + this->currentState = State::InsideArguments; + break; + } + case State::InsideArguments: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(END_TAG); + std::string argumentsPart = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + END_TAG.length(); + this->currentState = State::Content; + this->toolCallPositions.end.push(this->lastProcessedPosition); + ToolCall toolCall{generateRandomId(), this->currentFunctionName, argumentsPart}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Adding tool call: id={}, name={}, arguments={}", toolCall.id, toolCall.name, toolCall.arguments); + toolCalls.emplace_back(std::move(toolCall)); + this->currentFunctionName.clear(); + break; + } + } + return previousState != this->currentState; +} + +std::optional OnyxToolParserImpl::parseChunk(const std::string& chunk) { + if (chunk.empty()) { + return std::nullopt; + } + ToolCalls_t toolCalls; + this->streamContent += chunk; + while (parseUntilStateChange(toolCalls)) { + } + if (!toolCalls.empty()) { + return std::move(toolCalls); + } + return std::nullopt; +} + +std::optional OnyxToolParserImpl::getCurrentFunctionName() const { + if (this->currentFunctionName.empty()) { + return std::nullopt; + } + return this->currentFunctionName; +} + +Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); + return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); + } + while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { + auto posBegin = toolCallPositions.begin.top(); + auto posEnd = toolCallPositions.end.top(); + // Also consume the leading " " the chat template renders before "to=" (a single + // generate() call only ever produces one such segment, see OnyxReasoningParser's + // class comment for why this can't collide with anything preceding it). + if (posBegin > 0 && outContent[posBegin - 1] == ' ') { + posBegin -= 1; + } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); + outContent.erase(posBegin, posEnd - posBegin); + toolCallPositions.begin.pop(); + toolCallPositions.end.pop(); + } + return StatusCode::OK; +} + +void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> + // + // Mirrors Qwen3CoderToolParser::parse(): drive the same streamParser used for + // streaming with the whole content as a single chunk, and reuse whatever it + // assembled -- unary is a single-shot edge case of streaming, not a parallel + // reimplementation of the tag walk. + auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); + if (!toolCallsOpt.has_value()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Parsing ended, no tool calls found"); + return; + } + parsedOutput.toolCalls = std::move(toolCallsOpt.value()); + for (const auto& toolCall : parsedOutput.toolCalls) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); + } + auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to remove tool calls from content: {}", status.string()); + } +} + +std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { + if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { + // already sent the first delta for the function currently being read + return std::nullopt; + } + int currentToolCallIndex = ++this->toolCallIndex; + rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); + this->returnedFirstDeltas.insert(currentToolCallIndex); + return doc; +} + +std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { + // ASSUMPTION: mirroring Qwen3CoderToolParser, in streaming we only ever complete one + // tool call per parseChunk() call -- there is no way to send multiple tool calls to + // the client in a single streaming delta. + if (toolCalls.size() != 1) { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "For streaming we expected one tool call, got: {}", toolCalls.size()); + throw std::runtime_error("For streaming we expected one tool call"); + } + const auto& toolCall = toolCalls[0]; + this->returnedCompleteDeltas.insert(this->toolCallIndex); + rapidjson::Document argumentsWrapper; + argumentsWrapper.SetObject(); + rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); + argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); + return wrapDelta(argumentsWrapper, this->toolCallIndex); +} + +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // streamParser returns assembled toolCalls once a call closes ("<|eom|>" seen); until + // then, if the function name is already known, send the first delta for it once. + if (newChunk.empty()) { + return std::nullopt; + } + auto toolCallsOpt = this->streamParser.parseChunk(newChunk); + if (toolCallsOpt.has_value()) { + return this->sendFullDelta(toolCallsOpt.value()); + } + auto functionNameOpt = this->streamParser.getCurrentFunctionName(); + if (functionNameOpt.has_value()) { + return this->sendFirstDeltaIfNeeded(functionNameOpt.value()); + } + return std::nullopt; +} +} // namespace ovms + diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp new file mode 100644 index 0000000000..d8d48b6bc1 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -0,0 +1,120 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/status.hpp" + +namespace ovms { + +// Onyx (early preview model) tool-call framing: +// TODO @atobiszei is functions namespace always "functions"? +// <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> +// Unlike qwen3coder/hermes3, Onyx never wraps arguments in a schema-validated, +// per-parameter structure -- the segment between "<|message|>" and "<|eom|>" is +// already the complete, raw JSON arguments blob the caller is expected to forward +// as-is (per the model card: "the SFT tokenizer tokenizes message content ... +// (raw body)"). So no tool-schema-driven type coercion is needed here, unlike +// Qwen3CoderToolParser. + +// Pure state machine that accumulates raw generated text and hands back fully +// assembled tool calls -- mirrors Qwen3CoderToolParserImpl's split between "parse the +// framing" and "turn it into OpenAI delta JSON" (done by the owning OnyxToolParser). +// Because Onyx's arguments are already a complete raw JSON blob (no per-parameter +// schema coercion needed), a tool call is fully known as soon as its end tag is seen -- +// unlike Qwen3Coder there is no incremental per-parameter streaming to do. +struct OnyxToolParserImpl { + enum class State { + Content, // looking for the next "to=functions." recipient tag + InsideName, // accumulating the function name, looking for messageTag + InsideArguments // accumulating the raw JSON arguments blob, looking for endTag + }; + + // Marks the start of a tool-call turn; the function name follows immediately. + static const std::string FUNCTIONS_RECIPIENT_TAG; + // Separates the function name from the raw JSON arguments blob. + static const std::string MESSAGE_TAG; + // Tool calls always end the turn as a continuation (never a full turn end). + static const std::string END_TAG; + + // Return all tool calls fully closed (end tag seen) in the aggregated content so far + // that were not returned before -- nullopt if none completed yet. + std::optional parseChunk(const std::string& chunk); + std::optional getCurrentFunctionName() const; + Status removeToolCallsFromContentIfNeeded(std::string& outContent); + +private: + State currentState = State::Content; + std::string streamContent; // content accumulated from stream chunks + size_t lastProcessedPosition{0}; + std::string currentFunctionName; + struct ToolCallPositions { + std::stack begin; + std::stack end; + }; + ToolCallPositions toolCallPositions; + + // Process streamContent from lastProcessedPosition until a state change happens; + // return true if the state changed (caller should keep looping), false once no more + // progress is possible with the currently available content. + bool parseUntilStateChange(ToolCalls_t& toolCalls); +}; + +class OnyxToolParser : public BaseOutputParser { +private: + // for streaming parsing we need to keep the parser as a member + OnyxToolParserImpl streamParser; + int toolCallIndex{-1}; + std::set returnedFirstDeltas; + std::set returnedCompleteDeltas; + + std::optional sendFirstDeltaIfNeeded(const std::string& functionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + +public: + OnyxToolParser() = delete; + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return OnyxToolParserImpl::END_TAG; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } + +}; +} // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index e676f2bea6..a51da1d108 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -17,8 +17,8 @@ #include #include -#include "../../logging.hpp" -#include "../../stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" #include "output_parser.hpp" #include "parser_config_validation.hpp" #include "llama3/tool_parser.hpp" @@ -33,6 +33,8 @@ #include "gptoss/reasoning_parser.hpp" #include "lfm2/lfm2_tool_parser.hpp" #include "gemma4/gemma4_tool_parser.hpp" +#include "onyx/onyx_tool_parser.hpp" +#include "onyx/onyx_reasoning_parser.hpp" namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { @@ -196,6 +198,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to toolParser = std::make_unique(tokenizer); } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "onyx") { + toolParser = std::make_unique(tokenizer); } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -207,6 +211,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "onyx") { + reasoningParser = std::make_unique(tokenizer); } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index 31d645cadc..8bd1508375 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -32,6 +32,7 @@ const std::vector& getSupportedToolParserNames() { "devstral", "lfm2", "gemma4", + "onyx", }; return names; } @@ -41,6 +42,7 @@ const std::vector& getSupportedReasoningParserNames() { "qwen3", "gemma4", "gptoss", + "onyx", }; return names; } diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index e889cff35e..4bdd3cb195 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -61,6 +61,21 @@ TEST_F(ChatTemplateAnalyzerTest, detectsGptOss) { EXPECT_TRUE(result.caps.supportsToolCalls); } +// --- Onyx --- + +TEST_F(ChatTemplateAnalyzerTest, detectsOnyx) { + std::string tmpl = loadTemplate("chat_template_onyx.jinja"); + ASSERT_FALSE(tmpl.empty()); + auto result = ChatTemplateAnalyzer::analyze(tmpl); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "onyx"); + // Onyx's template never reads the OpenAI "tool_calls" array, so unlike every + // other detected family, supportsToolCalls stays false -- see analyzer.cpp. + EXPECT_FALSE(result.caps.supportsToolCalls); +} + // --- Gemma4 --- TEST_F(ChatTemplateAnalyzerTest, detectsGemma4) { diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp new file mode 100644 index 0000000000..b253081edf --- /dev/null +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -0,0 +1,178 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#pragma GCC diagnostic pop + +#include "src/llm/io_processing/output_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// ============================================================================= +// Genuine request/response round trip for Onyx: render a request with the real +// chat template (minja, via ov::genai::Tokenizer::apply_chat_template -- same +// engine exercised by ChatTemplateEndToEndMinjaTest), hand-author the exact +// continuation the model would emit for that rendered prompt (grounded in what +// ChatTemplateEndToEndMinjaTest's Onyx_* tests already proved the template +// produces), and feed ONLY that continuation into OutputParser -- exactly what +// OVMS does in production (the parser only ever sees newly generated tokens, +// never the prompt). This is the missing link between: +// - chat_template_end_to_end_{minja,jinja}_test.cpp (request side only) +// - output_parsers/onyx_output_parser_test.cpp (response side only, with +// hand-written segments not derived from an actual rendered prompt) +// ============================================================================= +class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { +protected: + // TODO @atobiszei change tokenizer for onyx + const std::string& tokenizerPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m", false); + const std::string& chatTemplatesPath = getGenericFullPathForSrcTest("/ovms/src/test/llm/chat_templates", false); + + static std::string loadTemplateFile(const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) { + return ""; + } + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + } + + // Renders chatHistory with the real Onyx template via minja and asserts the + // generation prompt ends with the bare "<|start|>assistant" the parser tests + // assume (no trailing "<|message|>", no implicit recipient). + std::string renderPrompt(ov::genai::ChatHistory& chatHistory) { + std::string chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + ov::genai::Tokenizer tokenizer(tokenizerPath); + tokenizer.set_chat_template(chatTemplate); + std::string rendered = tokenizer.apply_chat_template(chatHistory, /*add_generation_prompt=*/true); + static const std::string generationPromptTail = "<|start|>assistant"; + EXPECT_TRUE(rendered.size() >= generationPromptTail.size() && + rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) + << "Generation prompt tail changed, Onyx parser assumptions may be stale: " << rendered; + return rendered; + } + + // Simulates generation: appends modelContinuation to the rendered prompt (for + // documentation / sanity only) and runs OutputParser on modelContinuation alone, + // matching what OVMS actually hands the parser. + ParsedOutput parseModelContinuation(const std::string& modelContinuation, bool toolsAvailable = true) { + ov::genai::Tokenizer tokenizer(tokenizerPath); + auto generatedTensor = tokenizer.encode(modelContinuation, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + + static ToolsSchemas_t emptyToolsSchema{}; // Onyx tool parser is not schema-driven, see onyx_tool_parser.hpp + OutputParser outputParser(tokenizer, "onyx", "onyx", emptyToolsSchema); + return outputParser.parse(generatedTokens, toolsAvailable); + } +}; + +// ============================================================================= +// Turn 1 of the muse/README.md "get_weather" example: user asks a question, the +// prompt is rendered, and the model's tool-call continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("You can call get_weather(city)."), std::string::npos) << prompt; + + // Model continuation for a tool call, exactly as documented in muse/README.md. + std::string modelContinuation = R"( to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city": "SF"})"); +} + +// ============================================================================= +// Turn 2 of the same example: tool result fed back into history (Onyx's own +// "name" + role="tool" shape, NOT OpenAI's tool_call_id), then the model's final +// answer continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinalAnswer) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"city\": \"SF\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"functions.get_weather","content":"{\"temp\": 65}"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; + // NOTE (important, non-obvious): OpenVINO GenAI's own minja-path history + // preprocessing -- independent of the raw Jinja template's `elif role == + // 'tool'` branch -- rewrites role="tool" messages into role="user" with a + // generic wrapped "tool_response" JSON object whenever it determines the + // template lacks native tool-call support (the same probe underlying + // caps.supportsToolCalls == false, see chat_template_end_to_end_minja_test.cpp's + // Onyx tests). So Onyx's own `elif role == 'tool'` template branch is + // effectively DEAD CODE on the minja path today -- it never actually fires. + EXPECT_NE(prompt.find(R"(<|start|>user<|message|>{ + "tool_response": { + "tool": "functions.get_weather", + "content": "{\"temp\": 65}" + } +}<|eot|>)"), + std::string::npos) + << prompt; + + std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Private reasoning ("recipient": "self") round trip: the model reasons first, +// which the OnyxReasoningParser must classify as reasoning, not content. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsPrivateReasoning) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's 2+2?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("What's 2+2?"), std::string::npos) << prompt; + + std::string modelContinuation = R"( to=self<|message|>2+2 is a basic addition.<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "2+2 is a basic addition."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 3ec42b0f94..4ac2f0c62b 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -532,3 +532,65 @@ What's the weather in Paris?<|im_end|> )"; EXPECT_EQ(appliedOutput, expectedOutput); } + +// ============================================================================= +// Onyx (early preview model) chat template, rendered via the real Python Jinja2 +// engine. Onyx's template does not read the standard OpenAI "tool_calls" array +// at all -- only message['content'] (plain string) and an Onyx-specific +// message['recipient'] field (e.g. "functions.get_weather", "self", "user"). +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp), +// but deliberately leaves caps.supportsToolCalls false: that flag means "this +// template natively re-serializes an incoming OpenAI tool_calls array", which +// this test demonstrates Onyx's template does NOT do (the tool call is silently +// dropped below). +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // Unlike the minja path (which has its own generic tool-call fallback), + // the real Python Jinja2 engine has no such fallback: the template renders + // message['content'] literally, i.e. the empty string, and the tool call + // information is silently dropped. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|><|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape via Python Jinja2: "recipient" field instead of the +// OpenAI "tool_calls" array. Ends the turn with "<|eom|>" (continuation marker) +// rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index 83e057275a..b25f01640a 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -488,6 +488,145 @@ What's the weather in Paris?<|im_end|> EXPECT_EQ(appliedOutput, expectedOutput); } +// ============================================================================= +// Onyx (early preview model) chat template. Unlike every other template in this +// suite, Onyx's own Jinja template does not consume the standard OpenAI +// "tool_calls" list at all -- it only reads message['content'] (a plain string) +// and an Onyx-specific message['recipient'] field (e.g. "functions.get_weather", +// "self", "user"). Feeding it a standard tool_calls-shaped assistant message +// therefore renders an effectively empty assistant turn. +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp) and +// sets detectedToolParser/detectedReasoningParser, but deliberately leaves +// caps.supportsToolCalls false since the template can't natively round-trip an +// OpenAI tool_calls history (demonstrated by this very test), so no input-side +// workaround is applied either -- detectedToolParser only affects output parsing. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // The template itself never reads "tool_calls" (it only looks at + // message['content'] and message['recipient']). Because caps.supportsToolCalls + // is false here, OVMS does not apply its own tool-call workaround either. + // Minja's own generic fallback (used for templates it detects have no native + // tool-call rendering) kicks in instead and serializes the whole message + // (tool_calls + content) as a JSON blob into message['content'] -- the + // function name/args are NOT lost, but they end up as raw, unparsed JSON text + // rather than in Onyx's native " to=functions." / <|eom|> framing. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|>{ + "tool_calls": [ + { + "name": "get_weather", + "arguments": { + "location": "Paris", + "unit": "celsius" + }, + "id": "call_abc123" + } + ], + "content": "" +}<|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape: instead of the OpenAI "tool_calls" array, the +// assistant turn carries a "recipient" field (here "functions.get_weather") +// and a plain-string content holding the raw JSON arguments. This is the shape +// Onyx's template actually understands, ending the turn with "<|eom|>" (a +// continuation marker) rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Full-scope round trip: exercises every message shape the Onyx template +// natively understands in a single history, not just one shape in isolation -- +// user prompt -> assistant tool call (recipient=functions., continuation +// "<|eom|>") -> tool call response (role="tool") -> assistant final answer +// (recipient="user", "<|eot|>"). +// +// This surfaces a second, previously undocumented gap alongside the tool_calls +// one above (see muse/chat_template_issues.md): because caps.supportsToolCalls +// is false for Onyx, ChatTemplateAdapter's generic fallback also intercepts +// plain role="tool" messages -- not just assistant tool_calls -- and rewrites +// them into a synthetic role="user" message serializing {tool, content} as a +// JSON blob, rather than passing them through to the template's own native +// "tool"-role branch (which expects message['name'] + message['content'] and +// would render "<|start|>tool <|message|>...<|eot|>"). So even a chat +// history built entirely out of Onyx's own native fields (recipient) still +// does not round-trip once a plain OpenAI-shaped tool response message is +// mixed in. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"get_weather","content":"{\"temperature\":15,\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"user","content":"It's 15C in Paris."})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + EXPECT_FALSE(caps.supportsToolCalls); + + // Known gap (see class comment above): the "tool" message is NOT rendered via + // the template's native "<|start|>tool <|message|>...<|eot|>" branch -- + // ChatTemplateAdapter's fallback rewrites it into a synthetic user message + // carrying a JSON blob first. + std::string expectedOutput = + R"(<|start|>system<|message|>You are a helpful assistant.<|eot|>)" + R"(<|start|>user<|message|>What's the weather in Paris?<|eot|>)" + R"(<|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|>)" + "<|start|>user<|message|>{\n" + " \"tool_response\": {\n" + " \"tool\": \"get_weather\",\n" + " \"content\": \"{\\\"temperature\\\":15,\\\"unit\\\":\\\"celsius\\\"}\"\n" + " }\n" + "}<|eot|>" + R"(<|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|>)" + R"(<|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. diff --git a/src/test/llm/chat_templates/chat_template_onyx.jinja b/src/test/llm/chat_templates/chat_template_onyx.jinja new file mode 100644 index 0000000000..0fa7b9d204 --- /dev/null +++ b/src/test/llm/chat_templates/chat_template_onyx.jinja @@ -0,0 +1,41 @@ +{{- bos_token -}} +{%- macro render_parts(content) -%} +{%- if content is string -%}{{- content -}} +{%- else -%} +{%- for part in content -%} +{%- if part['type'] == 'image' -%}{{- '<|image|>' -}} +{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}} +{%- elif part['type'] == 'text' -%}{{- part['text'] -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%}{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}{%- endfor -%} +{%- if add_generation_prompt and not ns.has_system -%} +{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}} +{%- endif -%} +{%- for message in messages -%} +{%- set role = message['role'] -%} +{%- if role == 'assistant' -%} +{%- set recipient = message.get('recipient') -%} +{%- set end_turn = message.get('end_turn') -%} +{%- if end_turn is none -%} +{%- set end_turn = not (recipient and recipient != 'user') -%} +{%- endif -%} +{{- '<|start|>assistant' -}} +{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%} +{{- '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- ('<|eot|>' if end_turn else '<|eom|>') -}} +{%- elif role == 'tool' -%} +{%- set name = message.get('name', '') -%} +{{- '<|start|>tool ' + name + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- else -%} +{%- set header = role -%} +{%- if message.get('name') -%}{%- set header = role + ' ' + message['name'] -%}{%- endif -%} +{{- '<|start|>' + header + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%} diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index 2b1840f0de..43ae0a6e1b 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -99,3 +99,23 @@ TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { ASSERT_TRUE(result[0]["content"].is_array()); EXPECT_EQ(result[0]["content"].size(), 2u); } + +TEST(TextContentNormalizationProcessorTest, NullContentNormalizedToEmptyString) { + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls": content is explicitly null (not just absent). Some templates + // (e.g. Onyx's) unconditionally render content for every message and error out + // on null, so this must be normalized to "" the same way a missing field is. + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("assistant")}}; + msg["content"] = ov::genai::JsonContainer(nullptr); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + TextContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + ASSERT_TRUE(result[0]["content"].is_string()); + EXPECT_EQ(result[0]["content"].as_string().value_or("__unset__"), ""); +} diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp new file mode 100644 index 0000000000..4de64ee3f0 --- /dev/null +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -0,0 +1,619 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" +#include "src/llm/io_processing/output_parser.hpp" +#include "src/logging.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// Onyx does not ship a converted HF tokenizer in this early preview, and none of the +// segments the parser looks for ("to=functions.", "<|message|>", "<|eom|>", "<|eot|>", +// "to=self") are real special tokens of the model this parser is designed for -- they are +// plain text sequences that must round-trip losslessly through encode()+decode() on ANY +// tokenizer. facebook/opt-125m is already used the same way for chat-template testing +// (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid pulling in a new +// model fixture. +// TODO @atobiszei replace when tokenizer is available +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/facebook/opt-125m"; +#endif + +static std::unique_ptr opt125mTokenizer; + +// Onyx never consults tool schemas (arguments are forwarded as a raw JSON blob verbatim, +// no per-parameter type coercion like Qwen3CoderToolParser) -- these mirror +// Qwen3CoderOutputParserTest's toolSchemasInput/toolsSchemas setup purely so the test +// fixture shape matches, and so a real schema is on hand if that ever changes. +static std::map toolSchemasInput = { + {"get_weather", R"({"properties":{"location":{"type":"string","description":"City name."},"unit":{"type":"string","description":"Temperature unit."}},"required":["location"]})"}, + {"get_time", R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"}, + {"get_current_location", R"({"properties":{},"required":[]})"}, + {"string_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."}},"required":["arg1"]})"}, + {"cd", R"({"properties":{"folder":{"type":"string","description":"Path"}},"required":["folder"]})"}, + {"string_int_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."},"arg2":{"type":"integer","description":"An integer argument."}},"required":["arg1","arg2"]})"}}; + +static std::vector> schemaDocsStorage; + +static ToolsSchemas_t convertStringToolSchemasStringToToolsSchemas( + const std::map& input) { + ToolsSchemas_t result; + schemaDocsStorage.clear(); + for (const auto& [name, schemaStr] : input) { + auto schemaDoc = std::make_unique(); + if (schemaDoc->Parse(schemaStr.c_str()).HasParseError()) { + throw std::runtime_error("Failed to parse schema for tool: " + name); + } + result[name] = {schemaDoc.get(), schemaStr}; + schemaDocsStorage.push_back(std::move(schemaDoc)); + } + return result; +} + +static ovms::ToolsSchemas_t toolsSchemas = convertStringToolSchemasStringToToolsSchemas(toolSchemasInput); + +class OnyxOutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParser; + + static void SetUpTestSuite() { + try { + opt125mTokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize opt-125m tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize opt-125m tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + opt125mTokenizer.reset(); + } + + void SetUp() override { + outputParser = std::make_unique(*opt125mTokenizer, "onyx", "onyx", toolsSchemas); + } + + ParsedOutput generateParsedOutput(const std::string& input, bool toolsAvailable = true) { + auto generatedTensor = opt125mTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + return outputParser->parse(generatedTokens, toolsAvailable); + } + + // Shared by streaming tests: compares a parseChunk() delta against the expected JSON, + // masking the randomly generated tool call id (kept as one helper rather than the same + // id-masking block duplicated per test, mirroring Qwen3CoderOutputParserTest usage). + static void assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk); + + // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, + // using rapidjson's serializer to handle all escaping. This lets us write raw PLC/Python + // code in tests without manually counting backslashes. + static std::string wrapRawCodeAsToolArgs(const std::string& rawCode) { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Value val(rawCode.c_str(), static_cast(rawCode.size()), doc.GetAllocator()); + doc.AddMember("arg1", val, doc.GetAllocator()); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + doc.Accept(writer); + return buf.GetString(); + } + + // Builds the expected arguments delta JSON string for a given tool call index and + // raw (unescaped) code content: {"delta":{"tool_calls":[{"index":N,"function":{"arguments":"..."}}]}} + static std::string expectedArgsDelta(int index, const std::string& rawCode) { + std::string argsJson = wrapRawCodeAsToolArgs(rawCode); + // argsJson is a valid JSON object string -- we need to embed it as an escaped + // string value inside the outer delta JSON. Use rapidjson to build the whole thing. + rapidjson::Document outer; + outer.SetObject(); + rapidjson::Document::AllocatorType& alloc = outer.GetAllocator(); + rapidjson::Value delta(rapidjson::kObjectType); + rapidjson::Value toolCalls(rapidjson::kArrayType); + rapidjson::Value tc(rapidjson::kObjectType); + tc.AddMember("index", index, alloc); + rapidjson::Value func(rapidjson::kObjectType); + rapidjson::Value argsVal(argsJson.c_str(), static_cast(argsJson.size()), alloc); + func.AddMember("arguments", argsVal, alloc); + tc.AddMember("function", func, alloc); + toolCalls.PushBack(tc, alloc); + delta.AddMember("tool_calls", toolCalls, alloc); + outer.AddMember("delta", delta, alloc); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + outer.Accept(writer); + return buf.GetString(); + } +}; + +// A single generate() call stops at the first "<|eom|>"/"<|eot|>" (both are configured as +// eos tokens for Onyx), so only one of these three segment shapes is ever produced at a time. + +TEST_F(OnyxOutputParserTest, FinalAnswerWithExplicitUserRecipient) { + ParsedOutput parsedOutput = generateParsedOutput(" to=user<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, FinalAnswerWithoutRecipient) { + // The template only renders " to=" when the caller supplies a recipient; the model may + // also just emit "<|message|>...<|eot|>" directly with no recipient at all. + ParsedOutput parsedOutput = generateParsedOutput("<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { + ParsedOutput parsedOutput = generateParsedOutput(" to=self<|message|>Let me think about this.<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "Let me think about this."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithRawJsonArguments) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + // Onyx passes arguments through verbatim -- no schema-driven reformatting like qwen3coder. + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { + // OutputParser::parse() only invokes the tool parser when toolsAvailable is true. + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>", /*toolsAvailable=*/false); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); + // Known current limitation: the reasoning parser (which always runs) intentionally + // leaves "to=functions." segments untouched so the tool parser can claim them -- but + // if the tool parser never runs, the raw wrapped segment is surfaced as-is in content. + EXPECT_EQ(parsedOutput.content, " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"); +} + +TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { + ParsedOutput parsedOutput = generateParsedOutput("some unexpected raw text without any framing"); + + EXPECT_EQ(parsedOutput.content, "some unexpected raw text without any framing"); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Streaming is implemented on top of OnyxToolParserImpl, a pure state machine that +// accumulates raw text and hands back a fully assembled tool call once its "<|eom|>" +// end tag is seen (mirroring Qwen3CoderToolParserImpl) -- unary parse() below drives +// that same impl with the whole content as a single chunk, so it is the single-shot +// degenerate case of streaming, not a parallel implementation of the tag walk. +// +// Because Onyx's arguments are already a complete raw JSON blob needing no per-parameter +// schema coercion, they are still sent to the client as a single delta once the tool call +// closes (matching Qwen3CoderToolParser's sendFullDelta) rather than streamed incrementally +// as raw text arrives -- only the function name streams as its own delta once known. +// +// Chunk boundaries below are deliberately awkward (splitting the function name and the +// JSON arguments mid-token) to exercise the Content/InsideName/InsideArguments state +// machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// ============================================================================= +void OnyxOutputParserTest::assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + if (!expectedDelta.has_value()) { + EXPECT_FALSE(doc.has_value()) << "Expected nullopt for chunk: " << chunk; + return; + } + ASSERT_TRUE(doc.has_value()) << "Expected a delta for chunk: " << chunk; + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + docStr.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expected.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + } + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; +} + +TEST_F(OnyxOutputParserTest, StreamingSimpleToolCall) { + // Mirrors Qwen3CoderOutputParserTest.StreamingSimpleToolCall's rigor: adversarial + // chunk boundaries, content before/between tool calls, complex argument values + // (PLC structured text, Python with triple-quotes/f-strings/escape sequences), + // adapted to Onyx's "to=functions.<|message|><|eom|>" format. + // + // Unlike qwen3coder there is no per-parameter incremental streaming to test + // (qwen3coder's tags) since Onyx's arguments are a single raw JSON blob. + // However, content before/between tool calls IS tested because the OutputParser + // streaming framework handles that generically (UNKNOWN -> CONTENT transition when no + // start tag is found). + // + // Key structural differences from qwen3coder: + // - Start tag is "to=functions." (not "") + // - Name delimiter is "<|message|>" (not ">") + // - End tag is "<|eom|>" (not "") + // - Arguments are a single raw JSON blob (not per-parameter XML tags) + // - PLC/Python code must be JSON-escaped within the arguments blob + + // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). + // Written as a raw string literal so it's human-readable; wrapRawCodeAsToolArgs() + // handles all the JSON escaping at runtime. + const std::string plcCode = R"(FUNCTION FC_CreateJsonPayload : STRING +VAR_INPUT + Value1 : REAL; + Value2 : INT; + Value3 : BOOL; + Value4 : STRING(100); +END_VAR +VAR_OUTPUT + JsonPayload : STRING(1000); +END_VAR +VAR + TempStr : STRING(100); +END_VAR + + JsonPayload := '{'; + JsonPayload := JsonPayload + '"value1":' + REAL_TO_STRING(Value1, '', 2) + ','; + JsonPayload := JsonPayload + '"value2":' + INT_TO_STRING(Value2) + ','; + JsonPayload := JsonPayload + '"value3":' + BOOL_TO_STRING(Value3) + ','; + JsonPayload := JsonPayload + '"value4":"' + Value4 + '"'; + JsonPayload := JsonPayload + '}'; + +END_FUNCTION)"; + + // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). + const std::string pythonCode = R"( +if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r'))"; + + int i = -1; + std::vector>> chunkToDeltaVec{ + // Content before any tool call -- OutputParser sees no start tag match, emits content. + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + // Start tag "to=functions." split across several arbitrarily small chunks. + // Note: leading space before "to=" is just normal content/separator; the start + // tag the framework looks for is "to=functions." without the space. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=fun", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ctions.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Function name streams in across several small chunks -- still no delta + // (OnyxToolParserImpl is in InsideName state, accumulating). + {"get", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"_", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"weath", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"er", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|message|>" itself split mid-tag -- name delta emitted once the full tag lands. + {"<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Raw JSON argument text (with a nested object) split at awkward byte boundaries. + {"{\"locat", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ion\":\"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ris\",\"opt", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ions\":{\"unit", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\":\"cel", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sius\"}}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|eom|>" split mid-tag -- closes the tool call once complete. + {"<|e", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"om|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"options\":{\"unit\":\"celsius\"}}"}}]}})"}, + // Content between tool calls (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). + // In TOOL_CALLS_WAITING_FOR_TOOL phase, text without start tag match waits for more. + {"POTENTIALLY EXISINT CONTENT", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Second tool call -- start tag + name + <|message|> split across tiny chunks. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=functi", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ons.str", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ing_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|messa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ge|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, + // Arguments split across chunks. + {"{\"arg1\":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"NG_VALUE\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eo", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"m|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, + // More content between tool calls (mirrors "CONTENT_AFTER_TOOL_CALL"). + {"CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Third tool call -- string_int_tool with two parameters in JSON (integer stays + // numeric). Start tag + name + <|message|> split differently from previous calls. + {" to=func", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"tions.strin", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g_int_tool<|", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, + // Arguments with a leading \n in the string value (matches qwen3coder's + // "\nANOTHER_STRING_VALUE" pattern) and an integer parameter. + {"{\"arg1\":\"\\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_STRING_VALUE\",\"ar", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g2\":314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"1522}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, + // "NOTHING IMPORTANT HERE" content between calls (mirrors qwen3coder). + {"NOTHING IMPORTANT HERE", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // A "bfcl draft" style call -- cd tool. Start tag arrives with some preceding + // text just like qwen3coder's "part of bfcl 'draft'.\n\n\n" pattern. + {"part of bfcl 'draft'.\n\n to=functions.cd<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, + {"{\"folder\":\"ResearchDocs\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, + // PLC structured text code as a tool argument (mirrors qwen3coder's + // FC_CreateJsonPayload test). Raw code is defined above as plcCode; the helper + // wrapRawCodeAsToolArgs() handles all JSON escaping via rapidjson so we don't + // need to manually count backslashes. Sent as a single chunk since the interesting + // escaping complexity is in the content, not in chunk-boundary splitting. + {" to=functions.string_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(plcCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, + // Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). Also sent as a single chunk -- the chunk-boundary + // adversarial testing is covered by the earlier tool calls above. + {" to=functions.string_tool<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(pythonCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk[" << i << "]: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk[" << i << "]: " << chunk; + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "] (ignoring id value): " << chunk; + } else { + SPDLOG_ERROR("Expected:\n{}", expected); + SPDLOG_ERROR("Got:\n{}", docStr); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + // Validate that arguments fields are valid JSON + if (expected.find("arguments") != std::string::npos) { + auto docJsonIt = doc->FindMember("delta"); + ASSERT_NE(docJsonIt, doc->MemberEnd()); + auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); + ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); + for (const auto& toolCall : toolCallsIt->value.GetArray()) { + auto functionIt = toolCall.FindMember("function"); + ASSERT_NE(functionIt, toolCall.MemberEnd()); + auto argumentsIt = functionIt->value.FindMember("arguments"); + ASSERT_NE(argumentsIt, functionIt->value.MemberEnd()); + const std::string& argumentsStr = argumentsIt->value.GetString(); + rapidjson::Document argsDoc; + argsDoc.Parse(argumentsStr.c_str()); + EXPECT_FALSE(argsDoc.HasParseError()) << "Arguments is not valid JSON for chunk[" << i << "]: " << chunk << "\nArguments string:\n" + << argumentsStr; + } + } + } + } else { + EXPECT_TRUE(false) << "Mismatch between expectedDelta and doc for id: " << i << " chunk:\n" + << chunk + << "\nexpectedDelta:\n" + << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") + << "\nGot doc:\n" + << (doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "NO_DOC"); + FAIL() << "Mismatch between expectedDelta and doc for chunk[" << i << "]: " << chunk; + } + } +} + +// ============================================================================= +// Proves the "unary is an edge case of streaming" property holds structurally, not +// just by coincidence: OnyxToolParser::parse() literally drives the same +// OnyxToolParserImpl used by parseChunk() (see onyx_tool_parser.cpp), so this is +// really just re-checking that the unary entry point wires into the same state +// machine already covered above. +// ============================================================================= +TEST_F(OnyxOutputParserTest, UnaryToolCallMatchesStreamingReuse) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); +} + +TEST_F(OnyxOutputParserTest, UnaryTwoSequentialToolCalls) { + ParsedOutput parsedOutput = generateParsedOutput( + " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "get_time"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"city\":\"SF\"}"); +} + +// ============================================================================= +// Direct OnyxToolParserImpl unit tests -- mirrors Qwen3CoderOutputParserTest's +// TestJustParserImplUnary*/TestJustParserImplStreamStep* layer (which exercises the +// state machine directly, below OutputParser/OnyxToolParser), previously untested here. +// ============================================================================= +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"location\":\"Paris\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { + const std::string input = "Unexpected void found. Philosophical crisis imminent."; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_FALSE(callsOpt.has_value()); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(content, input); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithTwoToolCalls) { + const std::string input = " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 2) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(calls[1].name, "get_time"); + EXPECT_EQ(calls[1].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithNoStateChange) { + const std::string input = "Some content without tool calls"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithPartialToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); + ASSERT_TRUE(parser.getCurrentFunctionName().has_value()); + EXPECT_EQ(parser.getCurrentFunctionName().value(), "get_weather"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { + const std::string input = " to=functions.get_current_location<|message|>{}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_TRUE(stepResult.has_value()); + auto& calls = stepResult.value(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_current_location"); + EXPECT_EQ(calls[0].arguments, "{}"); +} + +// ============================================================================= +// Qwen3CoderOutputParserTest test cases, for reference/parity comparison (this file +// intentionally does not have a 1:1 test for every one of these -- see inline notes +// on why some don't apply to Onyx's simpler, non-schema-driven, single-JSON-blob +// argument format): +// Parse1ToolCall1Function1ArgumentTagsNewline +// Parse1ToolCall1Function1ArgumentNoProperBeginTag +// Parse1ToolCallNestedXmlNotFromSchema +// ParseTwoToolCalls1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentMultilineValue +// TestJustParserImplUnaryToolCall -- covered above +// TestJustParserImplUnaryWithNoToolCall -- covered above +// TestJustParserImplUnaryWithContent -- N/A: Onyx's grammar never +// has plain content before/after a tool-call tag within the same generated turn +// TestJustParserImplUnaryWithThreeParameters -- N/A: no per-parameter +// schema-driven typing; arguments are always a single opaque JSON blob +// TestJustParserImplUnaryWithEnforcementOfStringParameter -- N/A, same reason +// TestJustParserImplUnaryWithNotPresentToolSchema -- N/A, same reason (Onyx +// never even looks at tool schemas -- see ToolCallWithRawJsonArguments above) +// TestJustParserImplUnaryWithJsonObjectArgument -- covered by nested-object +// case in StreamingSimpleToolCall above +// TestJustParserImplUnaryWithTwoToolCalls -- covered above +// TestJustParserImplUnaryToolCallNoMatchingToolParameterTypeMapEntry -- N/A, same reason +// TestJustParserImplUnaryToolCallWithRepeatedArgument -- N/A, same reason (no +// per-parameter parsing to have a "repeated argument" concept at all) +// TestJustParserImplStreamStepWithMoreThan1StateChange -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above (both calls resolve in one parseChunk) +// TestJustParserImplStreamStepWithNoStateChange -- covered above +// TestJustParserImplStreamStepWithPartialToolCall -- covered above +// TestJustParserImplStreamStepWithTwoToolCalls -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above +// TestJustParserImplStreamStepWithToolCallNoArgs -- covered above +// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes -- N/A: +// parametrized over per-parameter type coercion (string/int/float/bool/object/list), +// which does not exist for Onyx (raw JSON passthrough only) +// StreamingSimpleToolCall -- covered above (adapted; +// see comment on that test for what was intentionally omitted/adjusted) +// ============================================================================= + From a5587707bdde13493f888a57aed9891ca21a94a4 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Tue, 28 Jul 2026 11:01:31 +0200 Subject: [PATCH 02/38] Build files --- Dockerfile.ubuntu | 8 ++++++-- Makefile | 9 ++++++++- versions.mk | 17 +++++++++++++++-- windows_install_build_dependencies.bat | 6 ++++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index ed38a3cb02..2efdcd42bc 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -156,10 +156,14 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" +# onyx-support patches (temporary, one-off - see patches/*/readme.md for the +# commits they apply to) +COPY patches /patches/ + ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ ARG SDL_OPS="-Wl,-z,relro,-z,now,-z,noexecstack -fPIE -pie -fstack-protector-strong -fexceptions -fasynchronous-unwind-tables -fcf-protection -fpic -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fno-strict-overflow -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wuninitialized" # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git apply /patches/openvino/0001-openvino.patch && git apply /patches/openvino/0002-openvino.patch && git apply /patches/openvino/0003-openvino.patch && git apply /patches/openvino/0004-openvino.patch && git apply /patches/openvino/0005-openvino.patch && git submodule update --init --recursive WORKDIR /openvino/build RUN if [ "$ov_use_binary" == "0" ]; then \ if [[ $debug_bazel_flags == *"py_off"* ]]; then \ @@ -228,7 +232,7 @@ ARG ov_genai_org=openvinotoolkit WORKDIR /openvino_genai/ # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then \ - git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive && \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git apply /patches/openvino.genai/0001-openvino.genai.patch && git submodule update --init --recursive && \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} " -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ && \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ && \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ && \ diff --git a/Makefile b/Makefile index 654875576c..fdab8ee227 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,14 @@ endif ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" - OV_USE_BINARY ?= 1 + # ubuntu24 defaults to building OpenVINO/GenAI from source (see versions.mk) + # so that the onyx-support patches in ./patches can be applied. Other ubuntu + # flavors keep using the prebuilt binary package by default. + ifeq ($(BASE_OS),ubuntu24) + OV_USE_BINARY ?= 0 + else + OV_USE_BINARY ?= 1 + endif ifeq ($(findstring ubuntu22,$(BASE_OS)),ubuntu22) ifeq ($(OV_USE_BINARY),0) $(error OV_USE_BINARY = 0 not supported on Ubuntu22 OS) diff --git a/versions.mk b/versions.mk index eaf471b495..1eb3cd062b 100644 --- a/versions.mk +++ b/versions.mk @@ -19,9 +19,22 @@ # Any variable can be overridden by the environment or command-line. # Source repository git commits / branches (used for source builds) -OV_SOURCE_BRANCH ?= d08e55c64c37fde1f4f6157cc5f5e07dd36ce5e8 +# NOTE: pinned to the commits required by the onyx-support patches in +# ./patches (see patches/openvino/readme.md and patches/openvino.genai/readme.md). +# This is a temporary, one-off pin - restore the previous commits below once +# the patches are no longer needed: +# OV_SOURCE_BRANCH ?= d08e55c64c37fde1f4f6157cc5f5e07dd36ce5e8 (pre-patch branch tip) +# OV_GENAI_BRANCH ?= 8981d6f848f17985979be0a9224251d181f68c56 (pre-patch branch tip) +# NOTE: OV_TOKENIZERS_BRANCH is intentionally left at its original commit - +# the tokenizers commit referenced by the genai patch's submodule bump +# (935443f5275ce93f362f9eb4fa2d9fa762dd3f22) does not exist in the +# openvinotoolkit/openvino_tokenizers repo (only reachable from a fork used +# during genai development), and OVMS builds tokenizers as a separate +# component (BUILD_TOKENIZERS=OFF in the genai cmake invocation) so this pin +# does not affect the OVMS build. +OV_SOURCE_BRANCH ?= 5b6997da03a7a0713fb4376f9109b4832383cc24 OV_TOKENIZERS_BRANCH ?= a8d763dee39cb18e33edd01eca1995a07c8b247d -OV_GENAI_BRANCH ?= 8981d6f848f17985979be0a9224251d181f68c56 +OV_GENAI_BRANCH ?= c637ed85efebf1a44d5f0433845849a2d80b353c # Source repository organizations OV_SOURCE_ORG ?= openvinotoolkit diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 3ac79c2920..4ab3f5962c 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -256,6 +256,10 @@ cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin git checkout %OV_SOURCE_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +for %%P in (0001 0002 0003 0004 0005) do ( + git apply "%BACK_CWD%\patches\openvino\%%P-openvino.patch" + if !errorlevel! neq 0 exit /b !errorlevel! +) git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules @@ -304,6 +308,8 @@ cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin git checkout %OV_GENAI_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +git apply "%BACK_CWD%\patches\openvino.genai\0001-openvino.genai.patch" +if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( mkdir build From 1cedf231eea6d86bbbda112f05f5981afe3d4a8c Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 30 Jul 2026 09:13:24 +0200 Subject: [PATCH 03/38] Experimenting with decode special tokens --- src/llm/io_processing/output_parser.cpp | 3 ++- src/llm/io_processing/output_parser.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index a51da1d108..e787b09a64 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -213,6 +213,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "onyx") { reasoningParser = std::make_unique(tokenizer); + decodeWithSpecialTokens = true; } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); @@ -277,7 +278,7 @@ ParsedOutput OutputParser::parse(const std::vector& generatedTokens, co SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); } ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); + parsedOutput.content = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(!decodeWithSpecialTokens)); if (reasoningParser) { reasoningParser->parse(parsedOutput, generatedTokens); } diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 4e7c467f81..a6d0f7bbc5 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -58,6 +58,7 @@ class OutputParser { ov::genai::Tokenizer tokenizer; std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content + bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) // Streaming related members ProcessingPhase processingPhase = UNKNOWN; From 69aa215984b5ed0809fec200636a13de6e81473c Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 31 Jul 2026 10:51:24 +0200 Subject: [PATCH 04/38] WIP rewritten tool_parser --- src/llm/BUILD | 2 + .../onyx/onyx_reasoning_parser.cpp | 58 +- .../onyx/onyx_reasoning_parser.hpp | 2 - .../io_processing/onyx/onyx_tool_parser.cpp | 330 ++++++++-- .../io_processing/onyx/onyx_tool_parser.hpp | 146 +++-- src/llm/io_processing/output_parser.cpp | 2 +- ...emplate_and_parser_onyx_roundtrip_test.cpp | 63 +- .../onyx_output_parser_test.cpp | 572 +++++++++++------- 8 files changed, 803 insertions(+), 372 deletions(-) diff --git a/src/llm/BUILD b/src/llm/BUILD index 0f83815b70..79c719a10e 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -360,10 +360,12 @@ ovms_cc_library( deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src/port:rapidjson_document", + "//src/utils:rapidjson_utils", "//src:libovmslogging", "//src:libovmsstatus", ":io_processing_utils", ":io_processing_base_output_parser", + ":apis_tool_schema_wrapper", "//third_party:genai", ], visibility = ["//visibility:public"], diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp index 43b45a1e8f..19542db01f 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -33,41 +33,47 @@ void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector leave untouched, OnyxToolParser - // (which runs after this parser, see OutputParser::parse()) is responsible for it. - if (parsedOutput.content.find(functionsRecipientTag) != std::string::npos) { - return; + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); + parsedOutput.reasoning = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + // Erase ONLY the reasoning segment (through its "<|eom|>" terminator), including the + // leading " " the template renders before "to=". With eos suppressed the model may + // continue into a tool-call / final-answer turn after reasoning; that turn must survive + // for the envelope strip below (and OnyxToolParser) to process. + size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; + size_t eraseEnd = (endPos != std::string::npos) ? endPos + continuationEndTag.length() : parsedOutput.content.length(); + parsedOutput.content.erase(segmentStart, eraseEnd - segmentStart); + // fall through to strip the envelope of any following (tool-call / final-answer) turn } - // Case 3: plain final answer (recipient="user" or absent) -> strip the generic - // " to=user"? + "<|message|>" + "<|eot|>" envelope, leaving just the clean text. + // Case 2: any other turn -- a tool-call turn (recipient="", e.g. "get_weather", now + // a BARE name after the new drop's chat template, not "functions.") or a plain final + // answer (recipient="user" or absent). Strip the generic harmony routing prefix + // ("[ to=]" + "<|message|>") and a single trailing turn terminator + // ("<|eom|>" or "<|eot|>"), leaving just the body. For a tool-call turn the body is the ATEM + // block, which OnyxToolParser (running next, see OutputParser::parse()) then extracts, + // leaving content empty; for a final answer the body is the clean text. size_t messagePos = parsedOutput.content.find(messageTag); if (messagePos == std::string::npos) { // No framing found at all -- unexpected/malformed output, leave content as-is. return; } - size_t bodyStart = messagePos + messageTag.length(); - size_t endPos = parsedOutput.content.find(turnFinalEndTag, bodyStart); - std::string body = (endPos != std::string::npos) - ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) - : parsedOutput.content.substr(bodyStart); - parsedOutput.content = body; + // Drop everything up to and including the first "<|message|>" (the routing prefix). + parsedOutput.content.erase(0, messagePos + messageTag.length()); + // Drop a single trailing terminator if the turn ends with one. + for (const auto& term : {continuationEndTag, turnFinalEndTag}) { + if (parsedOutput.content.size() >= term.size() && + parsedOutput.content.compare(parsedOutput.content.size() - term.size(), term.size(), term) == 0) { + parsedOutput.content.erase(parsedOutput.content.size() - term.size()); + break; + } + } } std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp index 27218d8414..4dc54046d9 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -47,8 +47,6 @@ class OnyxReasoningParser : public BaseOutputParser { protected: // Marks a private chain-of-thought turn (recipient="self"). const std::string selfRecipientTag = "to=self"; - // Marks a tool-call turn (recipient="functions.") -- left untouched here. - const std::string functionsRecipientTag = "to=functions."; // Separates the routing prefix from the turn's body. const std::string messageTag = "<|message|>"; // Terminator for continuation turns (reasoning and tool calls). diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 76b6a65e49..a43f5b0bbe 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -15,55 +15,262 @@ //***************************************************************************** #include +#include +#include #include #include +#include "rapidjson/error/en.h" + #include "src/port/rapidjson_document.hpp" -#include "src/logging.hpp" #include "src/llm/io_processing/utils.hpp" +#include "src/logging.hpp" +#include "src/utils/rapidjson_utils.hpp" #include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" namespace ovms { -const std::string OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG = "to=functions."; -const std::string OnyxToolParserImpl::MESSAGE_TAG = "<|message|>"; -const std::string OnyxToolParserImpl::END_TAG = "<|eom|>"; +const std::string OnyxToolParser::TOOL_START_TAG = ""; +const std::string OnyxToolParser::TOOL_END_TAG = ""; +const std::string OnyxToolParser::FUNCTION_NAME_TAG = ""; + +// Static empty map the default-constructed impl binds its const-ref member to (used by the +// direct-impl unit tests, which pass plain-string values that need no schema typing). +static const ToolsParameterTypeMap_t EMPTY_TOOLS_PARAMETER_TYPE_MAP{}; + +OnyxToolParserImpl::OnyxToolParserImpl() : + toolsParametersTypeMap(EMPTY_TOOLS_PARAMETER_TYPE_MAP) {} + +OnyxToolParserImpl::OnyxToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap) : + toolsParametersTypeMap(toolsParametersTypeMap) {} + +static void trimNewline(std::string& str) { + if (str.empty()) { + return; + } + if (str.back() == '\n') { + str.pop_back(); + } + if (str.empty()) { + return; + } + if (str.front() == '\n') { + str.erase(str.begin()); + } +} + +// Build parameterName -> ParameterType from a tool JSON schema (same shape as +// Qwen3CoderToolParser's parseToolSchema). +static const ParametersTypeMap_t parseToolSchema(const std::string& functionName, const rapidjson::Value& schema) { + ParametersTypeMap_t result; + if (!schema.IsObject()) { + SPDLOG_DEBUG("Tool schema is not a JSON object for tool: {}", functionName); + return result; + } + if (!schema.HasMember("properties") || !schema["properties"].IsObject()) { + SPDLOG_DEBUG("Tool schema does not have properties object for tool: {}", functionName); + return result; + } + const rapidjson::Value& properties = schema["properties"]; + for (auto it = properties.MemberBegin(); it != properties.MemberEnd(); ++it) { + if (!it->value.IsObject()) { + continue; + } + if (!it->value.HasMember("type") || !it->value["type"].IsString()) { + continue; + } + std::string paramName = it->name.GetString(); + std::string typeStr = it->value["type"].GetString(); + ParameterType type = ParameterType::UNKNOWN; + if (typeStr == "string") { + type = ParameterType::STRING; + } else if (typeStr == "number" || typeStr == "integer") { + type = ParameterType::NUMBER; + } else if (typeStr == "boolean") { + type = ParameterType::BOOLEAN; + } else if (typeStr == "array") { + type = ParameterType::ARRAY; + } else if (typeStr == "object") { + type = ParameterType::OBJECT; + } else { + SPDLOG_DEBUG("Tool schema property: {} has unknown type: {} for tool: {}", paramName, typeStr, functionName); + } + result.emplace(paramName, type); + } + return result; +} + +static ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& toolsSchemas) { + ToolsParameterTypeMap_t toolsParametersTypes; + for (const auto& [toolName, toolSchemaWrapper] : toolsSchemas) { + toolsParametersTypes.emplace(toolName, parseToolSchema(toolName, *toolSchemaWrapper.rapidjsonRepr)); + } + return toolsParametersTypes; +} + +static const char* jsonTypeOf(const rapidjson::Value& val) { + if (val.IsObject()) + return "object"; + if (val.IsArray()) + return "array"; + if (val.IsString()) + return "string"; + if (val.IsBool()) + return "bool"; + if (val.IsNumber()) + return "number"; + if (val.IsNull()) + return "null"; + return "unknown"; +} + +static void enforceStringValue(rapidjson::Value& v, rapidjson::Document::AllocatorType& alloc) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + v.Accept(writer); + v.SetString(buffer.GetString(), buffer.GetLength(), alloc); +} + +void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameterValueAsString) { + if (this->removeNewlineAroundParameters) + trimNewline(parameterValueAsString); + // Serialize the untyped ATEM value into JSON using the tool schema to decide the type. + auto paramIt = this->toolsParametersTypeMap.find(this->currentFunction.name); + auto& currentFunctionArgsDoc = this->currentFunction.argumentsAsDocument; + auto& allocator = currentFunctionArgsDoc.GetAllocator(); + auto& key = this->currentParameterName; + rapidjson::Value keyVal(key.c_str(), allocator); + rapidjson::Document temp; + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::BOOLEAN)) { + if (parameterValueAsString == "True" || parameterValueAsString == "TRUE") { + parameterValueAsString = "true"; + } else if (parameterValueAsString == "False" || parameterValueAsString == "FALSE") { + parameterValueAsString = "false"; + } + } + } + temp.Parse(parameterValueAsString.c_str()); + if (temp.HasParseError()) { + // Not valid JSON -> insert as a string value. + rapidjson::Value v; + v.SetString(parameterValueAsString.c_str(), static_cast(parameterValueAsString.size()), allocator); + if (!currentFunctionArgsDoc.HasMember(keyVal)) { + currentFunctionArgsDoc.AddMember(keyVal, v, allocator); + } else { + SPDLOG_DEBUG("Parameter: {} already exists in document", key); + } + } else { + rapidjson::Value valueCopy; + valueCopy.CopyFrom(temp, allocator); + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::STRING)) { + enforceStringValue(valueCopy, allocator); + } + } + if (!currentFunctionArgsDoc.HasMember(keyVal)) { + SPDLOG_TRACE("Will add key:{} val:{} type:{}", key, parameterValueAsString, jsonTypeOf(valueCopy)); + currentFunctionArgsDoc.AddMember(keyVal, valueCopy, allocator); + } else { + SPDLOG_DEBUG("Parameter: {} already exists in document.", key); + } + } +} -#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ - auto pos = this->streamContent.find(TAG, this->lastProcessedPosition); \ - if (pos == std::string::npos) { \ - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Did not find: {}", TAG); \ - break; \ +#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ + auto pos = this->streamContent.find(TAG, this->getLastProcessedPosition()); \ + if (pos == std::string::npos) { \ + SPDLOG_TRACE("Did not find: {}", TAG); \ + break; \ } bool OnyxToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + SPDLOG_TRACE("State: {}", this->currentState); auto previousState = this->currentState; switch (this->currentState) { case State::Content: { - DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(FUNCTIONS_RECIPIENT_TAG); - this->toolCallPositions.begin.push(pos); - this->lastProcessedPosition = pos + FUNCTIONS_RECIPIENT_TAG.length(); - this->currentState = State::InsideName; + // Normally "" precedes "lastProcessedPosition = posTool + OnyxToolParser::TOOL_START_TAG.length(); + this->currentState = State::InsideToolCall; + this->toolCallPositions.begin.push(posTool); + } else { + SPDLOG_DEBUG("Did not find: {}, assuming it should exist", OnyxToolParser::TOOL_START_TAG); + this->lastProcessedPosition = posFunc + OnyxToolParser::FUNCTION_NAME_TAG.length(); + this->currentState = State::InsideFunctionName; + this->toolCallPositions.begin.push(posFunc); + } + break; + } + case State::InsideToolCall: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::FUNCTION_NAME_TAG); + this->lastProcessedPosition = pos + OnyxToolParser::FUNCTION_NAME_TAG.length(); + this->currentState = State::InsideFunctionName; + break; + } + case State::InsideFunctionName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::NAME_ATTR_END_TAG); + this->currentFunction.name = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + OnyxToolParser::NAME_ATTR_END_TAG.length(); + this->currentState = State::InsideFunction; + break; + } + case State::InsideFunction: { + auto funcEnd = streamContent.find(OnyxToolParser::FUNCTION_END_TAG, this->lastProcessedPosition); + auto paramStart = streamContent.find(OnyxToolParser::PARAMETER_NAME_TAG, this->lastProcessedPosition); + if (funcEnd == std::string::npos && paramStart == std::string::npos) { + } else if (paramStart < funcEnd) { // next parameter + this->lastProcessedPosition = paramStart + OnyxToolParser::PARAMETER_NAME_TAG.length(); + this->currentState = State::InsideParameterName; + } else { // end of function + this->lastProcessedPosition = funcEnd + OnyxToolParser::FUNCTION_END_TAG.length(); + this->currentState = State::AfterFunction; + } + break; + } + case State::InsideParameterName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::NAME_ATTR_END_TAG); + this->currentParameterName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + OnyxToolParser::NAME_ATTR_END_TAG.length(); + this->currentState = State::InsideParameter; break; } - case State::InsideName: { - DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(MESSAGE_TAG); - this->currentFunctionName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); - this->lastProcessedPosition = pos + MESSAGE_TAG.length(); - this->currentState = State::InsideArguments; + case State::InsideParameter: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::PARAMETER_END_TAG); + std::string parameterValueAsString(streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition)); + addParameterToCurrentFunctionDoc(parameterValueAsString); + this->lastProcessedPosition = pos + OnyxToolParser::PARAMETER_END_TAG.length(); + this->currentState = State::InsideFunction; break; } - case State::InsideArguments: { - DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(END_TAG); - std::string argumentsPart = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); - this->lastProcessedPosition = pos + END_TAG.length(); + case State::AfterFunction: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(OnyxToolParser::TOOL_END_TAG); + this->lastProcessedPosition = pos + OnyxToolParser::TOOL_END_TAG.length(); this->currentState = State::Content; - this->toolCallPositions.end.push(this->lastProcessedPosition); - ToolCall toolCall{generateRandomId(), this->currentFunctionName, argumentsPart}; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Adding tool call: id={}, name={}, arguments={}", toolCall.id, toolCall.name, toolCall.arguments); + std::string argumentsAsString; + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + this->currentFunction.argumentsAsDocument.Accept(writer); + argumentsAsString = buffer.GetString(); + } + ToolCall toolCall{generateRandomId(), this->currentFunction.name, argumentsAsString}; + SPDLOG_TRACE("Adding tool call: id={}, name={}, params={}", toolCall.id, toolCall.name, toolCall.arguments); toolCalls.emplace_back(std::move(toolCall)); - this->currentFunctionName.clear(); + this->currentFunction.clear(); + this->toolCallPositions.end.push(this->lastProcessedPosition); break; } } @@ -85,27 +292,35 @@ std::optional OnyxToolParserImpl::parseChunk(const std::string& chu } std::optional OnyxToolParserImpl::getCurrentFunctionName() const { - if (this->currentFunctionName.empty()) { + if (this->currentFunction.name.empty()) { return std::nullopt; } - return this->currentFunctionName; + return this->currentFunction.name; } Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + // Generation can be truncated mid-tool-call (max_tokens hit, or eos suppressed) so an opening + // "" is recorded with no matching "" close. That + // leaves begin with more entries than end. The unterminated call is always the most recent one + // (top of the begin stack), so drop it -- erasing from its start to end-of-content -- rather + // than bailing and leaving every (including completed) block in the content returned to the user. + while (toolCallPositions.begin.size() > toolCallPositions.end.size()) { + auto posBegin = toolCallPositions.begin.top(); + toolCallPositions.begin.pop(); + if (posBegin <= outContent.size()) { + SPDLOG_TRACE("Removing unterminated tool call from outContent begin:{} to end, removing:{}", posBegin, outContent.substr(posBegin)); + outContent.erase(posBegin); + } + } if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); + // Unexpected shape (more closes than opens) -- leave content untouched to avoid corrupting it. + SPDLOG_DEBUG("Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); } while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { auto posBegin = toolCallPositions.begin.top(); auto posEnd = toolCallPositions.end.top(); - // Also consume the leading " " the chat template renders before "to=" (a single - // generate() call only ever produces one such segment, see OnyxReasoningParser's - // class comment for why this can't collide with anything preceding it). - if (posBegin > 0 && outContent[posBegin - 1] == ' ') { - posBegin -= 1; - } - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); + SPDLOG_TRACE("Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); outContent.erase(posBegin, posEnd - posBegin); toolCallPositions.begin.pop(); toolCallPositions.end.pop(); @@ -113,25 +328,37 @@ Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outCo return StatusCode::OK; } +OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : + BaseOutputParser(tokenizer), + toolSchemas(toolSchemas), + streamParser(this->toolsParametersTypes) { +} + +void OnyxToolParser::lazyFillInitToolParametersTypesMap() { + if (this->filledParametersTypesMap) { + return; + } + this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); + this->filledParametersTypesMap = true; + SPDLOG_DEBUG("OnyxToolParser created with {} tools", this->toolsParametersTypes.size()); +} + void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> - // - // Mirrors Qwen3CoderToolParser::parse(): drive the same streamParser used for - // streaming with the whole content as a single chunk, and reuse whatever it - // assembled -- unary is a single-shot edge case of streaming, not a parallel - // reimplementation of the tag walk. + // Unary is the single-shot edge case of streaming: drive the same streamParser with the + // whole content as one chunk (mirrors Qwen3CoderToolParser::parse()). + this->lazyFillInitToolParametersTypesMap(); auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); if (!toolCallsOpt.has_value()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Parsing ended, no tool calls found"); + SPDLOG_DEBUG("Parsing ended, no tool calls found"); return; } parsedOutput.toolCalls = std::move(toolCallsOpt.value()); for (const auto& toolCall : parsedOutput.toolCalls) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); + SPDLOG_DEBUG("Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); } auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to remove tool calls from content: {}", status.string()); + SPDLOG_DEBUG("Failed to remove tool calls from content: {}", status.string()); } } @@ -147,11 +374,10 @@ std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const } std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { - // ASSUMPTION: mirroring Qwen3CoderToolParser, in streaming we only ever complete one - // tool call per parseChunk() call -- there is no way to send multiple tool calls to - // the client in a single streaming delta. + // ASSUMPTION (mirrors Qwen3CoderToolParser): in streaming we only ever complete one tool + // call per parseChunk() -- there is no way to send multiple tool calls in one delta. if (toolCalls.size() != 1) { - SPDLOG_LOGGER_ERROR(llm_calculator_logger, "For streaming we expected one tool call, got: {}", toolCalls.size()); + SPDLOG_ERROR("For streaming we expected one tool call, got: {}", toolCalls.size()); throw std::runtime_error("For streaming we expected one tool call"); } const auto& toolCall = toolCalls[0]; @@ -164,8 +390,9 @@ std::optional OnyxToolParser::sendFullDelta(const ToolCalls } std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { - // streamParser returns assembled toolCalls once a call closes ("<|eom|>" seen); until - // then, if the function name is already known, send the first delta for it once. + // streamParser returns assembled toolCalls once a call closes (""); + // until then, if the function name is already known, send its first delta once. + this->lazyFillInitToolParametersTypesMap(); if (newChunk.empty()) { return std::nullopt; } @@ -180,4 +407,3 @@ std::optional OnyxToolParser::parseChunk(const std::string& return std::nullopt; } } // namespace ovms - diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index d8d48b6bc1..ccf7270240 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -15,10 +15,12 @@ //***************************************************************************** #pragma once +#include #include #include #include #include +#include #include #include @@ -26,66 +28,115 @@ #include "src/port/rapidjson_document.hpp" #include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/apis/tool_schema_wrapper.hpp" +#include "src/logging.hpp" #include "src/status.hpp" namespace ovms { -// Onyx (early preview model) tool-call framing: -// TODO @atobiszei is functions namespace always "functions"? -// <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> -// Unlike qwen3coder/hermes3, Onyx never wraps arguments in a schema-validated, -// per-parameter structure -- the segment between "<|message|>" and "<|eom|>" is -// already the complete, raw JSON arguments blob the caller is expected to forward -// as-is (per the model card: "the SFT tokenizer tokenizes message content ... -// (raw body)"). So no tool-schema-driven type coercion is needed here, unlike -// Qwen3CoderToolParser. - -// Pure state machine that accumulates raw generated text and hands back fully -// assembled tool calls -- mirrors Qwen3CoderToolParserImpl's split between "parse the -// framing" and "turn it into OpenAI delta JSON" (done by the owning OnyxToolParser). -// Because Onyx's arguments are already a complete raw JSON blob (no per-parameter -// schema coercion needed), a tool call is fully known as soon as its end tag is seen -- -// unlike Qwen3Coder there is no incremental per-parameter streaming to do. +// Onyx (new drop) tool-call framing. The assistant turn is routed with the harmony +// envelope " to=<|message|>...{<|eom|>|<|eot|>}", and when the recipient is a +// function the body is an ATEM XML block -- Anthropic-style, structurally identical to +// qwen3coder's // walk, just with "atem:" tags: +// +// to=get_weather<|message|> +// +// 37.7749,-122.4194 +// 2026-07-30T18:00:00Z +// +// <|eom|> +// +// The parser triggers on the fixed "" marker (the "to=" +// recipient is variable and must NOT be used as a trigger -- the model emits the BARE tool +// name, e.g. "to=get_weather", not "to=functions.get_weather"), and reads the authoritative +// name from . The " to=<|message|>" prefix and the trailing +// terminator are stripped by OnyxReasoningParser (which runs first, see OutputParser::parse()). +// +// Parameter VALUES are rendered untyped/unquoted, so -- exactly like Qwen3CoderToolParser -- +// each value is serialized into the JSON arguments blob according to the tool JSON schema +// (string->quoted, integer/number->numeric, bool/array/object->parsed, otherwise a +// best-effort JSON parse falling back to string). +using ParametersValues_t = std::map; + +struct OnyxFunctool { + std::string name; + rapidjson::Document argumentsAsDocument; + OnyxFunctool() { + argumentsAsDocument.SetObject(); + } + void clear() { + name.clear(); + argumentsAsDocument.SetObject(); + } +}; + +// Pure state machine that accumulates raw generated text and hands back fully assembled tool +// calls -- mirrors Qwen3CoderToolParserImpl. Holds the parameter-type map BY REFERENCE (bound +// either to the owning OnyxToolParser's lazily-filled map, or to a static empty map for the +// default constructor used by the direct-impl unit tests, where plain-string values need no +// schema to serialize correctly). struct OnyxToolParserImpl { enum class State { - Content, // looking for the next "to=functions." recipient tag - InsideName, // accumulating the function name, looking for messageTag - InsideArguments // accumulating the raw JSON arguments blob, looking for endTag + Content, // expect tool start tag or end of content + InsideToolCall, // after "", expect "" + InsideFunction, // expect a "" end + InsideParameterName, // reading a parameter name, expect the closing "\">" + InsideParameter, // reading a parameter value, expect "" + AfterFunction // after "", expect "" }; - // Marks the start of a tool-call turn; the function name follows immediately. - static const std::string FUNCTIONS_RECIPIENT_TAG; - // Separates the function name from the raw JSON arguments blob. - static const std::string MESSAGE_TAG; - // Tool calls always end the turn as a continuation (never a full turn end). - static const std::string END_TAG; + OnyxToolParserImpl(); + explicit OnyxToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap); - // Return all tool calls fully closed (end tag seen) in the aggregated content so far - // that were not returned before -- nullopt if none completed yet. + // Return all tool calls fully closed ("" seen) in the aggregated + // content so far that were not returned before -- nullopt if none completed yet. std::optional parseChunk(const std::string& chunk); std::optional getCurrentFunctionName() const; Status removeToolCallsFromContentIfNeeded(std::string& outContent); + State getCurrentState() const { + return this->currentState; + } + size_t getLastProcessedPosition() const { + return this->lastProcessedPosition; + } private: + const ToolsParameterTypeMap_t& toolsParametersTypeMap; + // Onyx renders parameter values tight ("...\">VALUE"), so unlike + // qwen3coder there is no surrounding-newline convention to trim. + const bool removeNewlineAroundParameters = false; State currentState = State::Content; + OnyxFunctool currentFunction; + std::string currentParameterName; std::string streamContent; // content accumulated from stream chunks size_t lastProcessedPosition{0}; - std::string currentFunctionName; struct ToolCallPositions { std::stack begin; std::stack end; }; ToolCallPositions toolCallPositions; - // Process streamContent from lastProcessedPosition until a state change happens; - // return true if the state changed (caller should keep looping), false once no more - // progress is possible with the currently available content. + void addParameterToCurrentFunctionDoc(std::string& parameterValueAsString); + // Process streamContent from lastProcessedPosition until a state change happens; return + // true if the state changed (keep looping), false once no more progress is possible. bool parseUntilStateChange(ToolCalls_t& toolCalls); }; class OnyxToolParser : public BaseOutputParser { +public: + static const std::string TOOL_START_TAG; // "" + static const std::string TOOL_END_TAG; // "" + static const std::string FUNCTION_NAME_TAG; // "" + static const std::string PARAMETER_NAME_TAG; // "" + static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name + private: - // for streaming parsing we need to keep the parser as a member + const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) + ToolsParameterTypeMap_t toolsParametersTypes; + bool filledParametersTypesMap{false}; OnyxToolParserImpl streamParser; int toolCallIndex{-1}; std::set returnedFirstDeltas; @@ -93,28 +144,47 @@ class OnyxToolParser : public BaseOutputParser { std::optional sendFirstDeltaIfNeeded(const std::string& functionName); std::optional sendFullDelta(const ToolCalls_t& toolCalls); + void lazyFillInitToolParametersTypesMap(); public: OnyxToolParser() = delete; - explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer) : - BaseOutputParser(tokenizer) {} + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas); void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; const std::vector& getParsingStartTags() const override { - static const std::vector parsingStartTags{OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG}; - return parsingStartTags; + static const std::vector startTags{TOOL_START_TAG}; + return startTags; } const std::vector& getSpecialParsingStartTags() const override { static const std::vector specialParsingStartTags{}; return specialParsingStartTags; } const std::string& getParsingEndTag() const override { - return OnyxToolParserImpl::END_TAG; + return TOOL_END_TAG; } bool requiresStreamingWithSpecialTokens() const override { return true; } - }; } // namespace ovms + +template <> +struct fmt::formatter : fmt::formatter { + auto format(const ovms::OnyxToolParserImpl::State& state, fmt::format_context& ctx) const { + std::unordered_map stateMap = { + {ovms::OnyxToolParserImpl::State::Content, "Content"}, + {ovms::OnyxToolParserImpl::State::InsideToolCall, "InsideToolCall"}, + {ovms::OnyxToolParserImpl::State::InsideFunctionName, "InsideFunctionName"}, + {ovms::OnyxToolParserImpl::State::InsideFunction, "InsideFunction"}, + {ovms::OnyxToolParserImpl::State::InsideParameterName, "InsideParameterName"}, + {ovms::OnyxToolParserImpl::State::InsideParameter, "InsideParameter"}, + {ovms::OnyxToolParserImpl::State::AfterFunction, "AfterFunction"}}; + auto it = stateMap.find(state); + if (it != stateMap.end()) { + return fmt::formatter::format(it->second, ctx); + } else { + return fmt::formatter::format("Unknown", ctx); + } + } +}; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index e787b09a64..95f9bcdf9f 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -199,7 +199,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); } else if (toolParserName == "onyx") { - toolParser = std::make_unique(tokenizer); + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp index b253081edf..1f1f92c702 100644 --- a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -75,6 +75,21 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { return rendered; } + // Onyx's ATEM tool calls carry untyped parameter values, so (like qwen3coder) the parser + // needs the tool JSON schema to serialize each value with the right JSON type. get_weather + // takes a single string param "city" here. + static ToolsSchemas_t makeToolsSchemas() { + static std::unique_ptr getWeatherSchema = [] { + auto doc = std::make_unique(); + doc->Parse(R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"); + return doc; + }(); + static const std::string getWeatherSchemaStr = R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"; + ToolsSchemas_t schemas; + schemas["get_weather"] = {getWeatherSchema.get(), getWeatherSchemaStr}; + return schemas; + } + // Simulates generation: appends modelContinuation to the rendered prompt (for // documentation / sanity only) and runs OutputParser on modelContinuation alone, // matching what OVMS actually hands the parser. @@ -83,8 +98,8 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { auto generatedTensor = tokenizer.encode(modelContinuation, ov::genai::add_special_tokens(false)).input_ids; std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); - static ToolsSchemas_t emptyToolsSchema{}; // Onyx tool parser is not schema-driven, see onyx_tool_parser.hpp - OutputParser outputParser(tokenizer, "onyx", "onyx", emptyToolsSchema); + ToolsSchemas_t toolsSchemas = makeToolsSchemas(); + OutputParser outputParser(tokenizer, "onyx", "onyx", toolsSchemas); return outputParser.parse(generatedTokens, toolsAvailable); } }; @@ -103,15 +118,22 @@ TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) std::string prompt = renderPrompt(chatHistory); EXPECT_NE(prompt.find("You can call get_weather(city)."), std::string::npos) << prompt; - // Model continuation for a tool call, exactly as documented in muse/README.md. - std::string modelContinuation = R"( to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"; + // Model continuation for a tool call in the NEW ATEM format (bare recipient "to=get_weather", + // ATEM XML body), as captured live (muse/onyx_live_withargs_1000_raw.txt). "city" is a string + // per the schema, so it is serialized as a quoted JSON string. + std::string modelContinuation = + " to=get_weather<|message|>\n" + "\n" + "SF\n" + "\n" + "<|eom|>"; ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city": "SF"})"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"SF"})"); } // ============================================================================= @@ -131,23 +153,22 @@ TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinal R"({"role":"tool","name":"functions.get_weather","content":"{\"temp\": 65}"})")); std::string prompt = renderPrompt(chatHistory); + // Assistant tool-call history uses the recipient/content path (not `tool_calls`), which the + // new template still renders via the plain "to=<|message|>...<|eom|>" envelope. EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; - // NOTE (important, non-obvious): OpenVINO GenAI's own minja-path history - // preprocessing -- independent of the raw Jinja template's `elif role == - // 'tool'` branch -- rewrites role="tool" messages into role="user" with a - // generic wrapped "tool_response" JSON object whenever it determines the - // template lacks native tool-call support (the same probe underlying - // caps.supportsToolCalls == false, see chat_template_end_to_end_minja_test.cpp's - // Onyx tests). So Onyx's own `elif role == 'tool'` template branch is - // effectively DEAD CODE on the minja path today -- it never actually fires. - EXPECT_NE(prompt.find(R"(<|start|>user<|message|>{ - "tool_response": { - "tool": "functions.get_weather", - "content": "{\"temp\": 65}" - } -}<|eot|>)"), - std::string::npos) - << prompt; + // NOTE (verified against the new template's actual minja render): OpenVINO GenAI STILL + // rewrites role="tool" into a role="user" message wrapping a "tool_response" JSON object + // (caps.supportsToolCalls == false), even though the new template DOES read `tools`/render + // tool defs -- so Onyx's own `elif role == 'tool'` template branch remains DEAD CODE on the + // minja path. What changed vs the previous drop: the wrapper no longer carries a "tool" + // field (only "content"). The tool output content appears with backslash-escaped quotes + // inside that JSON string, e.g.: + // <|start|>user<|message|>{ + // "tool_response": { + // "content": "{\"temp\": 65}" + // }<|eot|> + EXPECT_NE(prompt.find(R"("tool_response")"), std::string::npos) << prompt; + EXPECT_NE(prompt.find(R"({\"temp\": 65})"), std::string::npos) << prompt; std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index 4de64ee3f0..e8b853d905 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "src/llm/io_processing/base_output_parser.hpp" @@ -32,13 +33,47 @@ using namespace ovms; +// ============================================================================= +// NEW Onyx tool-call format (ATEM), captured live from the running model +// (muse/onyx_live_withargs_1000_raw.txt, muse/onyx_live_nargs_1500_raw.txt) and +// matching `render_atem` in muse/onyx-ov-int4-v2/chat_template.jinja: +// +// to=<|message|> +// +// +// ... +// +// {<|eom|>|<|eot|>} +// +// Key differences from the previous Onyx drop (which these tests used to cover): +// - The recipient is the BARE tool name `to=get_weather`, NOT `to=functions.get_weather` +// (the "functions." prefix only appears if the tool itself is namespaced). The +// authoritative function name is therefore read from ``, +// not from the `to=` recipient. +// - Arguments are an ATEM XML block (Anthropic-style), essentially qwen3coder with +// `atem:` tags -- NOT a single raw JSON blob. Parameter values are rendered +// UNQUOTED (e.g. 37.7749,-122.4194), +// so arguments must be typed via the tool JSON schema exactly like +// Qwen3CoderToolParser (string->quoted, integer/number->numeric, +// bool/array/object->parsed, fall back to string). The toolsSchemas fixture below +// is therefore load-bearing now. +// +// Reasoning framing is UNCHANGED (" to=self<|message|>...<|eom|>") -- those tests are +// carried over verbatim. +// +// These tests define the TARGET contract; the parser implementation +// (src/llm/io_processing/onyx/*) is rewritten later against them. Until then the +// tool-call tests are expected to FAIL (the current parser still looks for the old +// "to=functions."/raw-JSON framing) while the reasoning tests still pass. +// ============================================================================= + // Onyx does not ship a converted HF tokenizer in this early preview, and none of the -// segments the parser looks for ("to=functions.", "<|message|>", "<|eom|>", "<|eot|>", -// "to=self") are real special tokens of the model this parser is designed for -- they are -// plain text sequences that must round-trip losslessly through encode()+decode() on ANY -// tokenizer. facebook/opt-125m is already used the same way for chat-template testing -// (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid pulling in a new -// model fixture. +// segments the parser looks for ("", "", "<|eom|>", "<|eot|>") are real special tokens of the model this parser is +// designed for -- they are plain text sequences that must round-trip losslessly through +// encode()+decode() on ANY tokenizer. facebook/opt-125m is already used the same way for +// chat-template testing (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid +// pulling in a new model fixture. // TODO @atobiszei replace when tokenizer is available #ifdef _WIN32 const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; @@ -48,10 +83,9 @@ const std::string tokenizerPath = "/ovms/src/test/llm_testing/facebook/opt-125m" static std::unique_ptr opt125mTokenizer; -// Onyx never consults tool schemas (arguments are forwarded as a raw JSON blob verbatim, -// no per-parameter type coercion like Qwen3CoderToolParser) -- these mirror -// Qwen3CoderOutputParserTest's toolSchemasInput/toolsSchemas setup purely so the test -// fixture shape matches, and so a real schema is on hand if that ever changes. +// Tool schemas drive argument typing (string vs integer vs object) exactly like +// Qwen3CoderOutputParserTest -- Onyx's ATEM parameter values are untyped raw text, so the +// parser must consult these to decide how to serialize each value into the JSON args blob. static std::map toolSchemasInput = { {"get_weather", R"({"properties":{"location":{"type":"string","description":"City name."},"unit":{"type":"string","description":"Temperature unit."}},"required":["location"]})"}, {"get_time", R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"}, @@ -79,6 +113,43 @@ static ToolsSchemas_t convertStringToolSchemasStringToToolsSchemas( static ovms::ToolsSchemas_t toolsSchemas = convertStringToolSchemasStringToToolsSchemas(toolSchemasInput); +// ----------------------------------------------------------------------------- +// ATEM block builders -- keep tool-call test inputs readable. A single tool call +// renders (mirroring the served template's render_atem, newlines included) as: +// \n\n +// V\n (repeated) +// \n +// The full generated turn wraps that in the harmony envelope +// " to=NAME<|message|>...{<|eom|>|<|eot|>}". +// ----------------------------------------------------------------------------- +using AtemParams = std::vector>; + +static std::string atemBlock(const std::string& name, const AtemParams& params) { + std::string s = "\n\n"; + for (const auto& [k, v] : params) { + s += "" + v + "\n"; + } + s += "\n"; + return s; +} + +// A full assistant tool-call turn as the model emits it: the leading " to=" recipient, +// "<|message|>", the ATEM block, then the turn terminator. +static std::string onyxToolTurn(const std::string& name, const AtemParams& params, const std::string& terminator = "<|eom|>") { + return " to=" + name + "<|message|>" + atemBlock(name, params) + terminator; +} + +// Pre-parsed parameter-type map for the direct-impl parametrized test below (mirrors +// Qwen3CoderOutputParserTest's toolsParametersTypeMap) -- drives schema-based typing of the +// otherwise-untyped ATEM parameter values. +static ovms::ToolsParameterTypeMap_t onyxToolsParametersTypeMap = { + {"string_tool", {{"arg1", ovms::ParameterType::STRING}}}, + {"int_tool", {{"arg1", ovms::ParameterType::NUMBER}}}, + {"float_tool", {{"arg1", ovms::ParameterType::NUMBER}}}, + {"bool_tool", {{"arg1", ovms::ParameterType::BOOLEAN}}}, + {"object_tool", {{"arg1", ovms::ParameterType::OBJECT}}}, + {"list_tool", {{"arg1", ovms::ParameterType::ARRAY}}}}; + class OnyxOutputParserTest : public ::testing::Test { protected: std::unique_ptr outputParser; @@ -107,11 +178,6 @@ class OnyxOutputParserTest : public ::testing::Test { return outputParser->parse(generatedTokens, toolsAvailable); } - // Shared by streaming tests: compares a parseChunk() delta against the expected JSON, - // masking the randomly generated tool call id (kept as one helper rather than the same - // id-masking block duplicated per test, mirroring Qwen3CoderOutputParserTest usage). - static void assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk); - // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, // using rapidjson's serializer to handle all escaping. This lets us write raw PLC/Python // code in tests without manually counting backslashes. @@ -153,9 +219,11 @@ class OnyxOutputParserTest : public ::testing::Test { } }; +// ============================================================================= +// Reasoning / final-answer framing (UNCHANGED by the new drop). // A single generate() call stops at the first "<|eom|>"/"<|eot|>" (both are configured as -// eos tokens for Onyx), so only one of these three segment shapes is ever produced at a time. - +// eos tokens for Onyx), so only one of these segment shapes is ever produced at a time. +// ============================================================================= TEST_F(OnyxOutputParserTest, FinalAnswerWithExplicitUserRecipient) { ParsedOutput parsedOutput = generateParsedOutput(" to=user<|message|>It's 65F in SF.<|eot|>"); @@ -182,27 +250,89 @@ TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { EXPECT_EQ(parsedOutput.toolCalls.size(), 0); } -TEST_F(OnyxOutputParserTest, ToolCallWithRawJsonArguments) { - ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); +// ============================================================================= +// Unary tool-call parsing (ATEM). Arguments are typed via the tool schema. +// ============================================================================= +TEST_F(OnyxOutputParserTest, ToolCallWithAtemArguments) { + // Live-model shape: " to=<|message|>...<|eom|>". + // Both params are strings per get_weather's schema, so both stay quoted. + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}})); EXPECT_EQ(parsedOutput.content, ""); EXPECT_EQ(parsedOutput.reasoning, ""); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); - // Onyx passes arguments through verbatim -- no schema-driven reformatting like qwen3coder. - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); } +TEST_F(OnyxOutputParserTest, ToolCallWithNoArguments) { + // No-argument call, exactly as captured live (get_location_gps/get_current_time). + ParsedOutput parsedOutput = generateParsedOutput(onyxToolTurn("get_current_location", {})); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_current_location"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithSchemaTypedIntegerArgument) { + // arg2 is declared integer in the schema, so the untyped ATEM value "3141522" must be + // serialized as a JSON number (not a quoted string), while arg1 stays a quoted string. + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("string_int_tool", {{"arg1", "hello"}, {"arg2", "3141522"}})); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "string_int_tool"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"arg1":"hello","arg2":3141522})"); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningThenToolCall) { + // Realistic live shape (see chat_with_tools.py): in a SINGLE generation the model first emits + // a private reasoning turn ("to=self...<|eom|>") and then a tool-call turn. The reasoning + // parser must lift the reasoning out AND strip the following turn's harmony envelope, leaving + // only the ATEM block for the tool parser to consume -- so content ends up empty. This is the + // path the eos-token fix unblocked (previously "<|eom|>" was an eos token and generation + // stopped after the reasoning turn, so the tool call was never produced). + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>Let me check the location first.<|eom|>" + + onyxToolTurn("get_current_location", {}, "<|eot|>")); + + EXPECT_EQ(parsedOutput.reasoning, "Let me check the location first."); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_current_location"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{}"); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningThenToolCallWithArgs) { + // Same combined reasoning+tool-call turn, but the tool call carries schema-typed arguments. + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>I should look up Paris weather.<|eom|>" + + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}}, "<|eot|>")); + + EXPECT_EQ(parsedOutput.reasoning, "I should look up Paris weather."); + EXPECT_EQ(parsedOutput.content, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); +} + TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { - // OutputParser::parse() only invokes the tool parser when toolsAvailable is true. - ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>", /*toolsAvailable=*/false); + // OutputParser::parse() only invokes the tool parser when toolsAvailable is true, so the + // ATEM block is never extracted and remains (verbatim) somewhere in content. + std::string turn = onyxToolTurn("get_weather", {{"location", "Paris"}}); + ParsedOutput parsedOutput = generateParsedOutput(turn, /*toolsAvailable=*/false); EXPECT_EQ(parsedOutput.toolCalls.size(), 0); - // Known current limitation: the reasoning parser (which always runs) intentionally - // leaves "to=functions." segments untouched so the tool parser can claim them -- but - // if the tool parser never runs, the raw wrapped segment is surfaced as-is in content. - EXPECT_EQ(parsedOutput.content, " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"); + // Known current limitation mirrored from the previous drop: with the tool parser + // disabled, the raw ATEM block is surfaced as-is in content. Use a substring check so + // the test does not over-constrain how much of the " to=..."/"<|message|>" envelope the + // reasoning parser strips around it. + EXPECT_NE(parsedOutput.content.find(atemBlock("get_weather", {{"location", "Paris"}})), std::string::npos) + << parsedOutput.content; } TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { @@ -214,75 +344,30 @@ TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { } // ============================================================================= -// Streaming is implemented on top of OnyxToolParserImpl, a pure state machine that -// accumulates raw text and hands back a fully assembled tool call once its "<|eom|>" -// end tag is seen (mirroring Qwen3CoderToolParserImpl) -- unary parse() below drives -// that same impl with the whole content as a single chunk, so it is the single-shot -// degenerate case of streaming, not a parallel implementation of the tag walk. +// Streaming. Implemented on top of OnyxToolParserImpl, a state machine that +// accumulates raw text and hands back a fully assembled tool call once its ATEM end +// tag ("") is seen -- mirroring Qwen3CoderToolParserImpl. Unary +// parse() above drives that same impl with the whole content as one chunk, so it is the +// single-shot degenerate case of streaming, not a parallel implementation of the tag walk. // -// Because Onyx's arguments are already a complete raw JSON blob needing no per-parameter -// schema coercion, they are still sent to the client as a single delta once the tool call -// closes (matching Qwen3CoderToolParser's sendFullDelta) rather than streamed incrementally -// as raw text arrives -- only the function name streams as its own delta once known. +// Like qwen3coder, the function name streams as its own first delta once +// "" closes, and the fully-typed arguments blob is sent as a single +// delta once the tool call closes (there is no per-parameter incremental streaming). // -// Chunk boundaries below are deliberately awkward (splitting the function name and the -// JSON arguments mid-token) to exercise the Content/InsideName/InsideArguments state -// machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// Chunk boundaries below are deliberately awkward (splitting tags and values mid-token) to +// exercise the Content/InsideToolCall/InsideFunctionName/InsideFunction/InsideParameter*/ +// AfterFunction state machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// +// NOTE on the harmony envelope: the leading " to=<|message|>" that precedes the ATEM +// block in real output is treated as generic content by the shared OutputParser streaming +// framework (the tool parser's start tag is ""). Streaming-mode +// stripping of that envelope is a separate, not-yet-designed concern (see the TODO in +// OnyxReasoningParser::parseChunk), so it is intentionally out of scope here -- this test +// focuses on the ATEM tool-call state machine itself, exactly as qwen3coder's does. // ============================================================================= -void OnyxOutputParserTest::assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { - if (!expectedDelta.has_value()) { - EXPECT_FALSE(doc.has_value()) << "Expected nullopt for chunk: " << chunk; - return; - } - ASSERT_TRUE(doc.has_value()) << "Expected a delta for chunk: " << chunk; - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc->Accept(writer); - std::string docStr = buffer.GetString(); - std::string expected = expectedDelta.value(); - // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings - const std::string idKey = "\"id\":\""; - auto docIdPos = docStr.find(idKey); - auto expectedIdPos = expected.find(idKey); - if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { - auto docIdStart = docIdPos + idKey.size(); - auto docIdEnd = docStr.find("\"", docIdStart); - auto expectedIdStart = expectedIdPos + idKey.size(); - auto expectedIdEnd = expected.find("\"", expectedIdStart); - ASSERT_NE(docIdEnd, std::string::npos); - ASSERT_NE(expectedIdEnd, std::string::npos); - std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); - std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); - EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; - EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; - docStr.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); - expected.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); - } - EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; -} - TEST_F(OnyxOutputParserTest, StreamingSimpleToolCall) { - // Mirrors Qwen3CoderOutputParserTest.StreamingSimpleToolCall's rigor: adversarial - // chunk boundaries, content before/between tool calls, complex argument values - // (PLC structured text, Python with triple-quotes/f-strings/escape sequences), - // adapted to Onyx's "to=functions.<|message|><|eom|>" format. - // - // Unlike qwen3coder there is no per-parameter incremental streaming to test - // (qwen3coder's tags) since Onyx's arguments are a single raw JSON blob. - // However, content before/between tool calls IS tested because the OutputParser - // streaming framework handles that generically (UNKNOWN -> CONTENT transition when no - // start tag is found). - // - // Key structural differences from qwen3coder: - // - Start tag is "to=functions." (not "") - // - Name delimiter is "<|message|>" (not ">") - // - End tag is "<|eom|>" (not "") - // - Arguments are a single raw JSON blob (not per-parameter XML tags) - // - PLC/Python code must be JSON-escaped within the arguments blob - - // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). - // Written as a raw string literal so it's human-readable; wrapRawCodeAsToolArgs() - // handles all the JSON escaping at runtime. + // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). Written as a + // raw string literal so it's human-readable; wrapRawCodeAsToolArgs() handles JSON escaping. const std::string plcCode = R"(FUNCTION FC_CreateJsonPayload : STRING VAR_INPUT Value1 : REAL; @@ -306,8 +391,7 @@ END_VAR END_FUNCTION)"; - // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors - // qwen3coder's last test case). + // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors qwen3coder). const std::string pythonCode = R"( if __name__ == "__main__": addresses = {} @@ -320,85 +404,66 @@ if __name__ == "__main__": std::vector>> chunkToDeltaVec{ // Content before any tool call -- OutputParser sees no start tag match, emits content. {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, - // Start tag "to=functions." split across several arbitrarily small chunks. - // Note: leading space before "to=" is just normal content/separator; the start - // tag the framework looks for is "to=functions." without the space. - {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"=fun", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ctions.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - // Function name streams in across several small chunks -- still no delta - // (OnyxToolParserImpl is in InsideName state, accumulating). - {"get", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"_", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // ATEM start tag "" split across several arbitrarily small chunks. + {"\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "" split mid-tag and mid-name -- name delta emitted + // once the full opening tag (up to the closing "\">") lands. + {"" itself split mid-tag -- name delta emitted once the full tag lands. - {"<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, - // Raw JSON argument text (with a nested object) split at awkward byte boundaries. - {"{\"locat", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ion\":\"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ris\",\"opt", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ions\":{\"unit", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"\":\"cel", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"sius\"}}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - // "<|eom|>" split mid-tag -- closes the tool call once complete. - {"<|e", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"om|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"options\":{\"unit\":\"celsius\"}}"}}]}})"}, - // Content between tool calls (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). - // In TOOL_CALLS_WAITING_FOR_TOOL phase, text without start tag match waits for more. - {"POTENTIALLY EXISINT CONTENT", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - // Second tool call -- start tag + name + <|message|> split across tiny chunks. - {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"=functi", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ons.str", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ing_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|messa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ge|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, - // Arguments split across chunks. - {"{\"arg1\":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"\"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"NG_VALUE\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|eo", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"m|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, - // More content between tool calls (mirrors "CONTENT_AFTER_TOOL_CALL"). - {"CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - // Third tool call -- string_int_tool with two parameters in JSON (integer stays - // numeric). Start tag + name + <|message|> split differently from previous calls. - {" to=func", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"tions.strin", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"g_int_tool<|", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, - // Arguments with a leading \n in the string value (matches qwen3coder's - // "\nANOTHER_STRING_VALUE" pattern) and an integer parameter. - {"{\"arg1\":\"\\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"ANOTHER_STRING_VALUE\",\"ar", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"g2\":314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"1522}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, - // "NOTHING IMPORTANT HERE" content between calls (mirrors qwen3coder). - {"NOTHING IMPORTANT HERE", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - // A "bfcl draft" style call -- cd tool. Start tag arrives with some preceding - // text just like qwen3coder's "part of bfcl 'draft'.\n\n\n" pattern. - {"part of bfcl 'draft'.\n\n to=functions.cd<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, - {"{\"folder\":\"ResearchDocs\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, - // PLC structured text code as a tool argument (mirrors qwen3coder's - // FC_CreateJsonPayload test). Raw code is defined above as plcCode; the helper - // wrapRawCodeAsToolArgs() handles all JSON escaping via rapidjson so we don't - // need to manually count backslashes. Sent as a single chunk since the interesting - // escaping complexity is in the content, not in chunk-boundary splitting. - {" to=functions.string_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, - {wrapRawCodeAsToolArgs(plcCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|eom|>", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, - // Python code with triple-quotes, f-strings, escape sequences (mirrors - // qwen3coder's last test case). Also sent as a single chunk -- the chunk-boundary - // adversarial testing is covered by the earlier tool calls above. - {" to=functions.string_tool<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, - {wrapRawCodeAsToolArgs(pythonCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, - {"<|eom|>", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, + {"er\">\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Parameter "location" -> "Paris" (string per schema, stays quoted). Split awkwardly. + {"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ris\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Parameter "unit" -> "celsius". + {"celsius\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "" then "" split mid-tag -- closes the tool call, + // full typed args delta emitted once the end tag completes. + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]}})"}, + // Harmony envelope + content between tool calls -- swallowed while waiting for the next + // "" (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). + {"<|eom|><|start|>assistant to=string_tool<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Second tool call -- start tag + invoke name split across tiny chunks. + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, + // arg1 (string) split across chunks. + {"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"NG_VALUE\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, + // More envelope/content between tool calls. + {"<|eom|><|start|>assistant to=string_int_tool<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Third tool call -- string_int_tool: arg1 string (with a leading escaped newline in + // the value) + arg2 integer (stays numeric per schema). + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, + {"\\nANOTHER_STRING_VALUE\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"1522\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // arg1's value is a literal backslash-n (C++ "\\n"), a STRING param. It round-trips + // through two JSON layers: value -> arguments string ("\\n") -> delta string ("\\\\n"). + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, + // Envelope/content before a "bfcl draft" style call -- cd tool, arriving with preceding + // text like qwen3coder's "part of bfcl 'draft'." pattern. + {"<|eom|><|start|>assistant to=cd<|message|>part of bfcl draft.\n\n\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, + {"ResearchDocs\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, + // PLC structured text code as a string tool argument (mirrors qwen3coder's + // FC_CreateJsonPayload test). Raw code defined above; wrapRawCodeAsToolArgs() / + // expectedArgsDelta() handle JSON escaping. Sent as a single chunk -- the interesting + // complexity here is the value escaping, not chunk-boundary splitting. + {"<|eom|><|start|>assistant to=string_tool<|message|>\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, + {"" + plcCode + "\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, + // Python code with triple-quotes, f-strings, escape sequences (mirrors qwen3coder's + // last case). Also single-chunk value; finishes generation with STOP. + {"<|eom|><|start|>assistant to=string_tool<|message|>\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, + {"" + pythonCode + "\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\n", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, }; for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { @@ -475,37 +540,44 @@ if __name__ == "__main__": // ============================================================================= // Proves the "unary is an edge case of streaming" property holds structurally, not -// just by coincidence: OnyxToolParser::parse() literally drives the same -// OnyxToolParserImpl used by parseChunk() (see onyx_tool_parser.cpp), so this is -// really just re-checking that the unary entry point wires into the same state -// machine already covered above. +// just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used +// by parseChunk() (see onyx_tool_parser.cpp), so this re-checks that the unary entry +// point wires into the same state machine covered above. // ============================================================================= TEST_F(OnyxOutputParserTest, UnaryToolCallMatchesStreamingReuse) { - ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + ParsedOutput parsedOutput = generateParsedOutput( + onyxToolTurn("get_weather", {{"location", "Paris"}, {"unit", "celsius"}})); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); } TEST_F(OnyxOutputParserTest, UnaryTwoSequentialToolCalls) { + // Two tool-call turns back to back, as the model emits them with ignore_eos (each turn is + // re-introduced by "<|start|>assistant to=<|message|>"). Both take a single string arg. ParsedOutput parsedOutput = generateParsedOutput( - " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"); + onyxToolTurn("get_weather", {{"location", "SF"}}) + + "<|start|>assistant" + onyxToolTurn("get_time", {{"city", "SF"}})); ASSERT_EQ(parsedOutput.toolCalls.size(), 2); EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); - EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"SF"})"); EXPECT_EQ(parsedOutput.toolCalls[1].name, "get_time"); - EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, R"({"city":"SF"})"); } // ============================================================================= -// Direct OnyxToolParserImpl unit tests -- mirrors Qwen3CoderOutputParserTest's -// TestJustParserImplUnary*/TestJustParserImplStreamStep* layer (which exercises the -// state machine directly, below OutputParser/OnyxToolParser), previously untested here. +// Direct OnyxToolParserImpl unit tests -- exercise the state machine directly, below +// OutputParser/OnyxToolParser (mirrors Qwen3CoderOutputParserTest's TestJustParserImpl* +// layer). These use plain-string parameter values whose correct JSON serialization does +// not depend on schema typing, so they can drive the impl through its current public API +// (parseChunk / getCurrentFunctionName / removeToolCallsFromContentIfNeeded) directly. +// NOTE: when the impl is made schema-driven like qwen3coder (its ctor then taking a +// ToolsParameterTypeMap_t), these direct constructions will need that argument. // ============================================================================= TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { - const std::string input = " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"; + const std::string input = onyxToolTurn("get_weather", {{"location", "Paris"}}); auto content = input; ovms::OnyxToolParserImpl parser; auto callsOpt = parser.parseChunk(content); @@ -515,8 +587,7 @@ TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { EXPECT_TRUE(status.ok()) << status.string(); ASSERT_EQ(calls.size(), 1) << input; EXPECT_EQ(calls[0].name, "get_weather"); - EXPECT_EQ(calls[0].arguments, "{\"location\":\"Paris\"}"); - EXPECT_EQ(content, ""); + EXPECT_EQ(calls[0].arguments, R"({"location":"Paris"})"); } TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { @@ -531,20 +602,40 @@ TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { } TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithTwoToolCalls) { - const std::string input = " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"; + const std::string input = + onyxToolTurn("get_weather", {{"location", "SF"}}) + + "<|start|>assistant" + onyxToolTurn("get_time", {{"city", "SF"}}); auto content = input; ovms::OnyxToolParserImpl parser; auto callsOpt = parser.parseChunk(content); ASSERT_TRUE(callsOpt.has_value()); ToolCalls_t& calls = callsOpt.value(); - auto status = parser.removeToolCallsFromContentIfNeeded(content); - EXPECT_TRUE(status.ok()) << status.string(); ASSERT_EQ(calls.size(), 2) << input; EXPECT_EQ(calls[0].name, "get_weather"); - EXPECT_EQ(calls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(calls[0].arguments, R"({"location":"SF"})"); EXPECT_EQ(calls[1].name, "get_time"); - EXPECT_EQ(calls[1].arguments, "{\"city\":\"SF\"}"); - EXPECT_EQ(content, ""); + EXPECT_EQ(calls[1].arguments, R"({"city":"SF"})"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCallThenTruncatedOpen) { + // A completed tool call followed by a second call truncated before "" + // (generation cut off mid-arguments). The completed call is still extracted, and content + // removal must drop BOTH the completed block and the dangling open block -- not bail out and + // leave every block in content (the pre-fix behavior on a begin/end tag-count mismatch). + const std::string input = + onyxToolTurn("get_weather", {{"location", "SF"}}) + + " to=get_time<|message|>\n\nS"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ASSERT_EQ(callsOpt.value().size(), 1) << input; + EXPECT_EQ(callsOpt.value()[0].name, "get_weather"); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + // Both ATEM blocks are gone (the impl strips only the blocks, not the surrounding harmony + // envelope -- that is the reasoning parser's job at the OutputParser level). + EXPECT_EQ(content.find("" is seen, even though the + // tool call has not closed yet (no ""). + const std::string input = " to=get_weather<|message|>\n\n"; auto content = input; ovms::OnyxToolParserImpl parser; auto stepResult = parser.parseChunk(content); @@ -566,7 +659,7 @@ TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithPartialToolCall) { } TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { - const std::string input = " to=functions.get_current_location<|message|>{}<|eom|>"; + const std::string input = onyxToolTurn("get_current_location", {}); auto content = input; ovms::OnyxToolParserImpl parser; auto stepResult = parser.parseChunk(content); @@ -578,42 +671,57 @@ TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { } // ============================================================================= -// Qwen3CoderOutputParserTest test cases, for reference/parity comparison (this file -// intentionally does not have a 1:1 test for every one of these -- see inline notes -// on why some don't apply to Onyx's simpler, non-schema-driven, single-JSON-blob -// argument format): -// Parse1ToolCall1Function1ArgumentTagsNewline -// Parse1ToolCall1Function1ArgumentNoProperBeginTag -// Parse1ToolCallNestedXmlNotFromSchema -// ParseTwoToolCalls1Function1ArgumentTagsNoNewline -// Parse1ToolCall1Function1ArgumentTagsNoNewline -// Parse1ToolCall1Function1ArgumentMultilineValue -// TestJustParserImplUnaryToolCall -- covered above -// TestJustParserImplUnaryWithNoToolCall -- covered above -// TestJustParserImplUnaryWithContent -- N/A: Onyx's grammar never -// has plain content before/after a tool-call tag within the same generated turn -// TestJustParserImplUnaryWithThreeParameters -- N/A: no per-parameter -// schema-driven typing; arguments are always a single opaque JSON blob -// TestJustParserImplUnaryWithEnforcementOfStringParameter -- N/A, same reason -// TestJustParserImplUnaryWithNotPresentToolSchema -- N/A, same reason (Onyx -// never even looks at tool schemas -- see ToolCallWithRawJsonArguments above) -// TestJustParserImplUnaryWithJsonObjectArgument -- covered by nested-object -// case in StreamingSimpleToolCall above -// TestJustParserImplUnaryWithTwoToolCalls -- covered above -// TestJustParserImplUnaryToolCallNoMatchingToolParameterTypeMapEntry -- N/A, same reason -// TestJustParserImplUnaryToolCallWithRepeatedArgument -- N/A, same reason (no -// per-parameter parsing to have a "repeated argument" concept at all) -// TestJustParserImplStreamStepWithMoreThan1StateChange -- covered by -// TestJustParserImplUnaryWithTwoToolCalls above (both calls resolve in one parseChunk) -// TestJustParserImplStreamStepWithNoStateChange -- covered above -// TestJustParserImplStreamStepWithPartialToolCall -- covered above -// TestJustParserImplStreamStepWithTwoToolCalls -- covered by -// TestJustParserImplUnaryWithTwoToolCalls above -// TestJustParserImplStreamStepWithToolCallNoArgs -- covered above -// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes -- N/A: -// parametrized over per-parameter type coercion (string/int/float/bool/object/list), -// which does not exist for Onyx (raw JSON passthrough only) -// StreamingSimpleToolCall -- covered above (adapted; -// see comment on that test for what was intentionally omitted/adjusted) +// Parametrized readability test for schema-driven argument typing -- direct analogue of +// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes, adapted +// to the ATEM block. A single value is typed according to the tool schema: +// string->quoted, integer/number->numeric, boolean->true/false (also normalizing Python-style +// True/False), object/array->parsed JSON. Feeds a bare ATEM block (no harmony envelope) so +// removeToolCallsFromContentIfNeeded() leaves content empty, mirroring qwen3coder feeding a +// bare block. // ============================================================================= +class OnyxOutputParserParametrizedTest : public OnyxOutputParserTest, public ::testing::WithParamInterface> { +}; + +TEST_P(OnyxOutputParserParametrizedTest, TestJustParserImplWithVariousArgumentTypes) { + const std::string& toolName = std::get<0>(GetParam()); + const std::string& argName = std::get<1>(GetParam()); + const std::string& paramValue = std::get<2>(GetParam()); + const std::string& expectedArguments = std::get<3>(GetParam()); + + const std::string input = atemBlock(toolName, {{argName, paramValue}}); + auto content = input; + ovms::OnyxToolParserImpl parser(onyxToolsParametersTypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()) << input; + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, toolName); + EXPECT_EQ(calls[0].arguments, expectedArguments) << input; + EXPECT_EQ(parser.getCurrentState(), ovms::OnyxToolParserImpl::State::Content) << input; + EXPECT_EQ(content, "") << input; +} +const std::vector> onyxParamValueAndExpectedArgumentsVec = { + {"string_tool", "arg1", "value1", R"({"arg1":"value1"})"}, + {"int_tool", "arg1", "42", R"({"arg1":42})"}, + {"float_tool", "arg1", "52.32", R"({"arg1":52.32})"}, + {"bool_tool", "arg1", "true", R"({"arg1":true})"}, + {"bool_tool", "arg1", "false", R"({"arg1":false})"}, + {"bool_tool", "arg1", "True", R"({"arg1":true})"}, + {"bool_tool", "arg1", "False", R"({"arg1":false})"}, + {"object_tool", "arg1", R"({"a":1,"b":{"c":"asd"}})", R"({"arg1":{"a":1,"b":{"c":"asd"}}})"}, + {"list_tool", "arg1", "[1, 2, 3]", R"({"arg1":[1,2,3]})"}, + {"list_tool", "arg1", R"(["a","b","c"])", R"({"arg1":["a","b","c"]})"}, + {"object_tool", "arg1", R"([{"a":1},{"b":2}])", R"({"arg1":[{"a":1},{"b":2}]})"}}; + +INSTANTIATE_TEST_SUITE_P( + OnyxOutputParserParametrizedTestInstance, + OnyxOutputParserParametrizedTest, + ::testing::ValuesIn(onyxParamValueAndExpectedArgumentsVec), + [](const ::testing::TestParamInfo& info) { + std::string name = std::get<0>(info.param) + "_" + std::get<2>(info.param); + std::replace_if(name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_'); + return name; + }); From d6e0b585b5df852f5fca942843bc6af01bd5d337 Mon Sep 17 00:00:00 2001 From: sys_k8sworker Date: Fri, 31 Jul 2026 13:46:55 +0200 Subject: [PATCH 05/38] changed install deps to use private repos --- windows_install_build_dependencies.bat | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 3ac79c2920..60ae7ef676 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -139,7 +139,7 @@ IF /I EXIST %bash_path% ( :: Set default OV_USE_BINARY if not set if "%OV_USE_BINARY%"=="" ( - set "OV_USE_BINARY=1" + set "OV_USE_BINARY=0" ) set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" @@ -248,13 +248,13 @@ IF /I EXIST %BAZEL_SHORT_PATH%\openvino_src ( ) IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( - git clone https://github.com/%OV_SOURCE_ORG%/openvino %BAZEL_SHORT_PATH%\openvino_src + git clone https://github.com/intel-sandbox/openvino.private %BAZEL_SHORT_PATH%\openvino_src ) set "BACK_CWD=%cd%" cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin -git checkout %OV_SOURCE_BRANCH% +git checkout muse_onyx if !errorlevel! neq 0 exit /b !errorlevel! git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! @@ -264,7 +264,7 @@ IF /I NOT EXIST build ( ) cd build set "TBB_DIR=" -cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF -DPython3_EXECUTABLE=%PYTHONHOME%\python.exe .. if !errorlevel! neq 0 exit /b !errorlevel! cmake --build . --config Release --verbose -j if !errorlevel! neq 0 exit /b !errorlevel! @@ -298,11 +298,11 @@ if !errorlevel! neq 0 exit /b !errorlevel! ::::::::::::::::::::::: OpenVINO GenAI IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( - git clone https://github.com/%OV_GENAI_ORG%/openvino.genai.git %BAZEL_SHORT_PATH%\openvino_genai_src + git clone https://github.com/intel-sandbox/openvino.genai.private %BAZEL_SHORT_PATH%\openvino_genai_src ) cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin -git checkout %OV_GENAI_BRANCH% +git checkout muse_onyx if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( From 7b10dabe24511801c58683ad29c11667e5e41bb6 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 31 Jul 2026 13:47:53 +0200 Subject: [PATCH 06/38] Logging --- src/llm/io_processing/onyx/onyx_tool_parser.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 1e05c4e4c4..7caa00d2c0 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -77,6 +77,9 @@ void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameter temp.Parse(parameterValueAsString.c_str()); if (temp.HasParseError()) { // Not valid JSON -> insert as a string value. + rapidjson::ParseErrorCode errorCode = temp.GetParseError(); + size_t errorOffset = temp.GetErrorOffset(); + SPDLOG_TRACE("RapidJSON can not parse parameter: {} with value: {}; error at offset: {}; code: {}; falling back to inserting value as string", this->currentParameterName, parameterValueAsString, errorOffset, rapidjson::GetParseError_En(errorCode)); rapidjson::Value v; v.SetString(parameterValueAsString.c_str(), static_cast(parameterValueAsString.size()), allocator); if (!currentFunctionArgsDoc.HasMember(keyVal)) { @@ -283,11 +286,13 @@ void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { // already sent the first delta for the function currently being read + SPDLOG_TRACE("Skipping first delta, already sent for current function, returnedFirstDeltas.size(): {} returnedCompleteDeltas.size(): {}", returnedFirstDeltas.size(), returnedCompleteDeltas.size()); return std::nullopt; } int currentToolCallIndex = ++this->toolCallIndex; rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); this->returnedFirstDeltas.insert(currentToolCallIndex); + SPDLOG_DEBUG("First delta doc: {}", documentToString(doc)); return doc; } @@ -303,13 +308,17 @@ std::optional OnyxToolParser::sendFullDelta(const ToolCalls rapidjson::Document argumentsWrapper; argumentsWrapper.SetObject(); rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); + SPDLOG_TRACE("Tool call arguments string: {}", toolCall.arguments); argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); - return wrapDelta(argumentsWrapper, this->toolCallIndex); + auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); + SPDLOG_DEBUG("Full delta doc: {}", documentToString(currentDelta)); + return currentDelta; } -std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { // streamParser returns assembled toolCalls once a call closes (""); // until then, if the function name is already known, send its first delta once. + SPDLOG_DEBUG("Chunk: '{}', finishReason: {}", newChunk, static_cast(finishReason)); this->lazyFillInitToolParametersTypesMap(); if (newChunk.empty()) { return std::nullopt; From ec9775778e37a026846139faed7f89e8a03c1706 Mon Sep 17 00:00:00 2001 From: sys_k8sworker Date: Fri, 31 Jul 2026 14:00:57 +0200 Subject: [PATCH 07/38] patch with input applied --- .../chat_template_onyx.jinja | 205 ++++++++++++++++ .../io_processing/chat_template/analyzer.cpp | 1 + src/llm/io_processing/chat_template/probe.cpp | 1 + .../chat_template_end_to_end_jinja_test.cpp | 37 +-- .../chat_template_end_to_end_minja_test.cpp | 98 ++------ .../chat_templates/chat_template_onyx.jinja | 231 +++++++++++++++--- 6 files changed, 429 insertions(+), 144 deletions(-) create mode 100644 extras/chat_template_examples/chat_template_onyx.jinja diff --git a/extras/chat_template_examples/chat_template_onyx.jinja b/extras/chat_template_examples/chat_template_onyx.jinja new file mode 100644 index 0000000000..08ed22eb07 --- /dev/null +++ b/extras/chat_template_examples/chat_template_onyx.jinja @@ -0,0 +1,205 @@ +{#- + Modifications: + * Adding support for arguments as string so that minja does not detect the need to apply polyfills. This is dead path for OVMS as it always converts arguments to dict anyway. +#} + +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|image|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {{- '\n\n' -}} + {%- if args is not mapping -%} + {{- '\n' + tc.function.arguments + '\n' -}} + {%- else -%} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%}true{%- else -%}false{%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {%- endif -%} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} +{%- endmacro -%} + +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} + +{{- bos_token -}} + +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} + +{%- if not ns.has_system and (add_generation_prompt or tools) -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} +{%- endif -%} + +{%- for message in messages -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + + {%- if role == 'system' -%} + {{- '<|start|>system<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} + +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%} diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index 86c0f30ff2..cecaae6071 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -57,6 +57,7 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp contains(templateSource, "<|eom|>") && contains(templateSource, "<|eot|>")) { result.detectedToolParser = "onyx"; result.detectedReasoningParser = "onyx"; + result.caps.supportsToolCalls = true; return result; } diff --git a/src/llm/io_processing/chat_template/probe.cpp b/src/llm/io_processing/chat_template/probe.cpp index f2b068b46d..887f8f88ed 100644 --- a/src/llm/io_processing/chat_template/probe.cpp +++ b/src/llm/io_processing/chat_template/probe.cpp @@ -96,6 +96,7 @@ static bool analyzeProbeToolArgumentResults(bool strOk, const std::string& strOu return output.find("\"" + PROBE_NEEDLE + "\": \"") != std::string::npos || output.find("\"" + PROBE_NEEDLE + "\":\"") != std::string::npos || output.find("") != std::string::npos || + output.find("") != std::string::npos || output.find("") != std::string::npos || output.find(PROBE_NEEDLE + ":<|") != std::string::npos || output.find(PROBE_NEEDLE + "=") != std::string::npos; diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index a8a7d1820e..13c73a9d9f 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -631,17 +631,6 @@ What's the weather in Paris?<|im_end|> EXPECT_EQ(appliedOutput, expectedOutput); } -// ============================================================================= -// Onyx (early preview model) chat template, rendered via the real Python Jinja2 -// engine. Onyx's template does not read the standard OpenAI "tool_calls" array -// at all -- only message['content'] (plain string) and an Onyx-specific -// message['recipient'] field (e.g. "functions.get_weather", "self", "user"). -// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp), -// but deliberately leaves caps.supportsToolCalls false: that flag means "this -// template natively re-serializes an incoming OpenAI tool_calls array", which -// this test demonstrates Onyx's template does NOT do (the tool call is silently -// dropped below). -// ============================================================================= TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); ASSERT_FALSE(chatTemplate.empty()); @@ -660,22 +649,18 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); - EXPECT_FALSE(caps.supportsToolCalls); - EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); - // Unlike the minja path (which has its own generic tool-call fallback), - // the real Python Jinja2 engine has no such fallback: the template renders - // message['content'] literally, i.e. the empty string, and the tool call - // information is silently dropped. - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|><|eot|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=get_weather<|message|> + +Paris +celsius + +<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } -// ============================================================================= -// Onyx's own message shape via Python Jinja2: "recipient" field instead of the -// OpenAI "tool_calls" array. Ends the turn with "<|eom|>" (continuation marker) -// rather than "<|eot|>". -// ============================================================================= TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); ASSERT_FALSE(chatTemplate.empty()); @@ -689,8 +674,8 @@ TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { ASSERT_FALSE(exceptionThrownDuringApplication); - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } // ============================================================================= diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index ec97945e04..0157bcab0e 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -620,19 +620,6 @@ TEST_F(ChatTemplateEndToEndMinjaTest, MiniCPM5_ToolCallWithStringArgsExpectedToF EXPECT_TRUE(caps.missnamedReasoningField.empty()); } -// ============================================================================= -// Onyx (early preview model) chat template. Unlike every other template in this -// suite, Onyx's own Jinja template does not consume the standard OpenAI -// "tool_calls" list at all -- it only reads message['content'] (a plain string) -// and an Onyx-specific message['recipient'] field (e.g. "functions.get_weather", -// "self", "user"). Feeding it a standard tool_calls-shaped assistant message -// therefore renders an effectively empty assistant turn. -// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp) and -// sets detectedToolParser/detectedReasoningParser, but deliberately leaves -// caps.supportsToolCalls false since the template can't natively round-trip an -// OpenAI tool_calls history (demonstrated by this very test), so no input-side -// workaround is applied either -- detectedToolParser only affects output parsing. -// ============================================================================= TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; @@ -651,40 +638,18 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); - EXPECT_FALSE(caps.supportsToolCalls); - EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); - // The template itself never reads "tool_calls" (it only looks at - // message['content'] and message['recipient']). Because caps.supportsToolCalls - // is false here, OVMS does not apply its own tool-call workaround either. - // Minja's own generic fallback (used for templates it detects have no native - // tool-call rendering) kicks in instead and serializes the whole message - // (tool_calls + content) as a JSON blob into message['content'] -- the - // function name/args are NOT lost, but they end up as raw, unparsed JSON text - // rather than in Onyx's native " to=functions." / <|eom|> framing. - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|>{ - "tool_calls": [ - { - "name": "get_weather", - "arguments": { - "location": "Paris", - "unit": "celsius" - }, - "id": "call_abc123" - } - ], - "content": "" -}<|eot|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=get_weather<|message|> + +Paris +celsius + +<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } -// ============================================================================= -// Onyx's own message shape: instead of the OpenAI "tool_calls" array, the -// assistant turn carries a "recipient" field (here "functions.get_weather") -// and a plain-string content holding the raw JSON arguments. This is the shape -// Onyx's template actually understands, ending the turn with "<|eom|>" (a -// continuation marker) rather than "<|eot|>". -// ============================================================================= TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; @@ -698,29 +663,10 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { ASSERT_FALSE(exceptionThrownDuringApplication); - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); + std::string expectedOutput = R"(<|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } -// ============================================================================= -// Full-scope round trip: exercises every message shape the Onyx template -// natively understands in a single history, not just one shape in isolation -- -// user prompt -> assistant tool call (recipient=functions., continuation -// "<|eom|>") -> tool call response (role="tool") -> assistant final answer -// (recipient="user", "<|eot|>"). -// -// This surfaces a second, previously undocumented gap alongside the tool_calls -// one above (see muse/chat_template_issues.md): because caps.supportsToolCalls -// is false for Onyx, ChatTemplateAdapter's generic fallback also intercepts -// plain role="tool" messages -- not just assistant tool_calls -- and rewrites -// them into a synthetic role="user" message serializing {tool, content} as a -// JSON blob, rather than passing them through to the template's own native -// "tool"-role branch (which expects message['name'] + message['content'] and -// would render "<|start|>tool <|message|>...<|eot|>"). So even a chat -// history built entirely out of Onyx's own native fields (recipient) still -// does not round-trip once a plain OpenAI-shaped tool response message is -// mixed in. -// ============================================================================= TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; @@ -738,25 +684,13 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { ASSERT_FALSE(exceptionThrownDuringApplication); - EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_TRUE(caps.supportsToolCalls); - // Known gap (see class comment above): the "tool" message is NOT rendered via - // the template's native "<|start|>tool <|message|>...<|eot|>" branch -- - // ChatTemplateAdapter's fallback rewrites it into a synthetic user message - // carrying a JSON blob first. std::string expectedOutput = - R"(<|start|>system<|message|>You are a helpful assistant.<|eot|>)" - R"(<|start|>user<|message|>What's the weather in Paris?<|eot|>)" - R"(<|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|>)" - "<|start|>user<|message|>{\n" - " \"tool_response\": {\n" - " \"tool\": \"get_weather\",\n" - " \"content\": \"{\\\"temperature\\\":15,\\\"unit\\\":\\\"celsius\\\"}\"\n" - " }\n" - "}<|eot|>" - R"(<|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|>)" - R"(<|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); +R"(# Valid recipients: "self", "user".<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>tool get_weather<|message|> +{"temperature":15,"unit":"celsius"} +<|eot|><|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|><|start|>assistant)"; + EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } // ============================================================================= diff --git a/src/test/llm/chat_templates/chat_template_onyx.jinja b/src/test/llm/chat_templates/chat_template_onyx.jinja index 0fa7b9d204..4acb51bde3 100644 --- a/src/test/llm/chat_templates/chat_template_onyx.jinja +++ b/src/test/llm/chat_templates/chat_template_onyx.jinja @@ -1,41 +1,200 @@ -{{- bos_token -}} -{%- macro render_parts(content) -%} -{%- if content is string -%}{{- content -}} -{%- else -%} -{%- for part in content -%} -{%- if part['type'] == 'image' -%}{{- '<|image|>' -}} -{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}} -{%- elif part['type'] == 'text' -%}{{- part['text'] -}} -{%- endif -%} -{%- endfor -%} -{%- endif -%} +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|image|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {{- '\n\n' -}} + {%- if args is not mapping -%} + {{- '\n' + tc.function.arguments + '\n' -}} + {%- else -%} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%}true{%- else -%}false{%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {%- endif -%} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} + +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} {%- endmacro -%} + +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} + +{{- bos_token -}} + {%- set ns = namespace(has_system=false) -%} -{%- for m in messages -%}{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}{%- endfor -%} -{%- if add_generation_prompt and not ns.has_system -%} -{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} + +{%- if not ns.has_system and (add_generation_prompt or tools) -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} {%- endif -%} + {%- for message in messages -%} -{%- set role = message['role'] -%} -{%- if role == 'assistant' -%} -{%- set recipient = message.get('recipient') -%} -{%- set end_turn = message.get('end_turn') -%} -{%- if end_turn is none -%} -{%- set end_turn = not (recipient and recipient != 'user') -%} -{%- endif -%} -{{- '<|start|>assistant' -}} -{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%} -{{- '<|message|>' -}}{{- render_parts(message['content']) -}} -{{- ('<|eot|>' if end_turn else '<|eom|>') -}} -{%- elif role == 'tool' -%} -{%- set name = message.get('name', '') -%} -{{- '<|start|>tool ' + name + '<|message|>' -}}{{- render_parts(message['content']) -}} -{{- '<|eot|>' -}} -{%- else -%} -{%- set header = role -%} -{%- if message.get('name') -%}{%- set header = role + ' ' + message['name'] -%}{%- endif -%} -{{- '<|start|>' + header + '<|message|>' -}}{{- render_parts(message['content']) -}} -{{- '<|eot|>' -}} -{%- endif -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + + {%- if role == 'system' -%} + {{- '<|start|>system<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} {%- endfor -%} -{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%} + +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%} From e907e74d236302cbaf0c6b4837baa7179a88a4d7 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 31 Jul 2026 16:53:53 +0200 Subject: [PATCH 08/38] Fix reasoning parser --- .../onyx/onyx_reasoning_parser.cpp | 93 ++++++++++++------- .../onyx_output_parser_test.cpp | 31 +++++++ 2 files changed, 91 insertions(+), 33 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp index 19542db01f..4a7d7fad63 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -26,52 +26,79 @@ namespace ovms { void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // TODO @atobiszei overcomplicated I think? We just need t find recipient self & them cut that part out. - // Case 1: private chain-of-thought turn (recipient="self") -> extract reasoning, - // consume the whole segment (nothing meaningful is expected to follow it within the - // same generate() call, see class comment). - size_t selfPos = parsedOutput.content.find(selfRecipientTag); - if (selfPos != std::string::npos) { + // With EOS suppression the model may produce multiple interleaved turns in one + // generation (reasoning → tool call → reasoning → tool call → ... → answer). + // We must extract ALL reasoning segments and strip ALL turn boundaries/envelopes. + + // Step 1: Extract and remove ALL "to=self<|message|>...<|eom|>" reasoning segments. + for (;;) { + size_t selfPos = parsedOutput.content.find(selfRecipientTag); + if (selfPos == std::string::npos) + break; size_t messagePos = parsedOutput.content.find(messageTag, selfPos); if (messagePos == std::string::npos) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); - return; + break; } size_t bodyStart = messagePos + messageTag.length(); size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); - parsedOutput.reasoning = (endPos != std::string::npos) - ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) - : parsedOutput.content.substr(bodyStart); - // Erase ONLY the reasoning segment (through its "<|eom|>" terminator), including the - // leading " " the template renders before "to=". With eos suppressed the model may - // continue into a tool-call / final-answer turn after reasoning; that turn must survive - // for the envelope strip below (and OnyxToolParser) to process. + std::string reasoning = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + if (!parsedOutput.reasoning.empty()) + parsedOutput.reasoning += '\n'; + parsedOutput.reasoning += reasoning; + // Erase the segment including the leading space before "to=" if present. size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; size_t eraseEnd = (endPos != std::string::npos) ? endPos + continuationEndTag.length() : parsedOutput.content.length(); parsedOutput.content.erase(segmentStart, eraseEnd - segmentStart); - // fall through to strip the envelope of any following (tool-call / final-answer) turn } - // Case 2: any other turn -- a tool-call turn (recipient="", e.g. "get_weather", now - // a BARE name after the new drop's chat template, not "functions.") or a plain final - // answer (recipient="user" or absent). Strip the generic harmony routing prefix - // ("[ to=]" + "<|message|>") and a single trailing turn terminator - // ("<|eom|>" or "<|eot|>"), leaving just the body. For a tool-call turn the body is the ATEM - // block, which OnyxToolParser (running next, see OutputParser::parse()) then extracts, - // leaving content empty; for a final answer the body is the clean text. - size_t messagePos = parsedOutput.content.find(messageTag); - if (messagePos == std::string::npos) { - // No framing found at all -- unexpected/malformed output, leave content as-is. - return; + // Step 2: Remove all "<|start|>assistant" turn boundary markers (with optional trailing space). + static const std::string turnBoundary = "<|start|>assistant"; + for (;;) { + size_t pos = parsedOutput.content.find(turnBoundary); + if (pos == std::string::npos) + break; + size_t eraseLen = turnBoundary.length(); + // Also consume one trailing space if present (before "to="). + if (pos + eraseLen < parsedOutput.content.length() && parsedOutput.content[pos + eraseLen] == ' ') + ++eraseLen; + parsedOutput.content.erase(pos, eraseLen); } - // Drop everything up to and including the first "<|message|>" (the routing prefix). - parsedOutput.content.erase(0, messagePos + messageTag.length()); - // Drop a single trailing terminator if the turn ends with one. - for (const auto& term : {continuationEndTag, turnFinalEndTag}) { - if (parsedOutput.content.size() >= term.size() && - parsedOutput.content.compare(parsedOutput.content.size() - term.size(), term.size(), term) == 0) { - parsedOutput.content.erase(parsedOutput.content.size() - term.size()); + + // Step 3: Strip envelope framing from remaining turns. Each non-self turn has + // " to=<|message|>" before its body. Find each "<|message|>" tag, look + // backwards for the closest "to=" prefix, and erase the envelope (including a + // leading space if present). This preserves content between tool-call turns. + static const std::string toPrefix = "to="; + // The envelope (" to=<|message|>") is never longer than this. + static constexpr size_t maxEnvelopeLen = 128; + for (;;) { + size_t messagePos = parsedOutput.content.find(messageTag); + if (messagePos == std::string::npos) break; + // Bound the backwards search to avoid matching "to=" in body content. + size_t searchFrom = (messagePos > maxEnvelopeLen) ? messagePos - maxEnvelopeLen : 0; + size_t toPos = parsedOutput.content.rfind(toPrefix, messagePos); + size_t eraseStart; + if (toPos != std::string::npos && toPos >= searchFrom && parsedOutput.content.find(messageTag, toPos) == messagePos) { + // Include the leading space before "to=" if present. + eraseStart = (toPos > 0 && parsedOutput.content[toPos - 1] == ' ') ? toPos - 1 : toPos; + } else { + // No "to=" found within the envelope window; erase just the tag itself. + eraseStart = messagePos; + } + parsedOutput.content.erase(eraseStart, messagePos + messageTag.length() - eraseStart); + } + + // Step 4: Remove all remaining terminators. + for (const auto& term : {continuationEndTag, turnFinalEndTag}) { + for (;;) { + size_t pos = parsedOutput.content.find(term); + if (pos == std::string::npos) + break; + parsedOutput.content.erase(pos, term.length()); } } } diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index e8b853d905..ba4d688fc4 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -242,6 +242,16 @@ TEST_F(OnyxOutputParserTest, FinalAnswerWithoutRecipient) { EXPECT_EQ(parsedOutput.toolCalls.size(), 0); } +TEST_F(OnyxOutputParserTest, LiteralToEqualsInContentNotStripped) { + // "to=" appearing in the answer body must not be mistaken for an envelope prefix. + ParsedOutput parsedOutput = generateParsedOutput( + " to=user<|message|>Send email to=admin for help.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "Send email to=admin for help."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { ParsedOutput parsedOutput = generateParsedOutput(" to=self<|message|>Let me think about this.<|eom|>"); @@ -320,6 +330,27 @@ TEST_F(OnyxOutputParserTest, PrivateReasoningThenToolCallWithArgs) { EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris","unit":"celsius"})"); } +TEST_F(OnyxOutputParserTest, ReasoningAndToolCallAndContentIsolated) { + // Reasoning + tool call + final answer in one generation: verify no cross-contamination. + ParsedOutput parsedOutput = generateParsedOutput( + " to=self<|message|>I need the weather first.<|eom|>" + + onyxToolTurn("get_weather", {{"location", "Paris"}}) + + "<|start|>assistant to=user<|message|>Here is the result.<|eot|>"); + + EXPECT_EQ(parsedOutput.reasoning, "I need the weather first."); + EXPECT_EQ(parsedOutput.content, "Here is the result."); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris"})"); + // Ensure no leakage of reasoning or ATEM XML into content. + EXPECT_EQ(parsedOutput.content.find("self"), std::string::npos); + EXPECT_EQ(parsedOutput.content.find("atem"), std::string::npos); + EXPECT_EQ(parsedOutput.content.find("weather"), std::string::npos); + // Ensure no leakage of content or ATEM XML into reasoning. + EXPECT_EQ(parsedOutput.reasoning.find("result"), std::string::npos); + EXPECT_EQ(parsedOutput.reasoning.find("atem"), std::string::npos); +} + TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { // OutputParser::parse() only invokes the tool parser when toolsAvailable is true, so the // ATEM block is never extracted and remains (verbatim) somewhere in content. From 296a91c0745e2c72414306f9ac9b21e8be022f9f Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 31 Jul 2026 20:32:56 +0200 Subject: [PATCH 09/38] BFCL to check --- .../io_processing/onyx/onyx_tool_parser.cpp | 7 ++ .../io_processing/onyx/onyx_tool_parser.hpp | 4 +- .../onyx_output_parser_test.cpp | 82 ++++++++++++++++++- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 7caa00d2c0..1dcd5c5f0e 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -253,6 +253,13 @@ OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchem BaseOutputParser(tokenizer), toolSchemas(toolSchemas), streamParser(this->toolsParametersTypes) { + // Build dynamic start tags: "" (the ATEM block itself) plus + // "to=" for each tool in the schema — so the streaming framework can detect the + // harmony envelope prefix and route to this parser instead of leaking it as content. + parsingStartTags.push_back(TOOL_START_TAG); + for (const auto& [name, _] : toolSchemas) { + parsingStartTags.push_back("to=" + name); + } } void OnyxToolParser::lazyFillInitToolParametersTypesMap() { diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index ccf7270240..3157449caf 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -141,6 +141,7 @@ class OnyxToolParser : public BaseOutputParser { int toolCallIndex{-1}; std::set returnedFirstDeltas; std::set returnedCompleteDeltas; + std::vector parsingStartTags; std::optional sendFirstDeltaIfNeeded(const std::string& functionName); std::optional sendFullDelta(const ToolCalls_t& toolCalls); @@ -153,8 +154,7 @@ class OnyxToolParser : public BaseOutputParser { void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; const std::vector& getParsingStartTags() const override { - static const std::vector startTags{TOOL_START_TAG}; - return startTags; + return parsingStartTags; } const std::vector& getSpecialParsingStartTags() const override { static const std::vector specialParsingStartTags{}; diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index ba4d688fc4..ae109d51a6 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -50,7 +50,7 @@ using namespace ovms; // (the "functions." prefix only appears if the tool itself is namespaced). The // authoritative function name is therefore read from ``, // not from the `to=` recipient. -// - Arguments are an ATEM XML block (Anthropic-style), essentially qwen3coder with +// - Arguments are an ATEM XML block, essentially qwen3coder with // `atem:` tags -- NOT a single raw JSON blob. Parameter values are rendered // UNQUOTED (e.g. 37.7749,-122.4194), // so arguments must be typed via the tool JSON schema exactly like @@ -335,10 +335,10 @@ TEST_F(OnyxOutputParserTest, ReasoningAndToolCallAndContentIsolated) { ParsedOutput parsedOutput = generateParsedOutput( " to=self<|message|>I need the weather first.<|eom|>" + onyxToolTurn("get_weather", {{"location", "Paris"}}) + - "<|start|>assistant to=user<|message|>Here is the result.<|eot|>"); + "<|start|>assistant to=user<|message|>I will provide result when I have tool call result.<|eot|>"); EXPECT_EQ(parsedOutput.reasoning, "I need the weather first."); - EXPECT_EQ(parsedOutput.content, "Here is the result."); + EXPECT_EQ(parsedOutput.content, "I will provide result when I have tool call result."); ASSERT_EQ(parsedOutput.toolCalls.size(), 1); EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"location":"Paris"})"); @@ -569,6 +569,82 @@ if __name__ == "__main__": } } +// ============================================================================= +// Streaming reasoning followed by a tool call. Verifies that: +// - reasoning_content deltas are emitted for reasoning body chunks +// - framing tags (to=self, <|message|>, <|eom|>) are swallowed (nullopt) +// - after reasoning ends, the tool call streams normally with name + args deltas +// - no reasoning leaks into content or tool_calls deltas +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { + int i = -1; + std::vector>> chunkToDeltaVec{ + // Reasoning start tag -- framework detects "to=self", enters REASONING phase. + // OnyxReasoningParser::parseChunk sees the tag and returns nullopt (swallows framing). + {"to=self", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // <|message|> separator -- also swallowed by the reasoning parser. + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Actual reasoning body chunks -- emitted as reasoning_content deltas. + {"Let me think", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"Let me think"}})"}, + {" about the weather.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" about the weather."}})"}, + // Reasoning end tag -- swallowed, framework transitions back to UNKNOWN. + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Tool call envelope (harmony prefix) -- must be swallowed, not leaked as content. + // BUG: framework transitions from REASONING→UNKNOWN after <|eom|>, and in UNKNOWN the + // envelope doesn't match any start tag ("to=self" for reasoning, "" + // for tools), so it's flushed as content. Gptoss avoids this because its tool start tag + // IS the envelope prefix ("<|channel|>commentary to="). Onyx needs equivalent handling. + {" to=get_weather<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // ATEM start tag -- enters TOOL_CALLS phase. + {"\n\n", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Parameter. + {"Paris\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Close tool call. + {"\n", ov::genai::GenerationFinishReason::STOP, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\"}"}}]}})"}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + // Normalize tool call IDs (same approach as StreamingSimpleToolCall). + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docIdEnd - docIdStart, std::string(docIdEnd - docIdStart, '*')); + expectedNoId.replace(expectedIdStart, expectedIdEnd - expectedIdStart, std::string(expectedIdEnd - expectedIdStart, '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() : "nullopt"); + } + } +} + // ============================================================================= // Proves the "unary is an edge case of streaming" property holds structurally, not // just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used From fc10076c34743454f2365a562f962d3549c84f2d Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Mon, 27 Jul 2026 16:55:29 +0200 Subject: [PATCH 10/38] Model enablement WIP --- src/BUILD | 2 + src/llm/BUILD | 32 + .../io_processing/chat_template/analyzer.cpp | 23 + .../text_content_normalization_processor.cpp | 13 + .../text_content_normalization_processor.hpp | 14 +- .../onyx/onyx_reasoning_parser.cpp | 102 +++ .../onyx/onyx_reasoning_parser.hpp | 81 +++ .../io_processing/onyx/onyx_tool_parser.cpp | 183 ++++++ .../io_processing/onyx/onyx_tool_parser.hpp | 120 ++++ src/llm/io_processing/output_parser.cpp | 16 +- .../parser_config_validation.cpp | 2 + src/test/llm/chat_template_analyzer_test.cpp | 15 + ...emplate_and_parser_onyx_roundtrip_test.cpp | 178 +++++ .../chat_template_end_to_end_jinja_test.cpp | 49 ++ .../chat_template_end_to_end_minja_test.cpp | 139 ++++ .../chat_templates/chat_template_onyx.jinja | 41 ++ ...t_content_normalization_processor_test.cpp | 20 + .../onyx_output_parser_test.cpp | 619 ++++++++++++++++++ 18 files changed, 1642 insertions(+), 7 deletions(-) create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.hpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.hpp create mode 100644 src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp create mode 100644 src/test/llm/chat_templates/chat_template_onyx.jinja create mode 100644 src/test/llm/output_parsers/onyx_output_parser_test.cpp diff --git a/src/BUILD b/src/BUILD index b0365a1a18..dfa47dc911 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2940,12 +2940,14 @@ cc_library( "test/llm/chat_template_analyzer_test.cpp", "test/llm/chat_template_adapter_test.cpp", "test/llm/chat_template_end_to_end_minja_test.cpp", + "test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp", ], deps = [ "@com_google_googletest//:gtest", "//src/llm:chat_template_analyzer", "//src/llm:chat_template_probe", "//src/llm:io_processing_input_processors", + "//src/llm:output_parsers", "//src/utils:env_guard", "//third_party:genai", ":test_platform_utils", diff --git a/src/llm/BUILD b/src/llm/BUILD index c15c20f08f..358df0e96a 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -403,6 +403,36 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_onyx_tool_parser", + hdrs = ["io_processing/onyx/onyx_tool_parser.hpp"], + srcs = ["io_processing/onyx/onyx_tool_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstatus", + ":io_processing_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_onyx_reasoning_parser", + hdrs = ["io_processing/onyx/onyx_reasoning_parser.hpp"], + srcs = ["io_processing/onyx/onyx_reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_lfm25_reasoning_parser", hdrs = ["io_processing/lfm2/lfm25_reasoning_parser.hpp"], @@ -458,6 +488,8 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_gemma4_tool_parser", ":io_processing_minicpm5_tool_parser", ":io_processing_qwen3_reasoning_parser", + ":io_processing_onyx_tool_parser", + ":io_processing_onyx_reasoning_parser", ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index a2e8a0f82c..86c0f30ff2 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -37,6 +37,29 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } + // Onyx detection — Harmony-family framing ("<|start|>{role}[ to=]<|message|> + // {content}<|eom|>/<|eot|>") without gpt-oss's "<|channel|>" marker (already handled + // above). This combination of literal tokens is not used by any other template in + // this codebase (verified against every fixture under src/test/llm/chat_templates/). + // There is no separate reasoning-specific token: Onyx routes both tool calls + // (recipient="functions.") and private reasoning (recipient="self") through the + // exact same "<|message|>...<|eom|>" framing, so both parsers are tied together here, + // like gptoss/gemma4. + // NOTE: unlike every other branch below, this deliberately does NOT set + // caps.supportsToolCalls -- that flag means "the template natively re-serializes an + // incoming OpenAI-shaped `tool_calls` array back into the model's own format", which + // Onyx's template does not do at all (it only ever reads message['recipient']/ + // message['content'], never message['tool_calls'] -- see the Onyx_ToolCallWithStringArgs + // tests in chat_template_end_to_end_{jinja,minja}_test.cpp). detectedToolParser/ + // detectedReasoningParser only affect how OVMS parses the model's *output*, which is + // unrelated to (and unaffected by) whether input history round-trips correctly. + if (contains(templateSource, "<|start|>") && contains(templateSource, "<|message|>") && + contains(templateSource, "<|eom|>") && contains(templateSource, "<|eot|>")) { + result.detectedToolParser = "onyx"; + result.detectedReasoningParser = "onyx"; + return result; + } + // Gemma4 detection if (contains(templateSource, "'<|tool_call>call:'") || contains(templateSource, "<|tool_call>call:")) { result.detectedToolParser = "gemma4"; diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 2180d5faae..871a0f5f06 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -29,6 +29,19 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { ov::genai::ChatHistory& chatHistory = std::get(req.input); for (size_t i = 0; i < chatHistory.size(); i++) { const auto content = chatHistory[i]["content"]; + if (content.is_null()) { + // TODO @atobiszei to check if really needed when we have IR + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls" sets "content": null (openai_completions.cpp stores this + // verbatim -- only a *missing* content field is defaulted to ""). Some + // chat templates (e.g. Onyx's) unconditionally render content for every + // message regardless of role/tool_calls and are not written to expect + // null there, which raises a template error instead of just omitting + // the text. Normalize null the same way a missing field is already + // defaulted, so every template sees a plain string as before. + chatHistory[i]["content"] = std::string(""); + continue; + } if (!content.is_array()) { continue; } diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index e68d892490..b6292cb302 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,11 +19,15 @@ namespace ovms { -// Flattens text-only content arrays in ChatHistory messages to plain strings. -// Parts are joined with "\n" for backward compatibility with chat templates. -// Runs for both LM and VLM chat paths: arrays that contain images (or other -// non-text modalities) are left untouched. -// Must run before ChatTemplateProcessor and after Image/Audio decoding processors. +// Flattens text-only content arrays in ChatHistory messages to plain strings, and +// normalizes an explicit "content": null (the standard OpenAI shape for e.g. an +// assistant message that only carries tool_calls) to "" -- some chat templates +// (e.g. Onyx's) unconditionally render content for every message and are not +// written to expect null there. Parts/null are joined/replaced for backward +// compatibility with chat templates. Runs for both LM and VLM chat paths: arrays +// that contain images (or other non-text modalities) are left untouched for +// ImageDecodingProcessor. +// Must run before ChatTemplateProcessor. class TextContentNormalizationProcessor : public BaseInputProcessor { public: absl::Status process(InputRequest& req) override; diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp new file mode 100644 index 0000000000..43b45a1e8f --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -0,0 +1,102 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "onyx_reasoning_parser.hpp" + +namespace ovms { + +void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // TODO @atobiszei overcomplicated I think? We just need t find recipient self & them cut that part out. + // Case 1: private chain-of-thought turn (recipient="self") -> extract reasoning, + // consume the whole segment (nothing meaningful is expected to follow it within the + // same generate() call, see class comment). + size_t selfPos = parsedOutput.content.find(selfRecipientTag); + if (selfPos != std::string::npos) { + size_t messagePos = parsedOutput.content.find(messageTag, selfPos); + if (messagePos != std::string::npos) { + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.reasoning = body; + // Drop the leading " " before "to=" (rendered by the chat template) too, if present. + size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; + parsedOutput.content.erase(segmentStart); + return; + } + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); + return; + } + + // Case 2: tool-call turn (recipient="functions.") -> leave untouched, OnyxToolParser + // (which runs after this parser, see OutputParser::parse()) is responsible for it. + if (parsedOutput.content.find(functionsRecipientTag) != std::string::npos) { + return; + } + + // Case 3: plain final answer (recipient="user" or absent) -> strip the generic + // " to=user"? + "<|message|>" + "<|eot|>" envelope, leaving just the clean text. + size_t messagePos = parsedOutput.content.find(messageTag); + if (messagePos == std::string::npos) { + // No framing found at all -- unexpected/malformed output, leave content as-is. + return; + } + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(turnFinalEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.content = body; +} + +std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // TODO @atobiszei we need to stream between recipient=self & + // TODO: streaming support is a first draft. It only forwards the chunk as + // reasoning_content once we've seen the "to=self" start tag; stripping the + // generic final-answer envelope (Case 3 above) in streaming mode is not + // implemented yet and needs its own design (the envelope prefix/suffix can + // straddle multiple chunks). + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); + return std::nullopt; + } + if (chunk.find(selfRecipientTag) != std::string::npos || + chunk.find(messageTag) != std::string::npos || + chunk.find(continuationEndTag) != std::string::npos) { + return std::nullopt; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; +} +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp new file mode 100644 index 0000000000..27218d8414 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -0,0 +1,81 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +// Onyx (early preview model) framing: +// TODO @atobiszei simplify comment. tag naming convention. no need to define all tags here +// <|start|>assistant[ to=]<|message|>{content}{<|eom|>|<|eot|>} +// The chat template never emits a ""-style dedicated reasoning tag: private +// chain-of-thought is just an assistant turn routed with recipient="self", ending in +// the continuation marker "<|eom|>" (never "<|eot|>", which is reserved for turns that +// end the whole assistant turn -- i.e. the final answer). +// +// Because generation stops at the first "<|eom|>"/"<|eot|>"/"<|end_of_text|>" (see +// generation_config.json's eos_token_id list in the Onyx HF conversion script), a single +// generate() call only ever produces ONE such framed segment. This parser is therefore +// also responsible for stripping the generic " to="+"<|message|>"+terminator +// envelope from plain final-answer turns (recipient="user" or absent) -- this class runs +// before the tool parser (see OutputParser::parse()), so it must NOT touch content when +// the envelope routes to a function call (recipient="functions."); it leaves that +// segment untouched so OnyxToolParser can find and parse it afterwards. +class OnyxReasoningParser : public BaseOutputParser { +protected: + // Marks a private chain-of-thought turn (recipient="self"). + const std::string selfRecipientTag = "to=self"; + // Marks a tool-call turn (recipient="functions.") -- left untouched here. + const std::string functionsRecipientTag = "to=functions."; + // Separates the routing prefix from the turn's body. + const std::string messageTag = "<|message|>"; + // Terminator for continuation turns (reasoning and tool calls). + const std::string continuationEndTag = "<|eom|>"; + // Terminator for turn-final turns (plain final answers). + const std::string turnFinalEndTag = "<|eot|>"; + +public: + OnyxReasoningParser() = delete; + explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{selfRecipientTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return continuationEndTag; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp new file mode 100644 index 0000000000..76b6a65e49 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -0,0 +1,183 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "src/llm/io_processing/utils.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" + +namespace ovms { + +const std::string OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG = "to=functions."; +const std::string OnyxToolParserImpl::MESSAGE_TAG = "<|message|>"; +const std::string OnyxToolParserImpl::END_TAG = "<|eom|>"; + +#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ + auto pos = this->streamContent.find(TAG, this->lastProcessedPosition); \ + if (pos == std::string::npos) { \ + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Did not find: {}", TAG); \ + break; \ + } + +bool OnyxToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + auto previousState = this->currentState; + switch (this->currentState) { + case State::Content: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(FUNCTIONS_RECIPIENT_TAG); + this->toolCallPositions.begin.push(pos); + this->lastProcessedPosition = pos + FUNCTIONS_RECIPIENT_TAG.length(); + this->currentState = State::InsideName; + break; + } + case State::InsideName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(MESSAGE_TAG); + this->currentFunctionName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + MESSAGE_TAG.length(); + this->currentState = State::InsideArguments; + break; + } + case State::InsideArguments: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(END_TAG); + std::string argumentsPart = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + END_TAG.length(); + this->currentState = State::Content; + this->toolCallPositions.end.push(this->lastProcessedPosition); + ToolCall toolCall{generateRandomId(), this->currentFunctionName, argumentsPart}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Adding tool call: id={}, name={}, arguments={}", toolCall.id, toolCall.name, toolCall.arguments); + toolCalls.emplace_back(std::move(toolCall)); + this->currentFunctionName.clear(); + break; + } + } + return previousState != this->currentState; +} + +std::optional OnyxToolParserImpl::parseChunk(const std::string& chunk) { + if (chunk.empty()) { + return std::nullopt; + } + ToolCalls_t toolCalls; + this->streamContent += chunk; + while (parseUntilStateChange(toolCalls)) { + } + if (!toolCalls.empty()) { + return std::move(toolCalls); + } + return std::nullopt; +} + +std::optional OnyxToolParserImpl::getCurrentFunctionName() const { + if (this->currentFunctionName.empty()) { + return std::nullopt; + } + return this->currentFunctionName; +} + +Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); + return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); + } + while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { + auto posBegin = toolCallPositions.begin.top(); + auto posEnd = toolCallPositions.end.top(); + // Also consume the leading " " the chat template renders before "to=" (a single + // generate() call only ever produces one such segment, see OnyxReasoningParser's + // class comment for why this can't collide with anything preceding it). + if (posBegin > 0 && outContent[posBegin - 1] == ' ') { + posBegin -= 1; + } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); + outContent.erase(posBegin, posEnd - posBegin); + toolCallPositions.begin.pop(); + toolCallPositions.end.pop(); + } + return StatusCode::OK; +} + +void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> + // + // Mirrors Qwen3CoderToolParser::parse(): drive the same streamParser used for + // streaming with the whole content as a single chunk, and reuse whatever it + // assembled -- unary is a single-shot edge case of streaming, not a parallel + // reimplementation of the tag walk. + auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); + if (!toolCallsOpt.has_value()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Parsing ended, no tool calls found"); + return; + } + parsedOutput.toolCalls = std::move(toolCallsOpt.value()); + for (const auto& toolCall : parsedOutput.toolCalls) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); + } + auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to remove tool calls from content: {}", status.string()); + } +} + +std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { + if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { + // already sent the first delta for the function currently being read + return std::nullopt; + } + int currentToolCallIndex = ++this->toolCallIndex; + rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); + this->returnedFirstDeltas.insert(currentToolCallIndex); + return doc; +} + +std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { + // ASSUMPTION: mirroring Qwen3CoderToolParser, in streaming we only ever complete one + // tool call per parseChunk() call -- there is no way to send multiple tool calls to + // the client in a single streaming delta. + if (toolCalls.size() != 1) { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "For streaming we expected one tool call, got: {}", toolCalls.size()); + throw std::runtime_error("For streaming we expected one tool call"); + } + const auto& toolCall = toolCalls[0]; + this->returnedCompleteDeltas.insert(this->toolCallIndex); + rapidjson::Document argumentsWrapper; + argumentsWrapper.SetObject(); + rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); + argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); + return wrapDelta(argumentsWrapper, this->toolCallIndex); +} + +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // streamParser returns assembled toolCalls once a call closes ("<|eom|>" seen); until + // then, if the function name is already known, send the first delta for it once. + if (newChunk.empty()) { + return std::nullopt; + } + auto toolCallsOpt = this->streamParser.parseChunk(newChunk); + if (toolCallsOpt.has_value()) { + return this->sendFullDelta(toolCallsOpt.value()); + } + auto functionNameOpt = this->streamParser.getCurrentFunctionName(); + if (functionNameOpt.has_value()) { + return this->sendFirstDeltaIfNeeded(functionNameOpt.value()); + } + return std::nullopt; +} +} // namespace ovms + diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp new file mode 100644 index 0000000000..d8d48b6bc1 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -0,0 +1,120 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/status.hpp" + +namespace ovms { + +// Onyx (early preview model) tool-call framing: +// TODO @atobiszei is functions namespace always "functions"? +// <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> +// Unlike qwen3coder/hermes3, Onyx never wraps arguments in a schema-validated, +// per-parameter structure -- the segment between "<|message|>" and "<|eom|>" is +// already the complete, raw JSON arguments blob the caller is expected to forward +// as-is (per the model card: "the SFT tokenizer tokenizes message content ... +// (raw body)"). So no tool-schema-driven type coercion is needed here, unlike +// Qwen3CoderToolParser. + +// Pure state machine that accumulates raw generated text and hands back fully +// assembled tool calls -- mirrors Qwen3CoderToolParserImpl's split between "parse the +// framing" and "turn it into OpenAI delta JSON" (done by the owning OnyxToolParser). +// Because Onyx's arguments are already a complete raw JSON blob (no per-parameter +// schema coercion needed), a tool call is fully known as soon as its end tag is seen -- +// unlike Qwen3Coder there is no incremental per-parameter streaming to do. +struct OnyxToolParserImpl { + enum class State { + Content, // looking for the next "to=functions." recipient tag + InsideName, // accumulating the function name, looking for messageTag + InsideArguments // accumulating the raw JSON arguments blob, looking for endTag + }; + + // Marks the start of a tool-call turn; the function name follows immediately. + static const std::string FUNCTIONS_RECIPIENT_TAG; + // Separates the function name from the raw JSON arguments blob. + static const std::string MESSAGE_TAG; + // Tool calls always end the turn as a continuation (never a full turn end). + static const std::string END_TAG; + + // Return all tool calls fully closed (end tag seen) in the aggregated content so far + // that were not returned before -- nullopt if none completed yet. + std::optional parseChunk(const std::string& chunk); + std::optional getCurrentFunctionName() const; + Status removeToolCallsFromContentIfNeeded(std::string& outContent); + +private: + State currentState = State::Content; + std::string streamContent; // content accumulated from stream chunks + size_t lastProcessedPosition{0}; + std::string currentFunctionName; + struct ToolCallPositions { + std::stack begin; + std::stack end; + }; + ToolCallPositions toolCallPositions; + + // Process streamContent from lastProcessedPosition until a state change happens; + // return true if the state changed (caller should keep looping), false once no more + // progress is possible with the currently available content. + bool parseUntilStateChange(ToolCalls_t& toolCalls); +}; + +class OnyxToolParser : public BaseOutputParser { +private: + // for streaming parsing we need to keep the parser as a member + OnyxToolParserImpl streamParser; + int toolCallIndex{-1}; + std::set returnedFirstDeltas; + std::set returnedCompleteDeltas; + + std::optional sendFirstDeltaIfNeeded(const std::string& functionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + +public: + OnyxToolParser() = delete; + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return OnyxToolParserImpl::END_TAG; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } + +}; +} // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6bcf48bae5..6b9949632d 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -17,8 +17,8 @@ #include #include -#include "../../logging.hpp" -#include "../../stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" #include "output_parser.hpp" #include "parser_config_validation.hpp" #include "llama3/tool_parser.hpp" @@ -37,6 +37,8 @@ #include "gemma4/gemma4_tool_parser.hpp" #include "minicpm5/minicpm5_tool_parser.hpp" #include "minicpm5/minicpm5_reasoning_parser.hpp" +#include "onyx/onyx_tool_parser.hpp" +#include "onyx/onyx_reasoning_parser.hpp" namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { @@ -209,8 +211,13 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); +<<<<<<< HEAD } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); +======= + } else if (toolParserName == "onyx") { + toolParser = std::make_unique(tokenizer); +>>>>>>> 4d9b4248 (Model enablement WIP) } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -222,10 +229,15 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); +<<<<<<< HEAD } else if (reasoningParserName == "minicpm5") { reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); +======= + } else if (reasoningParserName == "onyx") { + reasoningParser = std::make_unique(tokenizer); +>>>>>>> 4d9b4248 (Model enablement WIP) } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index 770993cd1b..f705b21983 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -33,6 +33,7 @@ const std::vector& getSupportedToolParserNames() { "lfm2", "gemma4", "minicpm5", + "onyx", }; return names; } @@ -44,6 +45,7 @@ const std::vector& getSupportedReasoningParserNames() { "gptoss", "minicpm5", "lfm2", + "onyx", }; return names; } diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index 6ecbf7e14b..e4cbe0018b 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -61,6 +61,21 @@ TEST_F(ChatTemplateAnalyzerTest, detectsGptOss) { EXPECT_TRUE(result.caps.supportsToolCalls); } +// --- Onyx --- + +TEST_F(ChatTemplateAnalyzerTest, detectsOnyx) { + std::string tmpl = loadTemplate("chat_template_onyx.jinja"); + ASSERT_FALSE(tmpl.empty()); + auto result = ChatTemplateAnalyzer::analyze(tmpl); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "onyx"); + // Onyx's template never reads the OpenAI "tool_calls" array, so unlike every + // other detected family, supportsToolCalls stays false -- see analyzer.cpp. + EXPECT_FALSE(result.caps.supportsToolCalls); +} + // --- Gemma4 --- TEST_F(ChatTemplateAnalyzerTest, detectsGemma4) { diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp new file mode 100644 index 0000000000..b253081edf --- /dev/null +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -0,0 +1,178 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#pragma GCC diagnostic pop + +#include "src/llm/io_processing/output_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// ============================================================================= +// Genuine request/response round trip for Onyx: render a request with the real +// chat template (minja, via ov::genai::Tokenizer::apply_chat_template -- same +// engine exercised by ChatTemplateEndToEndMinjaTest), hand-author the exact +// continuation the model would emit for that rendered prompt (grounded in what +// ChatTemplateEndToEndMinjaTest's Onyx_* tests already proved the template +// produces), and feed ONLY that continuation into OutputParser -- exactly what +// OVMS does in production (the parser only ever sees newly generated tokens, +// never the prompt). This is the missing link between: +// - chat_template_end_to_end_{minja,jinja}_test.cpp (request side only) +// - output_parsers/onyx_output_parser_test.cpp (response side only, with +// hand-written segments not derived from an actual rendered prompt) +// ============================================================================= +class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { +protected: + // TODO @atobiszei change tokenizer for onyx + const std::string& tokenizerPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m", false); + const std::string& chatTemplatesPath = getGenericFullPathForSrcTest("/ovms/src/test/llm/chat_templates", false); + + static std::string loadTemplateFile(const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) { + return ""; + } + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + } + + // Renders chatHistory with the real Onyx template via minja and asserts the + // generation prompt ends with the bare "<|start|>assistant" the parser tests + // assume (no trailing "<|message|>", no implicit recipient). + std::string renderPrompt(ov::genai::ChatHistory& chatHistory) { + std::string chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + ov::genai::Tokenizer tokenizer(tokenizerPath); + tokenizer.set_chat_template(chatTemplate); + std::string rendered = tokenizer.apply_chat_template(chatHistory, /*add_generation_prompt=*/true); + static const std::string generationPromptTail = "<|start|>assistant"; + EXPECT_TRUE(rendered.size() >= generationPromptTail.size() && + rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) + << "Generation prompt tail changed, Onyx parser assumptions may be stale: " << rendered; + return rendered; + } + + // Simulates generation: appends modelContinuation to the rendered prompt (for + // documentation / sanity only) and runs OutputParser on modelContinuation alone, + // matching what OVMS actually hands the parser. + ParsedOutput parseModelContinuation(const std::string& modelContinuation, bool toolsAvailable = true) { + ov::genai::Tokenizer tokenizer(tokenizerPath); + auto generatedTensor = tokenizer.encode(modelContinuation, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + + static ToolsSchemas_t emptyToolsSchema{}; // Onyx tool parser is not schema-driven, see onyx_tool_parser.hpp + OutputParser outputParser(tokenizer, "onyx", "onyx", emptyToolsSchema); + return outputParser.parse(generatedTokens, toolsAvailable); + } +}; + +// ============================================================================= +// Turn 1 of the muse/README.md "get_weather" example: user asks a question, the +// prompt is rendered, and the model's tool-call continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("You can call get_weather(city)."), std::string::npos) << prompt; + + // Model continuation for a tool call, exactly as documented in muse/README.md. + std::string modelContinuation = R"( to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city": "SF"})"); +} + +// ============================================================================= +// Turn 2 of the same example: tool result fed back into history (Onyx's own +// "name" + role="tool" shape, NOT OpenAI's tool_call_id), then the model's final +// answer continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinalAnswer) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"city\": \"SF\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"functions.get_weather","content":"{\"temp\": 65}"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; + // NOTE (important, non-obvious): OpenVINO GenAI's own minja-path history + // preprocessing -- independent of the raw Jinja template's `elif role == + // 'tool'` branch -- rewrites role="tool" messages into role="user" with a + // generic wrapped "tool_response" JSON object whenever it determines the + // template lacks native tool-call support (the same probe underlying + // caps.supportsToolCalls == false, see chat_template_end_to_end_minja_test.cpp's + // Onyx tests). So Onyx's own `elif role == 'tool'` template branch is + // effectively DEAD CODE on the minja path today -- it never actually fires. + EXPECT_NE(prompt.find(R"(<|start|>user<|message|>{ + "tool_response": { + "tool": "functions.get_weather", + "content": "{\"temp\": 65}" + } +}<|eot|>)"), + std::string::npos) + << prompt; + + std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Private reasoning ("recipient": "self") round trip: the model reasons first, +// which the OnyxReasoningParser must classify as reasoning, not content. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsPrivateReasoning) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's 2+2?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("What's 2+2?"), std::string::npos) << prompt; + + std::string modelContinuation = R"( to=self<|message|>2+2 is a basic addition.<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "2+2 is a basic addition."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 3d284ea300..e8e8515ec5 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -665,3 +665,52 @@ What's the weather in Paris?<|im_end|> )"; EXPECT_EQ(appliedOutput, expectedOutput); } + +// Onyx (early preview model) chat template, rendered via the real Python Jinja2 +// engine. Onyx's template does not read the standard OpenAI "tool_calls" array +// at all -- only message['content'] (plain string) and an Onyx-specific +// message['recipient'] field (e.g. "functions.get_weather", "self", "user"). +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp), +// but deliberately leaves caps.supportsToolCalls false: that flag means "this +// template natively re-serializes an incoming OpenAI tool_calls array", which +// this test demonstrates Onyx's template does NOT do (the tool call is silently +// dropped below). +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // Unlike the minja path (which has its own generic tool-call fallback), + // the real Python Jinja2 engine has no such fallback: the template renders + // message['content'] literally, i.e. the empty string, and the tool call + // information is silently dropped. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|><|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape via Python Jinja2: "recipient" field instead of the +// OpenAI "tool_calls" array. Ends the turn with "<|eom|>" (continuation marker) +// rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index c271788a1a..ec97945e04 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -620,6 +620,145 @@ TEST_F(ChatTemplateEndToEndMinjaTest, MiniCPM5_ToolCallWithStringArgsExpectedToF EXPECT_TRUE(caps.missnamedReasoningField.empty()); } +// ============================================================================= +// Onyx (early preview model) chat template. Unlike every other template in this +// suite, Onyx's own Jinja template does not consume the standard OpenAI +// "tool_calls" list at all -- it only reads message['content'] (a plain string) +// and an Onyx-specific message['recipient'] field (e.g. "functions.get_weather", +// "self", "user"). Feeding it a standard tool_calls-shaped assistant message +// therefore renders an effectively empty assistant turn. +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp) and +// sets detectedToolParser/detectedReasoningParser, but deliberately leaves +// caps.supportsToolCalls false since the template can't natively round-trip an +// OpenAI tool_calls history (demonstrated by this very test), so no input-side +// workaround is applied either -- detectedToolParser only affects output parsing. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // The template itself never reads "tool_calls" (it only looks at + // message['content'] and message['recipient']). Because caps.supportsToolCalls + // is false here, OVMS does not apply its own tool-call workaround either. + // Minja's own generic fallback (used for templates it detects have no native + // tool-call rendering) kicks in instead and serializes the whole message + // (tool_calls + content) as a JSON blob into message['content'] -- the + // function name/args are NOT lost, but they end up as raw, unparsed JSON text + // rather than in Onyx's native " to=functions." / <|eom|> framing. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|>{ + "tool_calls": [ + { + "name": "get_weather", + "arguments": { + "location": "Paris", + "unit": "celsius" + }, + "id": "call_abc123" + } + ], + "content": "" +}<|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape: instead of the OpenAI "tool_calls" array, the +// assistant turn carries a "recipient" field (here "functions.get_weather") +// and a plain-string content holding the raw JSON arguments. This is the shape +// Onyx's template actually understands, ending the turn with "<|eom|>" (a +// continuation marker) rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Full-scope round trip: exercises every message shape the Onyx template +// natively understands in a single history, not just one shape in isolation -- +// user prompt -> assistant tool call (recipient=functions., continuation +// "<|eom|>") -> tool call response (role="tool") -> assistant final answer +// (recipient="user", "<|eot|>"). +// +// This surfaces a second, previously undocumented gap alongside the tool_calls +// one above (see muse/chat_template_issues.md): because caps.supportsToolCalls +// is false for Onyx, ChatTemplateAdapter's generic fallback also intercepts +// plain role="tool" messages -- not just assistant tool_calls -- and rewrites +// them into a synthetic role="user" message serializing {tool, content} as a +// JSON blob, rather than passing them through to the template's own native +// "tool"-role branch (which expects message['name'] + message['content'] and +// would render "<|start|>tool <|message|>...<|eot|>"). So even a chat +// history built entirely out of Onyx's own native fields (recipient) still +// does not round-trip once a plain OpenAI-shaped tool response message is +// mixed in. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"get_weather","content":"{\"temperature\":15,\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"user","content":"It's 15C in Paris."})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + EXPECT_FALSE(caps.supportsToolCalls); + + // Known gap (see class comment above): the "tool" message is NOT rendered via + // the template's native "<|start|>tool <|message|>...<|eot|>" branch -- + // ChatTemplateAdapter's fallback rewrites it into a synthetic user message + // carrying a JSON blob first. + std::string expectedOutput = + R"(<|start|>system<|message|>You are a helpful assistant.<|eot|>)" + R"(<|start|>user<|message|>What's the weather in Paris?<|eot|>)" + R"(<|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|>)" + "<|start|>user<|message|>{\n" + " \"tool_response\": {\n" + " \"tool\": \"get_weather\",\n" + " \"content\": \"{\\\"temperature\\\":15,\\\"unit\\\":\\\"celsius\\\"}\"\n" + " }\n" + "}<|eot|>" + R"(<|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|>)" + R"(<|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. diff --git a/src/test/llm/chat_templates/chat_template_onyx.jinja b/src/test/llm/chat_templates/chat_template_onyx.jinja new file mode 100644 index 0000000000..0fa7b9d204 --- /dev/null +++ b/src/test/llm/chat_templates/chat_template_onyx.jinja @@ -0,0 +1,41 @@ +{{- bos_token -}} +{%- macro render_parts(content) -%} +{%- if content is string -%}{{- content -}} +{%- else -%} +{%- for part in content -%} +{%- if part['type'] == 'image' -%}{{- '<|image|>' -}} +{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}} +{%- elif part['type'] == 'text' -%}{{- part['text'] -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%}{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}{%- endfor -%} +{%- if add_generation_prompt and not ns.has_system -%} +{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}} +{%- endif -%} +{%- for message in messages -%} +{%- set role = message['role'] -%} +{%- if role == 'assistant' -%} +{%- set recipient = message.get('recipient') -%} +{%- set end_turn = message.get('end_turn') -%} +{%- if end_turn is none -%} +{%- set end_turn = not (recipient and recipient != 'user') -%} +{%- endif -%} +{{- '<|start|>assistant' -}} +{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%} +{{- '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- ('<|eot|>' if end_turn else '<|eom|>') -}} +{%- elif role == 'tool' -%} +{%- set name = message.get('name', '') -%} +{{- '<|start|>tool ' + name + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- else -%} +{%- set header = role -%} +{%- if message.get('name') -%}{%- set header = role + ' ' + message['name'] -%}{%- endif -%} +{{- '<|start|>' + header + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%} diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index 2b1840f0de..43ae0a6e1b 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -99,3 +99,23 @@ TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { ASSERT_TRUE(result[0]["content"].is_array()); EXPECT_EQ(result[0]["content"].size(), 2u); } + +TEST(TextContentNormalizationProcessorTest, NullContentNormalizedToEmptyString) { + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls": content is explicitly null (not just absent). Some templates + // (e.g. Onyx's) unconditionally render content for every message and error out + // on null, so this must be normalized to "" the same way a missing field is. + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("assistant")}}; + msg["content"] = ov::genai::JsonContainer(nullptr); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + TextContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + ASSERT_TRUE(result[0]["content"].is_string()); + EXPECT_EQ(result[0]["content"].as_string().value_or("__unset__"), ""); +} diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp new file mode 100644 index 0000000000..4de64ee3f0 --- /dev/null +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -0,0 +1,619 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" +#include "src/llm/io_processing/output_parser.hpp" +#include "src/logging.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// Onyx does not ship a converted HF tokenizer in this early preview, and none of the +// segments the parser looks for ("to=functions.", "<|message|>", "<|eom|>", "<|eot|>", +// "to=self") are real special tokens of the model this parser is designed for -- they are +// plain text sequences that must round-trip losslessly through encode()+decode() on ANY +// tokenizer. facebook/opt-125m is already used the same way for chat-template testing +// (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid pulling in a new +// model fixture. +// TODO @atobiszei replace when tokenizer is available +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/facebook/opt-125m"; +#endif + +static std::unique_ptr opt125mTokenizer; + +// Onyx never consults tool schemas (arguments are forwarded as a raw JSON blob verbatim, +// no per-parameter type coercion like Qwen3CoderToolParser) -- these mirror +// Qwen3CoderOutputParserTest's toolSchemasInput/toolsSchemas setup purely so the test +// fixture shape matches, and so a real schema is on hand if that ever changes. +static std::map toolSchemasInput = { + {"get_weather", R"({"properties":{"location":{"type":"string","description":"City name."},"unit":{"type":"string","description":"Temperature unit."}},"required":["location"]})"}, + {"get_time", R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"}, + {"get_current_location", R"({"properties":{},"required":[]})"}, + {"string_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."}},"required":["arg1"]})"}, + {"cd", R"({"properties":{"folder":{"type":"string","description":"Path"}},"required":["folder"]})"}, + {"string_int_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."},"arg2":{"type":"integer","description":"An integer argument."}},"required":["arg1","arg2"]})"}}; + +static std::vector> schemaDocsStorage; + +static ToolsSchemas_t convertStringToolSchemasStringToToolsSchemas( + const std::map& input) { + ToolsSchemas_t result; + schemaDocsStorage.clear(); + for (const auto& [name, schemaStr] : input) { + auto schemaDoc = std::make_unique(); + if (schemaDoc->Parse(schemaStr.c_str()).HasParseError()) { + throw std::runtime_error("Failed to parse schema for tool: " + name); + } + result[name] = {schemaDoc.get(), schemaStr}; + schemaDocsStorage.push_back(std::move(schemaDoc)); + } + return result; +} + +static ovms::ToolsSchemas_t toolsSchemas = convertStringToolSchemasStringToToolsSchemas(toolSchemasInput); + +class OnyxOutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParser; + + static void SetUpTestSuite() { + try { + opt125mTokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize opt-125m tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize opt-125m tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + opt125mTokenizer.reset(); + } + + void SetUp() override { + outputParser = std::make_unique(*opt125mTokenizer, "onyx", "onyx", toolsSchemas); + } + + ParsedOutput generateParsedOutput(const std::string& input, bool toolsAvailable = true) { + auto generatedTensor = opt125mTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + return outputParser->parse(generatedTokens, toolsAvailable); + } + + // Shared by streaming tests: compares a parseChunk() delta against the expected JSON, + // masking the randomly generated tool call id (kept as one helper rather than the same + // id-masking block duplicated per test, mirroring Qwen3CoderOutputParserTest usage). + static void assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk); + + // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, + // using rapidjson's serializer to handle all escaping. This lets us write raw PLC/Python + // code in tests without manually counting backslashes. + static std::string wrapRawCodeAsToolArgs(const std::string& rawCode) { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Value val(rawCode.c_str(), static_cast(rawCode.size()), doc.GetAllocator()); + doc.AddMember("arg1", val, doc.GetAllocator()); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + doc.Accept(writer); + return buf.GetString(); + } + + // Builds the expected arguments delta JSON string for a given tool call index and + // raw (unescaped) code content: {"delta":{"tool_calls":[{"index":N,"function":{"arguments":"..."}}]}} + static std::string expectedArgsDelta(int index, const std::string& rawCode) { + std::string argsJson = wrapRawCodeAsToolArgs(rawCode); + // argsJson is a valid JSON object string -- we need to embed it as an escaped + // string value inside the outer delta JSON. Use rapidjson to build the whole thing. + rapidjson::Document outer; + outer.SetObject(); + rapidjson::Document::AllocatorType& alloc = outer.GetAllocator(); + rapidjson::Value delta(rapidjson::kObjectType); + rapidjson::Value toolCalls(rapidjson::kArrayType); + rapidjson::Value tc(rapidjson::kObjectType); + tc.AddMember("index", index, alloc); + rapidjson::Value func(rapidjson::kObjectType); + rapidjson::Value argsVal(argsJson.c_str(), static_cast(argsJson.size()), alloc); + func.AddMember("arguments", argsVal, alloc); + tc.AddMember("function", func, alloc); + toolCalls.PushBack(tc, alloc); + delta.AddMember("tool_calls", toolCalls, alloc); + outer.AddMember("delta", delta, alloc); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + outer.Accept(writer); + return buf.GetString(); + } +}; + +// A single generate() call stops at the first "<|eom|>"/"<|eot|>" (both are configured as +// eos tokens for Onyx), so only one of these three segment shapes is ever produced at a time. + +TEST_F(OnyxOutputParserTest, FinalAnswerWithExplicitUserRecipient) { + ParsedOutput parsedOutput = generateParsedOutput(" to=user<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, FinalAnswerWithoutRecipient) { + // The template only renders " to=" when the caller supplies a recipient; the model may + // also just emit "<|message|>...<|eot|>" directly with no recipient at all. + ParsedOutput parsedOutput = generateParsedOutput("<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { + ParsedOutput parsedOutput = generateParsedOutput(" to=self<|message|>Let me think about this.<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "Let me think about this."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithRawJsonArguments) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + // Onyx passes arguments through verbatim -- no schema-driven reformatting like qwen3coder. + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { + // OutputParser::parse() only invokes the tool parser when toolsAvailable is true. + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>", /*toolsAvailable=*/false); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); + // Known current limitation: the reasoning parser (which always runs) intentionally + // leaves "to=functions." segments untouched so the tool parser can claim them -- but + // if the tool parser never runs, the raw wrapped segment is surfaced as-is in content. + EXPECT_EQ(parsedOutput.content, " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"); +} + +TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { + ParsedOutput parsedOutput = generateParsedOutput("some unexpected raw text without any framing"); + + EXPECT_EQ(parsedOutput.content, "some unexpected raw text without any framing"); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Streaming is implemented on top of OnyxToolParserImpl, a pure state machine that +// accumulates raw text and hands back a fully assembled tool call once its "<|eom|>" +// end tag is seen (mirroring Qwen3CoderToolParserImpl) -- unary parse() below drives +// that same impl with the whole content as a single chunk, so it is the single-shot +// degenerate case of streaming, not a parallel implementation of the tag walk. +// +// Because Onyx's arguments are already a complete raw JSON blob needing no per-parameter +// schema coercion, they are still sent to the client as a single delta once the tool call +// closes (matching Qwen3CoderToolParser's sendFullDelta) rather than streamed incrementally +// as raw text arrives -- only the function name streams as its own delta once known. +// +// Chunk boundaries below are deliberately awkward (splitting the function name and the +// JSON arguments mid-token) to exercise the Content/InsideName/InsideArguments state +// machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// ============================================================================= +void OnyxOutputParserTest::assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + if (!expectedDelta.has_value()) { + EXPECT_FALSE(doc.has_value()) << "Expected nullopt for chunk: " << chunk; + return; + } + ASSERT_TRUE(doc.has_value()) << "Expected a delta for chunk: " << chunk; + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + docStr.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expected.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + } + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; +} + +TEST_F(OnyxOutputParserTest, StreamingSimpleToolCall) { + // Mirrors Qwen3CoderOutputParserTest.StreamingSimpleToolCall's rigor: adversarial + // chunk boundaries, content before/between tool calls, complex argument values + // (PLC structured text, Python with triple-quotes/f-strings/escape sequences), + // adapted to Onyx's "to=functions.<|message|><|eom|>" format. + // + // Unlike qwen3coder there is no per-parameter incremental streaming to test + // (qwen3coder's tags) since Onyx's arguments are a single raw JSON blob. + // However, content before/between tool calls IS tested because the OutputParser + // streaming framework handles that generically (UNKNOWN -> CONTENT transition when no + // start tag is found). + // + // Key structural differences from qwen3coder: + // - Start tag is "to=functions." (not "") + // - Name delimiter is "<|message|>" (not ">") + // - End tag is "<|eom|>" (not "") + // - Arguments are a single raw JSON blob (not per-parameter XML tags) + // - PLC/Python code must be JSON-escaped within the arguments blob + + // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). + // Written as a raw string literal so it's human-readable; wrapRawCodeAsToolArgs() + // handles all the JSON escaping at runtime. + const std::string plcCode = R"(FUNCTION FC_CreateJsonPayload : STRING +VAR_INPUT + Value1 : REAL; + Value2 : INT; + Value3 : BOOL; + Value4 : STRING(100); +END_VAR +VAR_OUTPUT + JsonPayload : STRING(1000); +END_VAR +VAR + TempStr : STRING(100); +END_VAR + + JsonPayload := '{'; + JsonPayload := JsonPayload + '"value1":' + REAL_TO_STRING(Value1, '', 2) + ','; + JsonPayload := JsonPayload + '"value2":' + INT_TO_STRING(Value2) + ','; + JsonPayload := JsonPayload + '"value3":' + BOOL_TO_STRING(Value3) + ','; + JsonPayload := JsonPayload + '"value4":"' + Value4 + '"'; + JsonPayload := JsonPayload + '}'; + +END_FUNCTION)"; + + // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). + const std::string pythonCode = R"( +if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r'))"; + + int i = -1; + std::vector>> chunkToDeltaVec{ + // Content before any tool call -- OutputParser sees no start tag match, emits content. + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + // Start tag "to=functions." split across several arbitrarily small chunks. + // Note: leading space before "to=" is just normal content/separator; the start + // tag the framework looks for is "to=functions." without the space. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=fun", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ctions.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Function name streams in across several small chunks -- still no delta + // (OnyxToolParserImpl is in InsideName state, accumulating). + {"get", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"_", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"weath", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"er", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|message|>" itself split mid-tag -- name delta emitted once the full tag lands. + {"<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Raw JSON argument text (with a nested object) split at awkward byte boundaries. + {"{\"locat", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ion\":\"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ris\",\"opt", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ions\":{\"unit", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\":\"cel", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sius\"}}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|eom|>" split mid-tag -- closes the tool call once complete. + {"<|e", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"om|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"options\":{\"unit\":\"celsius\"}}"}}]}})"}, + // Content between tool calls (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). + // In TOOL_CALLS_WAITING_FOR_TOOL phase, text without start tag match waits for more. + {"POTENTIALLY EXISINT CONTENT", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Second tool call -- start tag + name + <|message|> split across tiny chunks. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=functi", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ons.str", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ing_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|messa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ge|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, + // Arguments split across chunks. + {"{\"arg1\":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"NG_VALUE\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eo", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"m|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, + // More content between tool calls (mirrors "CONTENT_AFTER_TOOL_CALL"). + {"CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Third tool call -- string_int_tool with two parameters in JSON (integer stays + // numeric). Start tag + name + <|message|> split differently from previous calls. + {" to=func", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"tions.strin", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g_int_tool<|", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, + // Arguments with a leading \n in the string value (matches qwen3coder's + // "\nANOTHER_STRING_VALUE" pattern) and an integer parameter. + {"{\"arg1\":\"\\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_STRING_VALUE\",\"ar", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g2\":314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"1522}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, + // "NOTHING IMPORTANT HERE" content between calls (mirrors qwen3coder). + {"NOTHING IMPORTANT HERE", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // A "bfcl draft" style call -- cd tool. Start tag arrives with some preceding + // text just like qwen3coder's "part of bfcl 'draft'.\n\n\n" pattern. + {"part of bfcl 'draft'.\n\n to=functions.cd<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, + {"{\"folder\":\"ResearchDocs\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, + // PLC structured text code as a tool argument (mirrors qwen3coder's + // FC_CreateJsonPayload test). Raw code is defined above as plcCode; the helper + // wrapRawCodeAsToolArgs() handles all JSON escaping via rapidjson so we don't + // need to manually count backslashes. Sent as a single chunk since the interesting + // escaping complexity is in the content, not in chunk-boundary splitting. + {" to=functions.string_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(plcCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, + // Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). Also sent as a single chunk -- the chunk-boundary + // adversarial testing is covered by the earlier tool calls above. + {" to=functions.string_tool<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(pythonCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk[" << i << "]: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk[" << i << "]: " << chunk; + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "] (ignoring id value): " << chunk; + } else { + SPDLOG_ERROR("Expected:\n{}", expected); + SPDLOG_ERROR("Got:\n{}", docStr); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + // Validate that arguments fields are valid JSON + if (expected.find("arguments") != std::string::npos) { + auto docJsonIt = doc->FindMember("delta"); + ASSERT_NE(docJsonIt, doc->MemberEnd()); + auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); + ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); + for (const auto& toolCall : toolCallsIt->value.GetArray()) { + auto functionIt = toolCall.FindMember("function"); + ASSERT_NE(functionIt, toolCall.MemberEnd()); + auto argumentsIt = functionIt->value.FindMember("arguments"); + ASSERT_NE(argumentsIt, functionIt->value.MemberEnd()); + const std::string& argumentsStr = argumentsIt->value.GetString(); + rapidjson::Document argsDoc; + argsDoc.Parse(argumentsStr.c_str()); + EXPECT_FALSE(argsDoc.HasParseError()) << "Arguments is not valid JSON for chunk[" << i << "]: " << chunk << "\nArguments string:\n" + << argumentsStr; + } + } + } + } else { + EXPECT_TRUE(false) << "Mismatch between expectedDelta and doc for id: " << i << " chunk:\n" + << chunk + << "\nexpectedDelta:\n" + << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") + << "\nGot doc:\n" + << (doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "NO_DOC"); + FAIL() << "Mismatch between expectedDelta and doc for chunk[" << i << "]: " << chunk; + } + } +} + +// ============================================================================= +// Proves the "unary is an edge case of streaming" property holds structurally, not +// just by coincidence: OnyxToolParser::parse() literally drives the same +// OnyxToolParserImpl used by parseChunk() (see onyx_tool_parser.cpp), so this is +// really just re-checking that the unary entry point wires into the same state +// machine already covered above. +// ============================================================================= +TEST_F(OnyxOutputParserTest, UnaryToolCallMatchesStreamingReuse) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); +} + +TEST_F(OnyxOutputParserTest, UnaryTwoSequentialToolCalls) { + ParsedOutput parsedOutput = generateParsedOutput( + " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "get_time"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"city\":\"SF\"}"); +} + +// ============================================================================= +// Direct OnyxToolParserImpl unit tests -- mirrors Qwen3CoderOutputParserTest's +// TestJustParserImplUnary*/TestJustParserImplStreamStep* layer (which exercises the +// state machine directly, below OutputParser/OnyxToolParser), previously untested here. +// ============================================================================= +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"location\":\"Paris\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { + const std::string input = "Unexpected void found. Philosophical crisis imminent."; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_FALSE(callsOpt.has_value()); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(content, input); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithTwoToolCalls) { + const std::string input = " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 2) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(calls[1].name, "get_time"); + EXPECT_EQ(calls[1].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithNoStateChange) { + const std::string input = "Some content without tool calls"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithPartialToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); + ASSERT_TRUE(parser.getCurrentFunctionName().has_value()); + EXPECT_EQ(parser.getCurrentFunctionName().value(), "get_weather"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { + const std::string input = " to=functions.get_current_location<|message|>{}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_TRUE(stepResult.has_value()); + auto& calls = stepResult.value(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_current_location"); + EXPECT_EQ(calls[0].arguments, "{}"); +} + +// ============================================================================= +// Qwen3CoderOutputParserTest test cases, for reference/parity comparison (this file +// intentionally does not have a 1:1 test for every one of these -- see inline notes +// on why some don't apply to Onyx's simpler, non-schema-driven, single-JSON-blob +// argument format): +// Parse1ToolCall1Function1ArgumentTagsNewline +// Parse1ToolCall1Function1ArgumentNoProperBeginTag +// Parse1ToolCallNestedXmlNotFromSchema +// ParseTwoToolCalls1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentMultilineValue +// TestJustParserImplUnaryToolCall -- covered above +// TestJustParserImplUnaryWithNoToolCall -- covered above +// TestJustParserImplUnaryWithContent -- N/A: Onyx's grammar never +// has plain content before/after a tool-call tag within the same generated turn +// TestJustParserImplUnaryWithThreeParameters -- N/A: no per-parameter +// schema-driven typing; arguments are always a single opaque JSON blob +// TestJustParserImplUnaryWithEnforcementOfStringParameter -- N/A, same reason +// TestJustParserImplUnaryWithNotPresentToolSchema -- N/A, same reason (Onyx +// never even looks at tool schemas -- see ToolCallWithRawJsonArguments above) +// TestJustParserImplUnaryWithJsonObjectArgument -- covered by nested-object +// case in StreamingSimpleToolCall above +// TestJustParserImplUnaryWithTwoToolCalls -- covered above +// TestJustParserImplUnaryToolCallNoMatchingToolParameterTypeMapEntry -- N/A, same reason +// TestJustParserImplUnaryToolCallWithRepeatedArgument -- N/A, same reason (no +// per-parameter parsing to have a "repeated argument" concept at all) +// TestJustParserImplStreamStepWithMoreThan1StateChange -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above (both calls resolve in one parseChunk) +// TestJustParserImplStreamStepWithNoStateChange -- covered above +// TestJustParserImplStreamStepWithPartialToolCall -- covered above +// TestJustParserImplStreamStepWithTwoToolCalls -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above +// TestJustParserImplStreamStepWithToolCallNoArgs -- covered above +// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes -- N/A: +// parametrized over per-parameter type coercion (string/int/float/bool/object/list), +// which does not exist for Onyx (raw JSON passthrough only) +// StreamingSimpleToolCall -- covered above (adapted; +// see comment on that test for what was intentionally omitted/adjusted) +// ============================================================================= + From 6e4dded85eafec4ba2a50a11978cd78bd078c871 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Tue, 28 Jul 2026 11:01:31 +0200 Subject: [PATCH 11/38] Build files --- Dockerfile.ubuntu | 8 ++++++-- Makefile | 9 ++++++++- versions.mk | 19 ++++++++++++++++--- windows_install_build_dependencies.bat | 6 ++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 59a7fbdf23..b15749002f 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -156,10 +156,14 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" +# onyx-support patches (temporary, one-off - see patches/*/readme.md for the +# commits they apply to) +COPY patches /patches/ + ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ ARG SDL_OPS="-Wl,-z,relro,-z,now,-z,noexecstack -fPIE -pie -fstack-protector-strong -fexceptions -fasynchronous-unwind-tables -fcf-protection -fpic -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fno-strict-overflow -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wuninitialized" # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git apply /patches/openvino/0001-openvino.patch && git apply /patches/openvino/0002-openvino.patch && git apply /patches/openvino/0003-openvino.patch && git apply /patches/openvino/0004-openvino.patch && git apply /patches/openvino/0005-openvino.patch && git submodule update --init --recursive WORKDIR /openvino/build RUN if [ "$ov_use_binary" == "0" ]; then \ if [[ $debug_bazel_flags == *"py_off"* ]]; then \ @@ -228,7 +232,7 @@ ARG ov_genai_org=openvinotoolkit WORKDIR /openvino_genai/ # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then \ - git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive && \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git apply /patches/openvino.genai/0001-openvino.genai.patch && git submodule update --init --recursive && \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} " -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ && \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ && \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ && \ diff --git a/Makefile b/Makefile index ff800383ca..728c022ab6 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,14 @@ endif ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" - OV_USE_BINARY ?= 1 + # ubuntu24 defaults to building OpenVINO/GenAI from source (see versions.mk) + # so that the onyx-support patches in ./patches can be applied. Other ubuntu + # flavors keep using the prebuilt binary package by default. + ifeq ($(BASE_OS),ubuntu24) + OV_USE_BINARY ?= 0 + else + OV_USE_BINARY ?= 1 + endif ifeq ($(findstring ubuntu22,$(BASE_OS)),ubuntu22) ifeq ($(OV_USE_BINARY),0) $(error OV_USE_BINARY = 0 not supported on Ubuntu22 OS) diff --git a/versions.mk b/versions.mk index bae53c3c08..817b4cfb54 100644 --- a/versions.mk +++ b/versions.mk @@ -19,9 +19,22 @@ # Any variable can be overridden by the environment or command-line. # Source repository git commits / branches (used for source builds) -OV_SOURCE_BRANCH ?= 8a17657b995fd3b4a52f8484acfcf2bb61214623 -OV_TOKENIZERS_BRANCH ?= 183c6f25cda2a469cba5eff8b72022d2d51ba0ca -OV_GENAI_BRANCH ?= bd8d6542e3ca1ac30042d5d8d4202ce00b5f4af0 +# NOTE: pinned to the commits required by the onyx-support patches in +# ./patches (see patches/openvino/readme.md and patches/openvino.genai/readme.md). +# This is a temporary, one-off pin - restore the previous commits below once +# the patches are no longer needed: +# OV_SOURCE_BRANCH ?= d08e55c64c37fde1f4f6157cc5f5e07dd36ce5e8 (pre-patch branch tip) +# OV_GENAI_BRANCH ?= 8981d6f848f17985979be0a9224251d181f68c56 (pre-patch branch tip) +# NOTE: OV_TOKENIZERS_BRANCH is intentionally left at its original commit - +# the tokenizers commit referenced by the genai patch's submodule bump +# (935443f5275ce93f362f9eb4fa2d9fa762dd3f22) does not exist in the +# openvinotoolkit/openvino_tokenizers repo (only reachable from a fork used +# during genai development), and OVMS builds tokenizers as a separate +# component (BUILD_TOKENIZERS=OFF in the genai cmake invocation) so this pin +# does not affect the OVMS build. +OV_SOURCE_BRANCH ?= 5b6997da03a7a0713fb4376f9109b4832383cc24 +OV_TOKENIZERS_BRANCH ?= master +OV_GENAI_BRANCH ?= c637ed85efebf1a44d5f0433845849a2d80b353c # Source repository organizations OV_SOURCE_ORG ?= openvinotoolkit diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 3ac79c2920..4ab3f5962c 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -256,6 +256,10 @@ cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin git checkout %OV_SOURCE_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +for %%P in (0001 0002 0003 0004 0005) do ( + git apply "%BACK_CWD%\patches\openvino\%%P-openvino.patch" + if !errorlevel! neq 0 exit /b !errorlevel! +) git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules @@ -304,6 +308,8 @@ cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin git checkout %OV_GENAI_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +git apply "%BACK_CWD%\patches\openvino.genai\0001-openvino.genai.patch" +if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( mkdir build From 7ef6cea8ce50f86cecc9bab5da7a4b3f8ce430f7 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 30 Jul 2026 09:13:24 +0200 Subject: [PATCH 12/38] Experimenting with decode special tokens --- src/llm/io_processing/output_parser.cpp | 9 ++------- src/llm/io_processing/output_parser.hpp | 1 + 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6b9949632d..92e8a4dd7d 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -211,13 +211,10 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); -<<<<<<< HEAD } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); -======= } else if (toolParserName == "onyx") { toolParser = std::make_unique(tokenizer); ->>>>>>> 4d9b4248 (Model enablement WIP) } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -229,15 +226,13 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); -<<<<<<< HEAD } else if (reasoningParserName == "minicpm5") { reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); -======= } else if (reasoningParserName == "onyx") { reasoningParser = std::make_unique(tokenizer); ->>>>>>> 4d9b4248 (Model enablement WIP) + decodeWithSpecialTokens = true; } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); @@ -303,7 +298,7 @@ ParsedOutput OutputParser::parse(const std::vector& generatedTokens, co SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); } ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); + parsedOutput.content = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(!decodeWithSpecialTokens)); if (reasoningParser) { reasoningParser->parse(parsedOutput, generatedTokens); } diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 4e7c467f81..a6d0f7bbc5 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -58,6 +58,7 @@ class OutputParser { ov::genai::Tokenizer tokenizer; std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content + bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) // Streaming related members ProcessingPhase processingPhase = UNKNOWN; From 3ea71a8d55f150c59d7c51e6cc186d71d49233d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Rzepecki?= Date: Mon, 3 Aug 2026 10:28:03 +0100 Subject: [PATCH 13/38] force tokenizer master --- windows_install_build_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 60ae7ef676..1d72d79261 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -280,7 +280,7 @@ IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_tokenizers_src ( ) cd %BAZEL_SHORT_PATH%\openvino_tokenizers_src git fetch origin -git checkout %OV_TOKENIZERS_BRANCH% +git checkout master if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( From daabced5b59291e0743d484ae4f3628dc2ad7a3e Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 12:01:10 +0200 Subject: [PATCH 14/38] review --- versions.mk | 6 +++--- windows_install_build_dependencies.bat | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/versions.mk b/versions.mk index bae53c3c08..62667d1781 100644 --- a/versions.mk +++ b/versions.mk @@ -19,9 +19,9 @@ # Any variable can be overridden by the environment or command-line. # Source repository git commits / branches (used for source builds) -OV_SOURCE_BRANCH ?= 8a17657b995fd3b4a52f8484acfcf2bb61214623 -OV_TOKENIZERS_BRANCH ?= 183c6f25cda2a469cba5eff8b72022d2d51ba0ca -OV_GENAI_BRANCH ?= bd8d6542e3ca1ac30042d5d8d4202ce00b5f4af0 +OV_SOURCE_BRANCH ?= muse_onyx +OV_TOKENIZERS_BRANCH ?= master +OV_GENAI_BRANCH ?= muse_onyx # Source repository organizations OV_SOURCE_ORG ?= openvinotoolkit diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 1d72d79261..c02afd1e86 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -254,7 +254,7 @@ IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( set "BACK_CWD=%cd%" cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin -git checkout muse_onyx +git checkout %OV_SOURCE_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! @@ -264,7 +264,7 @@ IF /I NOT EXIST build ( ) cd build set "TBB_DIR=" -cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF -DPython3_EXECUTABLE=%PYTHONHOME%\python.exe .. +cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. if !errorlevel! neq 0 exit /b !errorlevel! cmake --build . --config Release --verbose -j if !errorlevel! neq 0 exit /b !errorlevel! @@ -280,7 +280,7 @@ IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_tokenizers_src ( ) cd %BAZEL_SHORT_PATH%\openvino_tokenizers_src git fetch origin -git checkout master +git checkout %OV_TOKENIZERS_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( @@ -302,7 +302,7 @@ IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( ) cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin -git checkout muse_onyx +git checkout %OV_GENAI_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( From e8780bf6f87792ff3d6e006f75ab059528481334 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Mon, 27 Jul 2026 16:55:29 +0200 Subject: [PATCH 15/38] Model enablement WIP --- src/BUILD | 2 + src/llm/BUILD | 32 + .../io_processing/chat_template/analyzer.cpp | 23 + .../text_content_normalization_processor.cpp | 13 + .../text_content_normalization_processor.hpp | 14 +- .../onyx/onyx_reasoning_parser.cpp | 102 +++ .../onyx/onyx_reasoning_parser.hpp | 81 +++ .../io_processing/onyx/onyx_tool_parser.cpp | 183 ++++++ .../io_processing/onyx/onyx_tool_parser.hpp | 120 ++++ src/llm/io_processing/output_parser.cpp | 16 +- .../parser_config_validation.cpp | 2 + src/test/llm/chat_template_analyzer_test.cpp | 15 + ...emplate_and_parser_onyx_roundtrip_test.cpp | 178 +++++ .../chat_template_end_to_end_jinja_test.cpp | 49 ++ .../chat_template_end_to_end_minja_test.cpp | 139 ++++ .../chat_templates/chat_template_onyx.jinja | 41 ++ ...t_content_normalization_processor_test.cpp | 20 + .../onyx_output_parser_test.cpp | 619 ++++++++++++++++++ 18 files changed, 1642 insertions(+), 7 deletions(-) create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_reasoning_parser.hpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.cpp create mode 100644 src/llm/io_processing/onyx/onyx_tool_parser.hpp create mode 100644 src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp create mode 100644 src/test/llm/chat_templates/chat_template_onyx.jinja create mode 100644 src/test/llm/output_parsers/onyx_output_parser_test.cpp diff --git a/src/BUILD b/src/BUILD index b0365a1a18..dfa47dc911 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2940,12 +2940,14 @@ cc_library( "test/llm/chat_template_analyzer_test.cpp", "test/llm/chat_template_adapter_test.cpp", "test/llm/chat_template_end_to_end_minja_test.cpp", + "test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp", ], deps = [ "@com_google_googletest//:gtest", "//src/llm:chat_template_analyzer", "//src/llm:chat_template_probe", "//src/llm:io_processing_input_processors", + "//src/llm:output_parsers", "//src/utils:env_guard", "//third_party:genai", ":test_platform_utils", diff --git a/src/llm/BUILD b/src/llm/BUILD index c15c20f08f..358df0e96a 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -403,6 +403,36 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_onyx_tool_parser", + hdrs = ["io_processing/onyx/onyx_tool_parser.hpp"], + srcs = ["io_processing/onyx/onyx_tool_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + "//src:libovmsstatus", + ":io_processing_utils", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "io_processing_onyx_reasoning_parser", + hdrs = ["io_processing/onyx/onyx_reasoning_parser.hpp"], + srcs = ["io_processing/onyx/onyx_reasoning_parser.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_document", + "//src:libovmslogging", + ":io_processing_base_output_parser", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_lfm25_reasoning_parser", hdrs = ["io_processing/lfm2/lfm25_reasoning_parser.hpp"], @@ -458,6 +488,8 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_gemma4_tool_parser", ":io_processing_minicpm5_tool_parser", ":io_processing_qwen3_reasoning_parser", + ":io_processing_onyx_tool_parser", + ":io_processing_onyx_reasoning_parser", ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index a2e8a0f82c..86c0f30ff2 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -37,6 +37,29 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } + // Onyx detection — Harmony-family framing ("<|start|>{role}[ to=]<|message|> + // {content}<|eom|>/<|eot|>") without gpt-oss's "<|channel|>" marker (already handled + // above). This combination of literal tokens is not used by any other template in + // this codebase (verified against every fixture under src/test/llm/chat_templates/). + // There is no separate reasoning-specific token: Onyx routes both tool calls + // (recipient="functions.") and private reasoning (recipient="self") through the + // exact same "<|message|>...<|eom|>" framing, so both parsers are tied together here, + // like gptoss/gemma4. + // NOTE: unlike every other branch below, this deliberately does NOT set + // caps.supportsToolCalls -- that flag means "the template natively re-serializes an + // incoming OpenAI-shaped `tool_calls` array back into the model's own format", which + // Onyx's template does not do at all (it only ever reads message['recipient']/ + // message['content'], never message['tool_calls'] -- see the Onyx_ToolCallWithStringArgs + // tests in chat_template_end_to_end_{jinja,minja}_test.cpp). detectedToolParser/ + // detectedReasoningParser only affect how OVMS parses the model's *output*, which is + // unrelated to (and unaffected by) whether input history round-trips correctly. + if (contains(templateSource, "<|start|>") && contains(templateSource, "<|message|>") && + contains(templateSource, "<|eom|>") && contains(templateSource, "<|eot|>")) { + result.detectedToolParser = "onyx"; + result.detectedReasoningParser = "onyx"; + return result; + } + // Gemma4 detection if (contains(templateSource, "'<|tool_call>call:'") || contains(templateSource, "<|tool_call>call:")) { result.detectedToolParser = "gemma4"; diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 2180d5faae..871a0f5f06 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -29,6 +29,19 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { ov::genai::ChatHistory& chatHistory = std::get(req.input); for (size_t i = 0; i < chatHistory.size(); i++) { const auto content = chatHistory[i]["content"]; + if (content.is_null()) { + // TODO @atobiszei to check if really needed when we have IR + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls" sets "content": null (openai_completions.cpp stores this + // verbatim -- only a *missing* content field is defaulted to ""). Some + // chat templates (e.g. Onyx's) unconditionally render content for every + // message regardless of role/tool_calls and are not written to expect + // null there, which raises a template error instead of just omitting + // the text. Normalize null the same way a missing field is already + // defaulted, so every template sees a plain string as before. + chatHistory[i]["content"] = std::string(""); + continue; + } if (!content.is_array()) { continue; } diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index e68d892490..b6292cb302 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,11 +19,15 @@ namespace ovms { -// Flattens text-only content arrays in ChatHistory messages to plain strings. -// Parts are joined with "\n" for backward compatibility with chat templates. -// Runs for both LM and VLM chat paths: arrays that contain images (or other -// non-text modalities) are left untouched. -// Must run before ChatTemplateProcessor and after Image/Audio decoding processors. +// Flattens text-only content arrays in ChatHistory messages to plain strings, and +// normalizes an explicit "content": null (the standard OpenAI shape for e.g. an +// assistant message that only carries tool_calls) to "" -- some chat templates +// (e.g. Onyx's) unconditionally render content for every message and are not +// written to expect null there. Parts/null are joined/replaced for backward +// compatibility with chat templates. Runs for both LM and VLM chat paths: arrays +// that contain images (or other non-text modalities) are left untouched for +// ImageDecodingProcessor. +// Must run before ChatTemplateProcessor. class TextContentNormalizationProcessor : public BaseInputProcessor { public: absl::Status process(InputRequest& req) override; diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp new file mode 100644 index 0000000000..43b45a1e8f --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -0,0 +1,102 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "onyx_reasoning_parser.hpp" + +namespace ovms { + +void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // TODO @atobiszei overcomplicated I think? We just need t find recipient self & them cut that part out. + // Case 1: private chain-of-thought turn (recipient="self") -> extract reasoning, + // consume the whole segment (nothing meaningful is expected to follow it within the + // same generate() call, see class comment). + size_t selfPos = parsedOutput.content.find(selfRecipientTag); + if (selfPos != std::string::npos) { + size_t messagePos = parsedOutput.content.find(messageTag, selfPos); + if (messagePos != std::string::npos) { + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(continuationEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.reasoning = body; + // Drop the leading " " before "to=" (rendered by the chat template) too, if present. + size_t segmentStart = (selfPos > 0 && parsedOutput.content[selfPos - 1] == ' ') ? selfPos - 1 : selfPos; + parsedOutput.content.erase(segmentStart); + return; + } + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Found '{}' without a following '{}', leaving content untouched", selfRecipientTag, messageTag); + return; + } + + // Case 2: tool-call turn (recipient="functions.") -> leave untouched, OnyxToolParser + // (which runs after this parser, see OutputParser::parse()) is responsible for it. + if (parsedOutput.content.find(functionsRecipientTag) != std::string::npos) { + return; + } + + // Case 3: plain final answer (recipient="user" or absent) -> strip the generic + // " to=user"? + "<|message|>" + "<|eot|>" envelope, leaving just the clean text. + size_t messagePos = parsedOutput.content.find(messageTag); + if (messagePos == std::string::npos) { + // No framing found at all -- unexpected/malformed output, leave content as-is. + return; + } + size_t bodyStart = messagePos + messageTag.length(); + size_t endPos = parsedOutput.content.find(turnFinalEndTag, bodyStart); + std::string body = (endPos != std::string::npos) + ? parsedOutput.content.substr(bodyStart, endPos - bodyStart) + : parsedOutput.content.substr(bodyStart); + parsedOutput.content = body; +} + +std::optional OnyxReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // TODO @atobiszei we need to stream between recipient=self & + // TODO: streaming support is a first draft. It only forwards the chunk as + // reasoning_content once we've seen the "to=self" start tag; stripping the + // generic final-answer envelope (Case 3 above) in streaming mode is not + // implemented yet and needs its own design (the envelope prefix/suffix can + // straddle multiple chunks). + if (chunk.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for OnyxReasoningParser"); + return std::nullopt; + } + if (chunk.find(selfRecipientTag) != std::string::npos || + chunk.find(messageTag) != std::string::npos || + chunk.find(continuationEndTag) != std::string::npos) { + return std::nullopt; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; +} +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp new file mode 100644 index 0000000000..27218d8414 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -0,0 +1,81 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" + +namespace ovms { + +// Onyx (early preview model) framing: +// TODO @atobiszei simplify comment. tag naming convention. no need to define all tags here +// <|start|>assistant[ to=]<|message|>{content}{<|eom|>|<|eot|>} +// The chat template never emits a ""-style dedicated reasoning tag: private +// chain-of-thought is just an assistant turn routed with recipient="self", ending in +// the continuation marker "<|eom|>" (never "<|eot|>", which is reserved for turns that +// end the whole assistant turn -- i.e. the final answer). +// +// Because generation stops at the first "<|eom|>"/"<|eot|>"/"<|end_of_text|>" (see +// generation_config.json's eos_token_id list in the Onyx HF conversion script), a single +// generate() call only ever produces ONE such framed segment. This parser is therefore +// also responsible for stripping the generic " to="+"<|message|>"+terminator +// envelope from plain final-answer turns (recipient="user" or absent) -- this class runs +// before the tool parser (see OutputParser::parse()), so it must NOT touch content when +// the envelope routes to a function call (recipient="functions."); it leaves that +// segment untouched so OnyxToolParser can find and parse it afterwards. +class OnyxReasoningParser : public BaseOutputParser { +protected: + // Marks a private chain-of-thought turn (recipient="self"). + const std::string selfRecipientTag = "to=self"; + // Marks a tool-call turn (recipient="functions.") -- left untouched here. + const std::string functionsRecipientTag = "to=functions."; + // Separates the routing prefix from the turn's body. + const std::string messageTag = "<|message|>"; + // Terminator for continuation turns (reasoning and tool calls). + const std::string continuationEndTag = "<|eom|>"; + // Terminator for turn-final turns (plain final answers). + const std::string turnFinalEndTag = "<|eot|>"; + +public: + OnyxReasoningParser() = delete; + explicit OnyxReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{selfRecipientTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return continuationEndTag; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp new file mode 100644 index 0000000000..76b6a65e49 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -0,0 +1,183 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "src/llm/io_processing/utils.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" + +namespace ovms { + +const std::string OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG = "to=functions."; +const std::string OnyxToolParserImpl::MESSAGE_TAG = "<|message|>"; +const std::string OnyxToolParserImpl::END_TAG = "<|eom|>"; + +#define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ + auto pos = this->streamContent.find(TAG, this->lastProcessedPosition); \ + if (pos == std::string::npos) { \ + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Did not find: {}", TAG); \ + break; \ + } + +bool OnyxToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + auto previousState = this->currentState; + switch (this->currentState) { + case State::Content: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(FUNCTIONS_RECIPIENT_TAG); + this->toolCallPositions.begin.push(pos); + this->lastProcessedPosition = pos + FUNCTIONS_RECIPIENT_TAG.length(); + this->currentState = State::InsideName; + break; + } + case State::InsideName: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(MESSAGE_TAG); + this->currentFunctionName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + MESSAGE_TAG.length(); + this->currentState = State::InsideArguments; + break; + } + case State::InsideArguments: { + DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(END_TAG); + std::string argumentsPart = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + END_TAG.length(); + this->currentState = State::Content; + this->toolCallPositions.end.push(this->lastProcessedPosition); + ToolCall toolCall{generateRandomId(), this->currentFunctionName, argumentsPart}; + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Adding tool call: id={}, name={}, arguments={}", toolCall.id, toolCall.name, toolCall.arguments); + toolCalls.emplace_back(std::move(toolCall)); + this->currentFunctionName.clear(); + break; + } + } + return previousState != this->currentState; +} + +std::optional OnyxToolParserImpl::parseChunk(const std::string& chunk) { + if (chunk.empty()) { + return std::nullopt; + } + ToolCalls_t toolCalls; + this->streamContent += chunk; + while (parseUntilStateChange(toolCalls)) { + } + if (!toolCalls.empty()) { + return std::move(toolCalls); + } + return std::nullopt; +} + +std::optional OnyxToolParserImpl::getCurrentFunctionName() const { + if (this->currentFunctionName.empty()) { + return std::nullopt; + } + return this->currentFunctionName; +} + +Status OnyxToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); + return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); + } + while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { + auto posBegin = toolCallPositions.begin.top(); + auto posEnd = toolCallPositions.end.top(); + // Also consume the leading " " the chat template renders before "to=" (a single + // generate() call only ever produces one such segment, see OnyxReasoningParser's + // class comment for why this can't collide with anything preceding it). + if (posBegin > 0 && outContent[posBegin - 1] == ' ') { + posBegin -= 1; + } + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Removing tool call from outContent begin:{}, end:{}, removing:{}", posBegin, posEnd, outContent.substr(posBegin, posEnd - posBegin)); + outContent.erase(posBegin, posEnd - posBegin); + toolCallPositions.begin.pop(); + toolCallPositions.end.pop(); + } + return StatusCode::OK; +} + +void OnyxToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + // <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> + // + // Mirrors Qwen3CoderToolParser::parse(): drive the same streamParser used for + // streaming with the whole content as a single chunk, and reuse whatever it + // assembled -- unary is a single-shot edge case of streaming, not a parallel + // reimplementation of the tag walk. + auto toolCallsOpt = this->streamParser.parseChunk(parsedOutput.content); + if (!toolCallsOpt.has_value()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Parsing ended, no tool calls found"); + return; + } + parsedOutput.toolCalls = std::move(toolCallsOpt.value()); + for (const auto& toolCall : parsedOutput.toolCalls) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Unary | Onyx Tool | id: [{}], name: [{}], arguments: [{}]", toolCall.id, toolCall.name, toolCall.arguments); + } + auto status = this->streamParser.removeToolCallsFromContentIfNeeded(parsedOutput.content); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Failed to remove tool calls from content: {}", status.string()); + } +} + +std::optional OnyxToolParser::sendFirstDeltaIfNeeded(const std::string& functionName) { + if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { + // already sent the first delta for the function currently being read + return std::nullopt; + } + int currentToolCallIndex = ++this->toolCallIndex; + rapidjson::Document doc = wrapFirstDelta(functionName, currentToolCallIndex); + this->returnedFirstDeltas.insert(currentToolCallIndex); + return doc; +} + +std::optional OnyxToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { + // ASSUMPTION: mirroring Qwen3CoderToolParser, in streaming we only ever complete one + // tool call per parseChunk() call -- there is no way to send multiple tool calls to + // the client in a single streaming delta. + if (toolCalls.size() != 1) { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "For streaming we expected one tool call, got: {}", toolCalls.size()); + throw std::runtime_error("For streaming we expected one tool call"); + } + const auto& toolCall = toolCalls[0]; + this->returnedCompleteDeltas.insert(this->toolCallIndex); + rapidjson::Document argumentsWrapper; + argumentsWrapper.SetObject(); + rapidjson::Value argumentsValue(toolCall.arguments.c_str(), static_cast(toolCall.arguments.size()), argumentsWrapper.GetAllocator()); + argumentsWrapper.AddMember("arguments", argumentsValue, argumentsWrapper.GetAllocator()); + return wrapDelta(argumentsWrapper, this->toolCallIndex); +} + +std::optional OnyxToolParser::parseChunk(const std::string& newChunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason /*finishReason*/) { + // streamParser returns assembled toolCalls once a call closes ("<|eom|>" seen); until + // then, if the function name is already known, send the first delta for it once. + if (newChunk.empty()) { + return std::nullopt; + } + auto toolCallsOpt = this->streamParser.parseChunk(newChunk); + if (toolCallsOpt.has_value()) { + return this->sendFullDelta(toolCallsOpt.value()); + } + auto functionNameOpt = this->streamParser.getCurrentFunctionName(); + if (functionNameOpt.has_value()) { + return this->sendFirstDeltaIfNeeded(functionNameOpt.value()); + } + return std::nullopt; +} +} // namespace ovms + diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp new file mode 100644 index 0000000000..d8d48b6bc1 --- /dev/null +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -0,0 +1,120 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/status.hpp" + +namespace ovms { + +// Onyx (early preview model) tool-call framing: +// TODO @atobiszei is functions namespace always "functions"? +// <|start|>assistant to=functions.<|message|>{raw JSON args}<|eom|> +// Unlike qwen3coder/hermes3, Onyx never wraps arguments in a schema-validated, +// per-parameter structure -- the segment between "<|message|>" and "<|eom|>" is +// already the complete, raw JSON arguments blob the caller is expected to forward +// as-is (per the model card: "the SFT tokenizer tokenizes message content ... +// (raw body)"). So no tool-schema-driven type coercion is needed here, unlike +// Qwen3CoderToolParser. + +// Pure state machine that accumulates raw generated text and hands back fully +// assembled tool calls -- mirrors Qwen3CoderToolParserImpl's split between "parse the +// framing" and "turn it into OpenAI delta JSON" (done by the owning OnyxToolParser). +// Because Onyx's arguments are already a complete raw JSON blob (no per-parameter +// schema coercion needed), a tool call is fully known as soon as its end tag is seen -- +// unlike Qwen3Coder there is no incremental per-parameter streaming to do. +struct OnyxToolParserImpl { + enum class State { + Content, // looking for the next "to=functions." recipient tag + InsideName, // accumulating the function name, looking for messageTag + InsideArguments // accumulating the raw JSON arguments blob, looking for endTag + }; + + // Marks the start of a tool-call turn; the function name follows immediately. + static const std::string FUNCTIONS_RECIPIENT_TAG; + // Separates the function name from the raw JSON arguments blob. + static const std::string MESSAGE_TAG; + // Tool calls always end the turn as a continuation (never a full turn end). + static const std::string END_TAG; + + // Return all tool calls fully closed (end tag seen) in the aggregated content so far + // that were not returned before -- nullopt if none completed yet. + std::optional parseChunk(const std::string& chunk); + std::optional getCurrentFunctionName() const; + Status removeToolCallsFromContentIfNeeded(std::string& outContent); + +private: + State currentState = State::Content; + std::string streamContent; // content accumulated from stream chunks + size_t lastProcessedPosition{0}; + std::string currentFunctionName; + struct ToolCallPositions { + std::stack begin; + std::stack end; + }; + ToolCallPositions toolCallPositions; + + // Process streamContent from lastProcessedPosition until a state change happens; + // return true if the state changed (caller should keep looping), false once no more + // progress is possible with the currently available content. + bool parseUntilStateChange(ToolCalls_t& toolCalls); +}; + +class OnyxToolParser : public BaseOutputParser { +private: + // for streaming parsing we need to keep the parser as a member + OnyxToolParserImpl streamParser; + int toolCallIndex{-1}; + std::set returnedFirstDeltas; + std::set returnedCompleteDeltas; + + std::optional sendFirstDeltaIfNeeded(const std::string& functionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + +public: + OnyxToolParser() = delete; + explicit OnyxToolParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{OnyxToolParserImpl::FUNCTIONS_RECIPIENT_TAG}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return OnyxToolParserImpl::END_TAG; + } + bool requiresStreamingWithSpecialTokens() const override { + return true; + } + +}; +} // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6bcf48bae5..6b9949632d 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -17,8 +17,8 @@ #include #include -#include "../../logging.hpp" -#include "../../stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" #include "output_parser.hpp" #include "parser_config_validation.hpp" #include "llama3/tool_parser.hpp" @@ -37,6 +37,8 @@ #include "gemma4/gemma4_tool_parser.hpp" #include "minicpm5/minicpm5_tool_parser.hpp" #include "minicpm5/minicpm5_reasoning_parser.hpp" +#include "onyx/onyx_tool_parser.hpp" +#include "onyx/onyx_reasoning_parser.hpp" namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { @@ -209,8 +211,13 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); +<<<<<<< HEAD } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); +======= + } else if (toolParserName == "onyx") { + toolParser = std::make_unique(tokenizer); +>>>>>>> 4d9b4248 (Model enablement WIP) } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -222,10 +229,15 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); +<<<<<<< HEAD } else if (reasoningParserName == "minicpm5") { reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); +======= + } else if (reasoningParserName == "onyx") { + reasoningParser = std::make_unique(tokenizer); +>>>>>>> 4d9b4248 (Model enablement WIP) } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index 770993cd1b..f705b21983 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -33,6 +33,7 @@ const std::vector& getSupportedToolParserNames() { "lfm2", "gemma4", "minicpm5", + "onyx", }; return names; } @@ -44,6 +45,7 @@ const std::vector& getSupportedReasoningParserNames() { "gptoss", "minicpm5", "lfm2", + "onyx", }; return names; } diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index 6ecbf7e14b..e4cbe0018b 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -61,6 +61,21 @@ TEST_F(ChatTemplateAnalyzerTest, detectsGptOss) { EXPECT_TRUE(result.caps.supportsToolCalls); } +// --- Onyx --- + +TEST_F(ChatTemplateAnalyzerTest, detectsOnyx) { + std::string tmpl = loadTemplate("chat_template_onyx.jinja"); + ASSERT_FALSE(tmpl.empty()); + auto result = ChatTemplateAnalyzer::analyze(tmpl); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "onyx"); + // Onyx's template never reads the OpenAI "tool_calls" array, so unlike every + // other detected family, supportsToolCalls stays false -- see analyzer.cpp. + EXPECT_FALSE(result.caps.supportsToolCalls); +} + // --- Gemma4 --- TEST_F(ChatTemplateAnalyzerTest, detectsGemma4) { diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp new file mode 100644 index 0000000000..b253081edf --- /dev/null +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -0,0 +1,178 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#pragma GCC diagnostic pop + +#include "src/llm/io_processing/output_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// ============================================================================= +// Genuine request/response round trip for Onyx: render a request with the real +// chat template (minja, via ov::genai::Tokenizer::apply_chat_template -- same +// engine exercised by ChatTemplateEndToEndMinjaTest), hand-author the exact +// continuation the model would emit for that rendered prompt (grounded in what +// ChatTemplateEndToEndMinjaTest's Onyx_* tests already proved the template +// produces), and feed ONLY that continuation into OutputParser -- exactly what +// OVMS does in production (the parser only ever sees newly generated tokens, +// never the prompt). This is the missing link between: +// - chat_template_end_to_end_{minja,jinja}_test.cpp (request side only) +// - output_parsers/onyx_output_parser_test.cpp (response side only, with +// hand-written segments not derived from an actual rendered prompt) +// ============================================================================= +class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { +protected: + // TODO @atobiszei change tokenizer for onyx + const std::string& tokenizerPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m", false); + const std::string& chatTemplatesPath = getGenericFullPathForSrcTest("/ovms/src/test/llm/chat_templates", false); + + static std::string loadTemplateFile(const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) { + return ""; + } + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + } + + // Renders chatHistory with the real Onyx template via minja and asserts the + // generation prompt ends with the bare "<|start|>assistant" the parser tests + // assume (no trailing "<|message|>", no implicit recipient). + std::string renderPrompt(ov::genai::ChatHistory& chatHistory) { + std::string chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + ov::genai::Tokenizer tokenizer(tokenizerPath); + tokenizer.set_chat_template(chatTemplate); + std::string rendered = tokenizer.apply_chat_template(chatHistory, /*add_generation_prompt=*/true); + static const std::string generationPromptTail = "<|start|>assistant"; + EXPECT_TRUE(rendered.size() >= generationPromptTail.size() && + rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) + << "Generation prompt tail changed, Onyx parser assumptions may be stale: " << rendered; + return rendered; + } + + // Simulates generation: appends modelContinuation to the rendered prompt (for + // documentation / sanity only) and runs OutputParser on modelContinuation alone, + // matching what OVMS actually hands the parser. + ParsedOutput parseModelContinuation(const std::string& modelContinuation, bool toolsAvailable = true) { + ov::genai::Tokenizer tokenizer(tokenizerPath); + auto generatedTensor = tokenizer.encode(modelContinuation, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + + static ToolsSchemas_t emptyToolsSchema{}; // Onyx tool parser is not schema-driven, see onyx_tool_parser.hpp + OutputParser outputParser(tokenizer, "onyx", "onyx", emptyToolsSchema); + return outputParser.parse(generatedTokens, toolsAvailable); + } +}; + +// ============================================================================= +// Turn 1 of the muse/README.md "get_weather" example: user asks a question, the +// prompt is rendered, and the model's tool-call continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("You can call get_weather(city)."), std::string::npos) << prompt; + + // Model continuation for a tool call, exactly as documented in muse/README.md. + std::string modelContinuation = R"( to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city": "SF"})"); +} + +// ============================================================================= +// Turn 2 of the same example: tool result fed back into history (Onyx's own +// "name" + role="tool" shape, NOT OpenAI's tool_call_id), then the model's final +// answer continuation is parsed. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinalAnswer) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"system","content":"You can call get_weather(city)."})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"Weather in SF?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"city\": \"SF\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"functions.get_weather","content":"{\"temp\": 65}"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; + // NOTE (important, non-obvious): OpenVINO GenAI's own minja-path history + // preprocessing -- independent of the raw Jinja template's `elif role == + // 'tool'` branch -- rewrites role="tool" messages into role="user" with a + // generic wrapped "tool_response" JSON object whenever it determines the + // template lacks native tool-call support (the same probe underlying + // caps.supportsToolCalls == false, see chat_template_end_to_end_minja_test.cpp's + // Onyx tests). So Onyx's own `elif role == 'tool'` template branch is + // effectively DEAD CODE on the minja path today -- it never actually fires. + EXPECT_NE(prompt.find(R"(<|start|>user<|message|>{ + "tool_response": { + "tool": "functions.get_weather", + "content": "{\"temp\": 65}" + } +}<|eot|>)"), + std::string::npos) + << prompt; + + std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Private reasoning ("recipient": "self") round trip: the model reasons first, +// which the OnyxReasoningParser must classify as reasoning, not content. +// ============================================================================= +TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsPrivateReasoning) { + ov::genai::ChatHistory chatHistory; + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's 2+2?"})")); + + std::string prompt = renderPrompt(chatHistory); + EXPECT_NE(prompt.find("What's 2+2?"), std::string::npos) << prompt; + + std::string modelContinuation = R"( to=self<|message|>2+2 is a basic addition.<|eom|>)"; + ParsedOutput parsedOutput = parseModelContinuation(modelContinuation); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "2+2 is a basic addition."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 3d284ea300..e8e8515ec5 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -665,3 +665,52 @@ What's the weather in Paris?<|im_end|> )"; EXPECT_EQ(appliedOutput, expectedOutput); } + +// Onyx (early preview model) chat template, rendered via the real Python Jinja2 +// engine. Onyx's template does not read the standard OpenAI "tool_calls" array +// at all -- only message['content'] (plain string) and an Onyx-specific +// message['recipient'] field (e.g. "functions.get_weather", "self", "user"). +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp), +// but deliberately leaves caps.supportsToolCalls false: that flag means "this +// template natively re-serializes an incoming OpenAI tool_calls array", which +// this test demonstrates Onyx's template does NOT do (the tool call is silently +// dropped below). +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // Unlike the minja path (which has its own generic tool-call fallback), + // the real Python Jinja2 engine has no such fallback: the template renders + // message['content'] literally, i.e. the empty string, and the tool call + // information is silently dropped. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|><|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape via Python Jinja2: "recipient" field instead of the +// OpenAI "tool_calls" array. Ends the turn with "<|eom|>" (continuation marker) +// rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index c271788a1a..ec97945e04 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -620,6 +620,145 @@ TEST_F(ChatTemplateEndToEndMinjaTest, MiniCPM5_ToolCallWithStringArgsExpectedToF EXPECT_TRUE(caps.missnamedReasoningField.empty()); } +// ============================================================================= +// Onyx (early preview model) chat template. Unlike every other template in this +// suite, Onyx's own Jinja template does not consume the standard OpenAI +// "tool_calls" list at all -- it only reads message['content'] (a plain string) +// and an Onyx-specific message['recipient'] field (e.g. "functions.get_weather", +// "self", "user"). Feeding it a standard tool_calls-shaped assistant message +// therefore renders an effectively empty assistant turn. +// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp) and +// sets detectedToolParser/detectedReasoningParser, but deliberately leaves +// caps.supportsToolCalls false since the template can't natively round-trip an +// OpenAI tool_calls history (demonstrated by this very test), so no input-side +// workaround is applied either -- detectedToolParser only affects output parsing. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + + // The template itself never reads "tool_calls" (it only looks at + // message['content'] and message['recipient']). Because caps.supportsToolCalls + // is false here, OVMS does not apply its own tool-call workaround either. + // Minja's own generic fallback (used for templates it detects have no native + // tool-call rendering) kicks in instead and serializes the whole message + // (tool_calls + content) as a JSON blob into message['content'] -- the + // function name/args are NOT lost, but they end up as raw, unparsed JSON text + // rather than in Onyx's native " to=functions." / <|eom|> framing. + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|>{ + "tool_calls": [ + { + "name": "get_weather", + "arguments": { + "location": "Paris", + "unit": "celsius" + }, + "id": "call_abc123" + } + ], + "content": "" +}<|eot|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Onyx's own message shape: instead of the OpenAI "tool_calls" array, the +// assistant turn carries a "recipient" field (here "functions.get_weather") +// and a plain-string content holding the raw JSON arguments. This is the shape +// Onyx's template actually understands, ending the turn with "<|eom|>" (a +// continuation marker) rather than "<|eot|>". +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + +// ============================================================================= +// Full-scope round trip: exercises every message shape the Onyx template +// natively understands in a single history, not just one shape in isolation -- +// user prompt -> assistant tool call (recipient=functions., continuation +// "<|eom|>") -> tool call response (role="tool") -> assistant final answer +// (recipient="user", "<|eot|>"). +// +// This surfaces a second, previously undocumented gap alongside the tool_calls +// one above (see muse/chat_template_issues.md): because caps.supportsToolCalls +// is false for Onyx, ChatTemplateAdapter's generic fallback also intercepts +// plain role="tool" messages -- not just assistant tool_calls -- and rewrites +// them into a synthetic role="user" message serializing {tool, content} as a +// JSON blob, rather than passing them through to the template's own native +// "tool"-role branch (which expects message['name'] + message['content'] and +// would render "<|start|>tool <|message|>...<|eot|>"). So even a chat +// history built entirely out of Onyx's own native fields (recipient) still +// does not round-trip once a plain OpenAI-shaped tool response message is +// mixed in. +// ============================================================================= +TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"tool","name":"get_weather","content":"{\"temperature\":15,\"unit\":\"celsius\"}"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","recipient":"user","content":"It's 15C in Paris."})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + EXPECT_FALSE(caps.supportsToolCalls); + + // Known gap (see class comment above): the "tool" message is NOT rendered via + // the template's native "<|start|>tool <|message|>...<|eot|>" branch -- + // ChatTemplateAdapter's fallback rewrites it into a synthetic user message + // carrying a JSON blob first. + std::string expectedOutput = + R"(<|start|>system<|message|>You are a helpful assistant.<|eot|>)" + R"(<|start|>user<|message|>What's the weather in Paris?<|eot|>)" + R"(<|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|>)" + "<|start|>user<|message|>{\n" + " \"tool_response\": {\n" + " \"tool\": \"get_weather\",\n" + " \"content\": \"{\\\"temperature\\\":15,\\\"unit\\\":\\\"celsius\\\"}\"\n" + " }\n" + "}<|eot|>" + R"(<|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|>)" + R"(<|start|>assistant)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} + // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. diff --git a/src/test/llm/chat_templates/chat_template_onyx.jinja b/src/test/llm/chat_templates/chat_template_onyx.jinja new file mode 100644 index 0000000000..0fa7b9d204 --- /dev/null +++ b/src/test/llm/chat_templates/chat_template_onyx.jinja @@ -0,0 +1,41 @@ +{{- bos_token -}} +{%- macro render_parts(content) -%} +{%- if content is string -%}{{- content -}} +{%- else -%} +{%- for part in content -%} +{%- if part['type'] == 'image' -%}{{- '<|image|>' -}} +{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}} +{%- elif part['type'] == 'text' -%}{{- part['text'] -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%}{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}{%- endfor -%} +{%- if add_generation_prompt and not ns.has_system -%} +{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}} +{%- endif -%} +{%- for message in messages -%} +{%- set role = message['role'] -%} +{%- if role == 'assistant' -%} +{%- set recipient = message.get('recipient') -%} +{%- set end_turn = message.get('end_turn') -%} +{%- if end_turn is none -%} +{%- set end_turn = not (recipient and recipient != 'user') -%} +{%- endif -%} +{{- '<|start|>assistant' -}} +{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%} +{{- '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- ('<|eot|>' if end_turn else '<|eom|>') -}} +{%- elif role == 'tool' -%} +{%- set name = message.get('name', '') -%} +{{- '<|start|>tool ' + name + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- else -%} +{%- set header = role -%} +{%- if message.get('name') -%}{%- set header = role + ' ' + message['name'] -%}{%- endif -%} +{{- '<|start|>' + header + '<|message|>' -}}{{- render_parts(message['content']) -}} +{{- '<|eot|>' -}} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%} diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index 2b1840f0de..43ae0a6e1b 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -99,3 +99,23 @@ TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { ASSERT_TRUE(result[0]["content"].is_array()); EXPECT_EQ(result[0]["content"].size(), 2u); } + +TEST(TextContentNormalizationProcessorTest, NullContentNormalizedToEmptyString) { + // Standard OpenAI shape for e.g. an assistant message that only carries + // "tool_calls": content is explicitly null (not just absent). Some templates + // (e.g. Onyx's) unconditionally render content for every message and error out + // on null, so this must be normalized to "" the same way a missing field is. + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("assistant")}}; + msg["content"] = ov::genai::JsonContainer(nullptr); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + TextContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + ASSERT_TRUE(result[0]["content"].is_string()); + EXPECT_EQ(result[0]["content"].as_string().value_or("__unset__"), ""); +} diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp new file mode 100644 index 0000000000..4de64ee3f0 --- /dev/null +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -0,0 +1,619 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/io_processing/onyx/onyx_tool_parser.hpp" +#include "src/llm/io_processing/output_parser.hpp" +#include "src/logging.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +// Onyx does not ship a converted HF tokenizer in this early preview, and none of the +// segments the parser looks for ("to=functions.", "<|message|>", "<|eom|>", "<|eot|>", +// "to=self") are real special tokens of the model this parser is designed for -- they are +// plain text sequences that must round-trip losslessly through encode()+decode() on ANY +// tokenizer. facebook/opt-125m is already used the same way for chat-template testing +// (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid pulling in a new +// model fixture. +// TODO @atobiszei replace when tokenizer is available +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/facebook/opt-125m"; +#endif + +static std::unique_ptr opt125mTokenizer; + +// Onyx never consults tool schemas (arguments are forwarded as a raw JSON blob verbatim, +// no per-parameter type coercion like Qwen3CoderToolParser) -- these mirror +// Qwen3CoderOutputParserTest's toolSchemasInput/toolsSchemas setup purely so the test +// fixture shape matches, and so a real schema is on hand if that ever changes. +static std::map toolSchemasInput = { + {"get_weather", R"({"properties":{"location":{"type":"string","description":"City name."},"unit":{"type":"string","description":"Temperature unit."}},"required":["location"]})"}, + {"get_time", R"({"properties":{"city":{"type":"string","description":"City name."}},"required":["city"]})"}, + {"get_current_location", R"({"properties":{},"required":[]})"}, + {"string_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."}},"required":["arg1"]})"}, + {"cd", R"({"properties":{"folder":{"type":"string","description":"Path"}},"required":["folder"]})"}, + {"string_int_tool", R"({"properties":{"arg1":{"type":"string","description":"A string argument."},"arg2":{"type":"integer","description":"An integer argument."}},"required":["arg1","arg2"]})"}}; + +static std::vector> schemaDocsStorage; + +static ToolsSchemas_t convertStringToolSchemasStringToToolsSchemas( + const std::map& input) { + ToolsSchemas_t result; + schemaDocsStorage.clear(); + for (const auto& [name, schemaStr] : input) { + auto schemaDoc = std::make_unique(); + if (schemaDoc->Parse(schemaStr.c_str()).HasParseError()) { + throw std::runtime_error("Failed to parse schema for tool: " + name); + } + result[name] = {schemaDoc.get(), schemaStr}; + schemaDocsStorage.push_back(std::move(schemaDoc)); + } + return result; +} + +static ovms::ToolsSchemas_t toolsSchemas = convertStringToolSchemasStringToToolsSchemas(toolSchemasInput); + +class OnyxOutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParser; + + static void SetUpTestSuite() { + try { + opt125mTokenizer = std::make_unique(tokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize opt-125m tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize opt-125m tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + opt125mTokenizer.reset(); + } + + void SetUp() override { + outputParser = std::make_unique(*opt125mTokenizer, "onyx", "onyx", toolsSchemas); + } + + ParsedOutput generateParsedOutput(const std::string& input, bool toolsAvailable = true) { + auto generatedTensor = opt125mTokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + std::vector generatedTokens(generatedTensor.data(), generatedTensor.data() + generatedTensor.get_size()); + return outputParser->parse(generatedTokens, toolsAvailable); + } + + // Shared by streaming tests: compares a parseChunk() delta against the expected JSON, + // masking the randomly generated tool call id (kept as one helper rather than the same + // id-masking block duplicated per test, mirroring Qwen3CoderOutputParserTest usage). + static void assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk); + + // Wraps a raw (unescaped) string value into a JSON object {"arg1":""}, + // using rapidjson's serializer to handle all escaping. This lets us write raw PLC/Python + // code in tests without manually counting backslashes. + static std::string wrapRawCodeAsToolArgs(const std::string& rawCode) { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Value val(rawCode.c_str(), static_cast(rawCode.size()), doc.GetAllocator()); + doc.AddMember("arg1", val, doc.GetAllocator()); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + doc.Accept(writer); + return buf.GetString(); + } + + // Builds the expected arguments delta JSON string for a given tool call index and + // raw (unescaped) code content: {"delta":{"tool_calls":[{"index":N,"function":{"arguments":"..."}}]}} + static std::string expectedArgsDelta(int index, const std::string& rawCode) { + std::string argsJson = wrapRawCodeAsToolArgs(rawCode); + // argsJson is a valid JSON object string -- we need to embed it as an escaped + // string value inside the outer delta JSON. Use rapidjson to build the whole thing. + rapidjson::Document outer; + outer.SetObject(); + rapidjson::Document::AllocatorType& alloc = outer.GetAllocator(); + rapidjson::Value delta(rapidjson::kObjectType); + rapidjson::Value toolCalls(rapidjson::kArrayType); + rapidjson::Value tc(rapidjson::kObjectType); + tc.AddMember("index", index, alloc); + rapidjson::Value func(rapidjson::kObjectType); + rapidjson::Value argsVal(argsJson.c_str(), static_cast(argsJson.size()), alloc); + func.AddMember("arguments", argsVal, alloc); + tc.AddMember("function", func, alloc); + toolCalls.PushBack(tc, alloc); + delta.AddMember("tool_calls", toolCalls, alloc); + outer.AddMember("delta", delta, alloc); + rapidjson::StringBuffer buf; + rapidjson::Writer writer(buf); + outer.Accept(writer); + return buf.GetString(); + } +}; + +// A single generate() call stops at the first "<|eom|>"/"<|eot|>" (both are configured as +// eos tokens for Onyx), so only one of these three segment shapes is ever produced at a time. + +TEST_F(OnyxOutputParserTest, FinalAnswerWithExplicitUserRecipient) { + ParsedOutput parsedOutput = generateParsedOutput(" to=user<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, FinalAnswerWithoutRecipient) { + // The template only renders " to=" when the caller supplies a recipient; the model may + // also just emit "<|message|>...<|eot|>" directly with no recipient at all. + ParsedOutput parsedOutput = generateParsedOutput("<|message|>It's 65F in SF.<|eot|>"); + + EXPECT_EQ(parsedOutput.content, "It's 65F in SF."); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, PrivateReasoningOnly) { + ParsedOutput parsedOutput = generateParsedOutput(" to=self<|message|>Let me think about this.<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, "Let me think about this."); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +TEST_F(OnyxOutputParserTest, ToolCallWithRawJsonArguments) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + // Onyx passes arguments through verbatim -- no schema-driven reformatting like qwen3coder. + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); + EXPECT_EQ(parsedOutput.toolCalls[0].id.empty(), false); +} + +TEST_F(OnyxOutputParserTest, ToolCallNotParsedWhenToolsUnavailable) { + // OutputParser::parse() only invokes the tool parser when toolsAvailable is true. + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>", /*toolsAvailable=*/false); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); + // Known current limitation: the reasoning parser (which always runs) intentionally + // leaves "to=functions." segments untouched so the tool parser can claim them -- but + // if the tool parser never runs, the raw wrapped segment is surfaced as-is in content. + EXPECT_EQ(parsedOutput.content, " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"); +} + +TEST_F(OnyxOutputParserTest, MalformedOutputWithoutMessageTagLeftUntouched) { + ParsedOutput parsedOutput = generateParsedOutput("some unexpected raw text without any framing"); + + EXPECT_EQ(parsedOutput.content, "some unexpected raw text without any framing"); + EXPECT_EQ(parsedOutput.reasoning, ""); + EXPECT_EQ(parsedOutput.toolCalls.size(), 0); +} + +// ============================================================================= +// Streaming is implemented on top of OnyxToolParserImpl, a pure state machine that +// accumulates raw text and hands back a fully assembled tool call once its "<|eom|>" +// end tag is seen (mirroring Qwen3CoderToolParserImpl) -- unary parse() below drives +// that same impl with the whole content as a single chunk, so it is the single-shot +// degenerate case of streaming, not a parallel implementation of the tag walk. +// +// Because Onyx's arguments are already a complete raw JSON blob needing no per-parameter +// schema coercion, they are still sent to the client as a single delta once the tool call +// closes (matching Qwen3CoderToolParser's sendFullDelta) rather than streamed incrementally +// as raw text arrives -- only the function name streams as its own delta once known. +// +// Chunk boundaries below are deliberately awkward (splitting the function name and the +// JSON arguments mid-token) to exercise the Content/InsideName/InsideArguments state +// machine, mirroring Qwen3CoderOutputParserTest.StreamingSimpleToolCall. +// ============================================================================= +void OnyxOutputParserTest::assertDeltaMatches(const std::optional& doc, const std::optional& expectedDelta, const std::string& chunk) { + if (!expectedDelta.has_value()) { + EXPECT_FALSE(doc.has_value()) << "Expected nullopt for chunk: " << chunk; + return; + } + ASSERT_TRUE(doc.has_value()) << "Expected a delta for chunk: " << chunk; + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + // If both strings contain "id":"...", compare id values by length and alphanumeric, else compare whole strings + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + docStr.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expected.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + } + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; +} + +TEST_F(OnyxOutputParserTest, StreamingSimpleToolCall) { + // Mirrors Qwen3CoderOutputParserTest.StreamingSimpleToolCall's rigor: adversarial + // chunk boundaries, content before/between tool calls, complex argument values + // (PLC structured text, Python with triple-quotes/f-strings/escape sequences), + // adapted to Onyx's "to=functions.<|message|><|eom|>" format. + // + // Unlike qwen3coder there is no per-parameter incremental streaming to test + // (qwen3coder's tags) since Onyx's arguments are a single raw JSON blob. + // However, content before/between tool calls IS tested because the OutputParser + // streaming framework handles that generically (UNKNOWN -> CONTENT transition when no + // start tag is found). + // + // Key structural differences from qwen3coder: + // - Start tag is "to=functions." (not "") + // - Name delimiter is "<|message|>" (not ">") + // - End tag is "<|eom|>" (not "") + // - Arguments are a single raw JSON blob (not per-parameter XML tags) + // - PLC/Python code must be JSON-escaped within the arguments blob + + // Raw PLC structured text code (mirrors qwen3coder's FC_CreateJsonPayload). + // Written as a raw string literal so it's human-readable; wrapRawCodeAsToolArgs() + // handles all the JSON escaping at runtime. + const std::string plcCode = R"(FUNCTION FC_CreateJsonPayload : STRING +VAR_INPUT + Value1 : REAL; + Value2 : INT; + Value3 : BOOL; + Value4 : STRING(100); +END_VAR +VAR_OUTPUT + JsonPayload : STRING(1000); +END_VAR +VAR + TempStr : STRING(100); +END_VAR + + JsonPayload := '{'; + JsonPayload := JsonPayload + '"value1":' + REAL_TO_STRING(Value1, '', 2) + ','; + JsonPayload := JsonPayload + '"value2":' + INT_TO_STRING(Value2) + ','; + JsonPayload := JsonPayload + '"value3":' + BOOL_TO_STRING(Value3) + ','; + JsonPayload := JsonPayload + '"value4":"' + Value4 + '"'; + JsonPayload := JsonPayload + '}'; + +END_FUNCTION)"; + + // Raw Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). + const std::string pythonCode = R"( +if __name__ == "__main__": + addresses = {} + addresses["Hodor"] = """The door""" + addresses["Arya"] = "Winterfell" + for name, address in addresses.items(): + print(f'\n\t{name} lives at {address}\n\r'))"; + + int i = -1; + std::vector>> chunkToDeltaVec{ + // Content before any tool call -- OutputParser sees no start tag match, emits content. + {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, + // Start tag "to=functions." split across several arbitrarily small chunks. + // Note: leading space before "to=" is just normal content/separator; the start + // tag the framework looks for is "to=functions." without the space. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=fun", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ctions.", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Function name streams in across several small chunks -- still no delta + // (OnyxToolParserImpl is in InsideName state, accumulating). + {"get", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"_", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"weath", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"er", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|message|>" itself split mid-tag -- name delta emitted once the full tag lands. + {"<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"get_weather"}}]}})"}, + // Raw JSON argument text (with a nested object) split at awkward byte boundaries. + {"{\"locat", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ion\":\"Pa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ris\",\"opt", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ions\":{\"unit", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\":\"cel", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"sius\"}}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // "<|eom|>" split mid-tag -- closes the tool call once complete. + {"<|e", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"om|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":\"Paris\",\"options\":{\"unit\":\"celsius\"}}"}}]}})"}, + // Content between tool calls (mirrors qwen3coder's "POTENTIALLY EXISINT CONTENT"). + // In TOOL_CALLS_WAITING_FOR_TOOL phase, text without start tag match waits for more. + {"POTENTIALLY EXISINT CONTENT", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Second tool call -- start tag + name + <|message|> split across tiny chunks. + {" to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=functi", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ons.str", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ing_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|messa", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ge|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":1,"function":{"name":"string_tool"}}]}})"}, + // Arguments split across chunks. + {"{\"arg1\":", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"\"STRI", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"NG_VALUE\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eo", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"m|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"arg1\":\"STRING_VALUE\"}"}}]}})"}, + // More content between tool calls (mirrors "CONTENT_AFTER_TOOL_CALL"). + {"CONTENT_AFTER_TOOL_CALL", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Third tool call -- string_int_tool with two parameters in JSON (integer stays + // numeric). Start tag + name + <|message|> split differently from previous calls. + {" to=func", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"tions.strin", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g_int_tool<|", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":2,"function":{"name":"string_int_tool"}}]}})"}, + // Arguments with a leading \n in the string value (matches qwen3coder's + // "\nANOTHER_STRING_VALUE" pattern) and an integer parameter. + {"{\"arg1\":\"\\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"ANOTHER_STRING_VALUE\",\"ar", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"g2\":314", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"1522}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"arg1\":\"\\nANOTHER_STRING_VALUE\",\"arg2\":3141522}"}}]}})"}, + // "NOTHING IMPORTANT HERE" content between calls (mirrors qwen3coder). + {"NOTHING IMPORTANT HERE", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // A "bfcl draft" style call -- cd tool. Start tag arrives with some preceding + // text just like qwen3coder's "part of bfcl 'draft'.\n\n\n" pattern. + {"part of bfcl 'draft'.\n\n to=functions.cd<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":3,"function":{"name":"cd"}}]}})"}, + {"{\"folder\":\"ResearchDocs\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":3,"function":{"arguments":"{\"folder\":\"ResearchDocs\"}"}}]}})"}, + // PLC structured text code as a tool argument (mirrors qwen3coder's + // FC_CreateJsonPayload test). Raw code is defined above as plcCode; the helper + // wrapRawCodeAsToolArgs() handles all JSON escaping via rapidjson so we don't + // need to manually count backslashes. Sent as a single chunk since the interesting + // escaping complexity is in the content, not in chunk-boundary splitting. + {" to=functions.string_tool", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":4,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(plcCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, expectedArgsDelta(4, plcCode)}, + // Python code with triple-quotes, f-strings, escape sequences (mirrors + // qwen3coder's last test case). Also sent as a single chunk -- the chunk-boundary + // adversarial testing is covered by the earlier tool calls above. + {" to=functions.string_tool<|mess", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"age|>", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":5,"function":{"name":"string_tool"}}]}})"}, + {wrapRawCodeAsToolArgs(pythonCode), ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|eom|>", ov::genai::GenerationFinishReason::STOP, expectedArgsDelta(5, pythonCode)}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + const std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk[" << i << "]: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk[" << i << "]: " << chunk; + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk[" << i << "] (ignoring id value): " << chunk; + } else { + SPDLOG_ERROR("Expected:\n{}", expected); + SPDLOG_ERROR("Got:\n{}", docStr); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + // Validate that arguments fields are valid JSON + if (expected.find("arguments") != std::string::npos) { + auto docJsonIt = doc->FindMember("delta"); + ASSERT_NE(docJsonIt, doc->MemberEnd()); + auto toolCallsIt = docJsonIt->value.FindMember("tool_calls"); + ASSERT_NE(toolCallsIt, docJsonIt->value.MemberEnd()); + for (const auto& toolCall : toolCallsIt->value.GetArray()) { + auto functionIt = toolCall.FindMember("function"); + ASSERT_NE(functionIt, toolCall.MemberEnd()); + auto argumentsIt = functionIt->value.FindMember("arguments"); + ASSERT_NE(argumentsIt, functionIt->value.MemberEnd()); + const std::string& argumentsStr = argumentsIt->value.GetString(); + rapidjson::Document argsDoc; + argsDoc.Parse(argumentsStr.c_str()); + EXPECT_FALSE(argsDoc.HasParseError()) << "Arguments is not valid JSON for chunk[" << i << "]: " << chunk << "\nArguments string:\n" + << argumentsStr; + } + } + } + } else { + EXPECT_TRUE(false) << "Mismatch between expectedDelta and doc for id: " << i << " chunk:\n" + << chunk + << "\nexpectedDelta:\n" + << (expectedDelta.has_value() ? expectedDelta.value() : "EMPTY_DELTA") + << "\nGot doc:\n" + << (doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "NO_DOC"); + FAIL() << "Mismatch between expectedDelta and doc for chunk[" << i << "]: " << chunk; + } + } +} + +// ============================================================================= +// Proves the "unary is an edge case of streaming" property holds structurally, not +// just by coincidence: OnyxToolParser::parse() literally drives the same +// OnyxToolParserImpl used by parseChunk() (see onyx_tool_parser.cpp), so this is +// really just re-checking that the unary entry point wires into the same state +// machine already covered above. +// ============================================================================= +TEST_F(OnyxOutputParserTest, UnaryToolCallMatchesStreamingReuse) { + ParsedOutput parsedOutput = generateParsedOutput(" to=functions.get_weather<|message|>{\"location\":\"Paris\",\"unit\":\"celsius\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"location\":\"Paris\",\"unit\":\"celsius\"}"); +} + +TEST_F(OnyxOutputParserTest, UnaryTwoSequentialToolCalls) { + ParsedOutput parsedOutput = generateParsedOutput( + " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "get_time"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, "{\"city\":\"SF\"}"); +} + +// ============================================================================= +// Direct OnyxToolParserImpl unit tests -- mirrors Qwen3CoderOutputParserTest's +// TestJustParserImplUnary*/TestJustParserImplStreamStep* layer (which exercises the +// state machine directly, below OutputParser/OnyxToolParser), previously untested here. +// ============================================================================= +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":\"Paris\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"location\":\"Paris\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithNoToolCall) { + const std::string input = "Unexpected void found. Philosophical crisis imminent."; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_FALSE(callsOpt.has_value()); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(content, input); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplUnaryWithTwoToolCalls) { + const std::string input = " to=functions.get_weather<|message|>{\"city\":\"SF\"}<|eom|> to=functions.get_time<|message|>{\"city\":\"SF\"}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ToolCalls_t& calls = callsOpt.value(); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + ASSERT_EQ(calls.size(), 2) << input; + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(calls[1].name, "get_time"); + EXPECT_EQ(calls[1].arguments, "{\"city\":\"SF\"}"); + EXPECT_EQ(content, ""); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithNoStateChange) { + const std::string input = "Some content without tool calls"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithPartialToolCall) { + const std::string input = " to=functions.get_weather<|message|>{\"location\":"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_FALSE(stepResult.has_value()); + ASSERT_TRUE(parser.getCurrentFunctionName().has_value()); + EXPECT_EQ(parser.getCurrentFunctionName().value(), "get_weather"); +} + +TEST_F(OnyxOutputParserTest, TestJustParserImplStreamStepWithToolCallNoArgs) { + const std::string input = " to=functions.get_current_location<|message|>{}<|eom|>"; + auto content = input; + ovms::OnyxToolParserImpl parser; + auto stepResult = parser.parseChunk(content); + ASSERT_TRUE(stepResult.has_value()); + auto& calls = stepResult.value(); + ASSERT_EQ(calls.size(), 1) << input; + EXPECT_EQ(calls[0].name, "get_current_location"); + EXPECT_EQ(calls[0].arguments, "{}"); +} + +// ============================================================================= +// Qwen3CoderOutputParserTest test cases, for reference/parity comparison (this file +// intentionally does not have a 1:1 test for every one of these -- see inline notes +// on why some don't apply to Onyx's simpler, non-schema-driven, single-JSON-blob +// argument format): +// Parse1ToolCall1Function1ArgumentTagsNewline +// Parse1ToolCall1Function1ArgumentNoProperBeginTag +// Parse1ToolCallNestedXmlNotFromSchema +// ParseTwoToolCalls1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentTagsNoNewline +// Parse1ToolCall1Function1ArgumentMultilineValue +// TestJustParserImplUnaryToolCall -- covered above +// TestJustParserImplUnaryWithNoToolCall -- covered above +// TestJustParserImplUnaryWithContent -- N/A: Onyx's grammar never +// has plain content before/after a tool-call tag within the same generated turn +// TestJustParserImplUnaryWithThreeParameters -- N/A: no per-parameter +// schema-driven typing; arguments are always a single opaque JSON blob +// TestJustParserImplUnaryWithEnforcementOfStringParameter -- N/A, same reason +// TestJustParserImplUnaryWithNotPresentToolSchema -- N/A, same reason (Onyx +// never even looks at tool schemas -- see ToolCallWithRawJsonArguments above) +// TestJustParserImplUnaryWithJsonObjectArgument -- covered by nested-object +// case in StreamingSimpleToolCall above +// TestJustParserImplUnaryWithTwoToolCalls -- covered above +// TestJustParserImplUnaryToolCallNoMatchingToolParameterTypeMapEntry -- N/A, same reason +// TestJustParserImplUnaryToolCallWithRepeatedArgument -- N/A, same reason (no +// per-parameter parsing to have a "repeated argument" concept at all) +// TestJustParserImplStreamStepWithMoreThan1StateChange -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above (both calls resolve in one parseChunk) +// TestJustParserImplStreamStepWithNoStateChange -- covered above +// TestJustParserImplStreamStepWithPartialToolCall -- covered above +// TestJustParserImplStreamStepWithTwoToolCalls -- covered by +// TestJustParserImplUnaryWithTwoToolCalls above +// TestJustParserImplStreamStepWithToolCallNoArgs -- covered above +// Qwen3CoderOutputParserParametrizedTest.TestJustParserImplWithVariousArgumentTypes -- N/A: +// parametrized over per-parameter type coercion (string/int/float/bool/object/list), +// which does not exist for Onyx (raw JSON passthrough only) +// StreamingSimpleToolCall -- covered above (adapted; +// see comment on that test for what was intentionally omitted/adjusted) +// ============================================================================= + From af40fa5d46508f063aec8c3ae8254db2238cbe5c Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Tue, 28 Jul 2026 11:01:31 +0200 Subject: [PATCH 16/38] Build files --- Dockerfile.ubuntu | 8 ++++++-- Makefile | 9 ++++++++- versions.mk | 19 ++++++++++++++++--- windows_install_build_dependencies.bat | 6 ++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index fef0ed40e0..40048f51db 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -156,10 +156,14 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" +# onyx-support patches (temporary, one-off - see patches/*/readme.md for the +# commits they apply to) +COPY patches /patches/ + ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ ARG SDL_OPS="-Wl,-z,relro,-z,now,-z,noexecstack -fPIE -pie -fstack-protector-strong -fexceptions -fasynchronous-unwind-tables -fcf-protection -fpic -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fno-strict-overflow -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wuninitialized" # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git apply /patches/openvino/0001-openvino.patch && git apply /patches/openvino/0002-openvino.patch && git apply /patches/openvino/0003-openvino.patch && git apply /patches/openvino/0004-openvino.patch && git apply /patches/openvino/0005-openvino.patch && git submodule update --init --recursive WORKDIR /openvino/build RUN if [ "$ov_use_binary" == "0" ]; then \ if [[ $debug_bazel_flags == *"py_off"* ]]; then \ @@ -228,7 +232,7 @@ ARG ov_genai_org=openvinotoolkit WORKDIR /openvino_genai/ # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then \ - git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive && \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git apply /patches/openvino.genai/0001-openvino.genai.patch && git submodule update --init --recursive && \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} " -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ && \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ && \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ && \ diff --git a/Makefile b/Makefile index 2fcc40b417..ff11a83ac4 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,14 @@ endif ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" - OV_USE_BINARY ?= 1 + # ubuntu24 defaults to building OpenVINO/GenAI from source (see versions.mk) + # so that the onyx-support patches in ./patches can be applied. Other ubuntu + # flavors keep using the prebuilt binary package by default. + ifeq ($(BASE_OS),ubuntu24) + OV_USE_BINARY ?= 0 + else + OV_USE_BINARY ?= 1 + endif ifeq ($(findstring ubuntu22,$(BASE_OS)),ubuntu22) ifeq ($(OV_USE_BINARY),0) $(error OV_USE_BINARY = 0 not supported on Ubuntu22 OS) diff --git a/versions.mk b/versions.mk index d111954e0b..2efa60cadc 100644 --- a/versions.mk +++ b/versions.mk @@ -19,9 +19,22 @@ # Any variable can be overridden by the environment or command-line. # Source repository git commits / branches (used for source builds) -OV_SOURCE_BRANCH ?= 894ddbcccf0755986b5a6bab807e7d9dd5cb8bb1 -OV_TOKENIZERS_BRANCH ?= 2eca683e943db7aa2ba784d19f7d62399bf3ae03 -OV_GENAI_BRANCH ?= fbbeb800ff5e6c6029b3747f87c1231a8b424d5e +# NOTE: pinned to the commits required by the onyx-support patches in +# ./patches (see patches/openvino/readme.md and patches/openvino.genai/readme.md). +# This is a temporary, one-off pin - restore the previous commits below once +# the patches are no longer needed: +# OV_SOURCE_BRANCH ?= d08e55c64c37fde1f4f6157cc5f5e07dd36ce5e8 (pre-patch branch tip) +# OV_GENAI_BRANCH ?= 8981d6f848f17985979be0a9224251d181f68c56 (pre-patch branch tip) +# NOTE: OV_TOKENIZERS_BRANCH is intentionally left at its original commit - +# the tokenizers commit referenced by the genai patch's submodule bump +# (935443f5275ce93f362f9eb4fa2d9fa762dd3f22) does not exist in the +# openvinotoolkit/openvino_tokenizers repo (only reachable from a fork used +# during genai development), and OVMS builds tokenizers as a separate +# component (BUILD_TOKENIZERS=OFF in the genai cmake invocation) so this pin +# does not affect the OVMS build. +OV_SOURCE_BRANCH ?= 5b6997da03a7a0713fb4376f9109b4832383cc24 +OV_TOKENIZERS_BRANCH ?= master +OV_GENAI_BRANCH ?= c637ed85efebf1a44d5f0433845849a2d80b353c # Source repository organizations OV_SOURCE_ORG ?= openvinotoolkit diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 3ac79c2920..4ab3f5962c 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -256,6 +256,10 @@ cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin git checkout %OV_SOURCE_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +for %%P in (0001 0002 0003 0004 0005) do ( + git apply "%BACK_CWD%\patches\openvino\%%P-openvino.patch" + if !errorlevel! neq 0 exit /b !errorlevel! +) git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules @@ -304,6 +308,8 @@ cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin git checkout %OV_GENAI_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! +git apply "%BACK_CWD%\patches\openvino.genai\0001-openvino.genai.patch" +if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( mkdir build From 64fdeec7eb6c19a8a938d703d9e2f858ee817931 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 30 Jul 2026 09:13:24 +0200 Subject: [PATCH 17/38] Experimenting with decode special tokens --- src/llm/io_processing/output_parser.cpp | 9 ++------- src/llm/io_processing/output_parser.hpp | 1 + 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6b9949632d..92e8a4dd7d 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -211,13 +211,10 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); -<<<<<<< HEAD } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); -======= } else if (toolParserName == "onyx") { toolParser = std::make_unique(tokenizer); ->>>>>>> 4d9b4248 (Model enablement WIP) } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -229,15 +226,13 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); -<<<<<<< HEAD } else if (reasoningParserName == "minicpm5") { reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); -======= } else if (reasoningParserName == "onyx") { reasoningParser = std::make_unique(tokenizer); ->>>>>>> 4d9b4248 (Model enablement WIP) + decodeWithSpecialTokens = true; } else if (!reasoningParserName.empty()) { throw std::runtime_error("Unsupported reasoning parser: \"" + reasoningParserName + "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); @@ -303,7 +298,7 @@ ParsedOutput OutputParser::parse(const std::vector& generatedTokens, co SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Raw model output: {}", tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(false))); } ParsedOutput parsedOutput; - parsedOutput.content = tokenizer.decode(generatedTokens); + parsedOutput.content = tokenizer.decode(generatedTokens, ov::genai::skip_special_tokens(!decodeWithSpecialTokens)); if (reasoningParser) { reasoningParser->parse(parsedOutput, generatedTokens); } diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 4e7c467f81..a6d0f7bbc5 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -58,6 +58,7 @@ class OutputParser { ov::genai::Tokenizer tokenizer; std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content + bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) // Streaming related members ProcessingPhase processingPhase = UNKNOWN; From 4ce873f5961f00934c6174f25bd54c91de84f4f8 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 14:05:19 +0200 Subject: [PATCH 18/38] rebase error fixed --- src/llm/io_processing/output_parser.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index dee5fe4c26..6a85001ab2 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -217,8 +217,6 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "minicpm5") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); - } else if (toolParserName == "onyx") { - toolParser = std::make_unique(tokenizer); } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); From f518ab9265180319de4c8c441d9362a2808a6905 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 14:07:55 +0200 Subject: [PATCH 19/38] rebase error fixed v2 --- .../chat_template_end_to_end_minja_test.cpp | 139 ------------------ 1 file changed, 139 deletions(-) diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index 16f03c2875..0157bcab0e 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -693,145 +693,6 @@ R"(# Valid recipients: "self", "user".<|eot|><|start|>user<|message|>What's the EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; } -// ============================================================================= -// Onyx (early preview model) chat template. Unlike every other template in this -// suite, Onyx's own Jinja template does not consume the standard OpenAI -// "tool_calls" list at all -- it only reads message['content'] (a plain string) -// and an Onyx-specific message['recipient'] field (e.g. "functions.get_weather", -// "self", "user"). Feeding it a standard tool_calls-shaped assistant message -// therefore renders an effectively empty assistant turn. -// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp) and -// sets detectedToolParser/detectedReasoningParser, but deliberately leaves -// caps.supportsToolCalls false since the template can't natively round-trip an -// OpenAI tool_calls history (demonstrated by this very test), so no input-side -// workaround is applied either -- detectedToolParser only affects output parsing. -// ============================================================================= -TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithStringArgs) { - chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); - ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; - - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"user","content":"What's the weather in Paris?"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); - - run(); - - ASSERT_FALSE(exceptionThrownDuringApplication); - - ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); - EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); - ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); - EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); - - EXPECT_FALSE(caps.supportsToolCalls); - EXPECT_FALSE(caps.requiresObjectArguments); - - // The template itself never reads "tool_calls" (it only looks at - // message['content'] and message['recipient']). Because caps.supportsToolCalls - // is false here, OVMS does not apply its own tool-call workaround either. - // Minja's own generic fallback (used for templates it detects have no native - // tool-call rendering) kicks in instead and serializes the whole message - // (tool_calls + content) as a JSON blob into message['content'] -- the - // function name/args are NOT lost, but they end up as raw, unparsed JSON text - // rather than in Onyx's native " to=functions." / <|eom|> framing. - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|>{ - "tool_calls": [ - { - "name": "get_weather", - "arguments": { - "location": "Paris", - "unit": "celsius" - }, - "id": "call_abc123" - } - ], - "content": "" -}<|eot|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); -} - -// ============================================================================= -// Onyx's own message shape: instead of the OpenAI "tool_calls" array, the -// assistant turn carries a "recipient" field (here "functions.get_weather") -// and a plain-string content holding the raw JSON arguments. This is the shape -// Onyx's template actually understands, ending the turn with "<|eom|>" (a -// continuation marker) rather than "<|eot|>". -// ============================================================================= -TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_ToolCallWithRecipientField) { - chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); - ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; - - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"user","content":"What's the weather in Paris?"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); - - run(true); - - ASSERT_FALSE(exceptionThrownDuringApplication); - - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); -} - -// ============================================================================= -// Full-scope round trip: exercises every message shape the Onyx template -// natively understands in a single history, not just one shape in isolation -- -// user prompt -> assistant tool call (recipient=functions., continuation -// "<|eom|>") -> tool call response (role="tool") -> assistant final answer -// (recipient="user", "<|eot|>"). -// -// This surfaces a second, previously undocumented gap alongside the tool_calls -// one above (see muse/chat_template_issues.md): because caps.supportsToolCalls -// is false for Onyx, ChatTemplateAdapter's generic fallback also intercepts -// plain role="tool" messages -- not just assistant tool_calls -- and rewrites -// them into a synthetic role="user" message serializing {tool, content} as a -// JSON blob, rather than passing them through to the template's own native -// "tool"-role branch (which expects message['name'] + message['content'] and -// would render "<|start|>tool <|message|>...<|eot|>"). So even a chat -// history built entirely out of Onyx's own native fields (recipient) still -// does not round-trip once a plain OpenAI-shaped tool response message is -// mixed in. -// ============================================================================= -TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { - chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); - ASSERT_FALSE(chatTemplate.empty()) << "Failed to load onyx template"; - - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"user","content":"What's the weather in Paris?"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"tool","name":"get_weather","content":"{\"temperature\":15,\"unit\":\"celsius\"}"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"assistant","recipient":"user","content":"It's 15C in Paris."})")); - - run(true); - - ASSERT_FALSE(exceptionThrownDuringApplication); - - EXPECT_FALSE(caps.supportsToolCalls); - - // Known gap (see class comment above): the "tool" message is NOT rendered via - // the template's native "<|start|>tool <|message|>...<|eot|>" branch -- - // ChatTemplateAdapter's fallback rewrites it into a synthetic user message - // carrying a JSON blob first. - std::string expectedOutput = - R"(<|start|>system<|message|>You are a helpful assistant.<|eot|>)" - R"(<|start|>user<|message|>What's the weather in Paris?<|eot|>)" - R"(<|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|>)" - "<|start|>user<|message|>{\n" - " \"tool_response\": {\n" - " \"tool\": \"get_weather\",\n" - " \"content\": \"{\\\"temperature\\\":15,\\\"unit\\\":\\\"celsius\\\"}\"\n" - " }\n" - "}<|eot|>" - R"(<|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|>)" - R"(<|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); -} - // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. From ec5d871f58f63e91a9795710436b75ae52f00a33 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 14:18:04 +0200 Subject: [PATCH 20/38] rebase error fixed v3 --- .../chat_template_end_to_end_jinja_test.cpp | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 068b52111c..13c73a9d9f 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -712,52 +712,3 @@ What's the weather in Paris?<|im_end|> )"; EXPECT_EQ(appliedOutput, expectedOutput); } - -// Onyx (early preview model) chat template, rendered via the real Python Jinja2 -// engine. Onyx's template does not read the standard OpenAI "tool_calls" array -// at all -- only message['content'] (plain string) and an Onyx-specific -// message['recipient'] field (e.g. "functions.get_weather", "self", "user"). -// ChatTemplateAnalyzer now recognizes Onyx's control tokens (see analyzer.cpp), -// but deliberately leaves caps.supportsToolCalls false: that flag means "this -// template natively re-serializes an incoming OpenAI tool_calls array", which -// this test demonstrates Onyx's template does NOT do (the tool call is silently -// dropped below). -// ============================================================================= -TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithStringArgs) { - chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); - EXPECT_EQ(analysisResult.detectedToolParser.value(), "onyx"); - ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); - EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "onyx"); - - EXPECT_FALSE(caps.supportsToolCalls); - EXPECT_FALSE(caps.requiresObjectArguments); - - // Unlike the minja path (which has its own generic tool-call fallback), - // the real Python Jinja2 engine has no such fallback: the template renders - // message['content'] literally, i.e. the empty string, and the tool call - // information is silently dropped. - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant<|message|><|eot|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); -} - -// ============================================================================= -// Onyx's own message shape via Python Jinja2: "recipient" field instead of the -// OpenAI "tool_calls" array. Ends the turn with "<|eom|>" (continuation marker) -// rather than "<|eot|>". -// ============================================================================= -TEST_F(ChatTemplateEndToEndJinjaTest, Onyx_ToolCallWithRecipientField) { - chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_onyx.jinja"); - ASSERT_FALSE(chatTemplate.empty()); - - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"user","content":"What's the weather in Paris?"})")); - chatHistory.push_back(ov::genai::JsonContainer::from_json_string( - R"({"role":"assistant","recipient":"functions.get_weather","content":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"})")); - - run(); - - ASSERT_FALSE(exceptionThrownDuringApplication); - - std::string expectedOutput = R"(<|start|>system<|message|>You are a helpful assistant.<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>assistant)"; - EXPECT_EQ(appliedOutput, expectedOutput); -} From 81d9577176e92ddba3c95a4921b453fb7accde38 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 14:22:42 +0200 Subject: [PATCH 21/38] remove patch application --- windows_install_build_dependencies.bat | 6 ------ 1 file changed, 6 deletions(-) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index dc51bfd32e..c02afd1e86 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -256,10 +256,6 @@ cd %BAZEL_SHORT_PATH%\openvino_src git fetch origin git checkout %OV_SOURCE_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! -for %%P in (0001 0002 0003 0004 0005) do ( - git apply "%BACK_CWD%\patches\openvino\%%P-openvino.patch" - if !errorlevel! neq 0 exit /b !errorlevel! -) git submodule update --init --recursive if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules @@ -308,8 +304,6 @@ cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin git checkout %OV_GENAI_BRANCH% if !errorlevel! neq 0 exit /b !errorlevel! -git apply "%BACK_CWD%\patches\openvino.genai\0001-openvino.genai.patch" -if !errorlevel! neq 0 exit /b !errorlevel! git pull --recurse-submodules IF /I NOT EXIST build ( mkdir build From 4174ad955578a7e6dbbf3b0f5932103b60cc93bc Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 3 Aug 2026 14:24:57 +0200 Subject: [PATCH 22/38] rebase error fix v4 --- src/llm/io_processing/parser_config_validation.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index ae96a800bc..1baafc550f 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -34,7 +34,6 @@ const std::vector& getSupportedToolParserNames() { "gemma4", "onyx", "minicpm5", - "onyx", }; return names; } @@ -47,7 +46,6 @@ const std::vector& getSupportedReasoningParserNames() { "onyx", "minicpm5", "lfm2", - "onyx", }; return names; } From ca285d6fb8e58cef77bf8abdd70f0d7fa6685e67 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 4 Aug 2026 07:13:35 +0200 Subject: [PATCH 23/38] rebase error fix v5 --- src/llm/io_processing/output_parser.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 6a85001ab2..3dbef13d6e 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -228,9 +228,6 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); - } else if (reasoningParserName == "onyx") { - reasoningParser = std::make_unique(tokenizer); - decodeWithSpecialTokens = true; } else if (reasoningParserName == "minicpm5") { reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { From 8500613318e6e95342accd84b2e4b1e8ff993c0d Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 4 Aug 2026 08:19:39 +0200 Subject: [PATCH 24/38] rebase error fix v6 --- src/llm/io_processing/output_parser.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 3dbef13d6e..8d39310533 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -39,8 +39,6 @@ #include "onyx/onyx_reasoning_parser.hpp" #include "minicpm5/minicpm5_tool_parser.hpp" #include "minicpm5/minicpm5_reasoning_parser.hpp" -#include "onyx/onyx_tool_parser.hpp" -#include "onyx/onyx_reasoning_parser.hpp" namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { From 4330f910610d5035f13513c462cc3aa5feca21cf Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 4 Aug 2026 08:41:52 +0200 Subject: [PATCH 25/38] update test --- src/test/llm/chat_template_analyzer_test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index e4cbe0018b..cb4e951e5d 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -71,9 +71,7 @@ TEST_F(ChatTemplateAnalyzerTest, detectsOnyx) { EXPECT_EQ(result.detectedToolParser.value(), "onyx"); ASSERT_TRUE(result.detectedReasoningParser.has_value()); EXPECT_EQ(result.detectedReasoningParser.value(), "onyx"); - // Onyx's template never reads the OpenAI "tool_calls" array, so unlike every - // other detected family, supportsToolCalls stays false -- see analyzer.cpp. - EXPECT_FALSE(result.caps.supportsToolCalls); + EXPECT_TRUE(result.caps.supportsToolCalls); } // --- Gemma4 --- From 88ae901e3f1e7f1ff18b3dece1c6b6367eccc644 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Tue, 4 Aug 2026 12:44:44 +0200 Subject: [PATCH 26/38] lookup tokens --- src/llm/io_processing/base_output_parser.hpp | 9 +++ .../io_processing/onyx/onyx_tool_parser.cpp | 6 ++ .../io_processing/onyx/onyx_tool_parser.hpp | 14 ++++ src/llm/io_processing/output_parser.cpp | 23 +++++-- .../onyx_output_parser_test.cpp | 66 ++++++++++++++++++- 5 files changed, 111 insertions(+), 7 deletions(-) diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 0b83ea3839..4064fcf1a3 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -134,5 +134,14 @@ class BaseOutputParser { static const std::vector emptyVector; return emptyVector; } + + // Get the vector of tags that should be erased from the content before parsing. + // This is useful for cleaning up the content from tags that are necessary for parsing + // but should not be present in the final output. + // This is temporary solution until we have a content parser for that kind of scenarios. + virtual const std::vector& getTagSequenceToErase() const { + static const std::vector emptyVector; + return emptyVector; + } }; } // namespace ovms diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 1dcd5c5f0e..d3141d312a 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -38,6 +38,12 @@ const std::string OnyxToolParser::FUNCTION_END_TAG = ""; const std::string OnyxToolParser::PARAMETER_NAME_TAG = ""; +const std::string OnyxToolParser::startAssistantTurnTagChunk1 = "<|start|>"; +const std::string OnyxToolParser::startAssistantTurnTagChunk2 = "assistant "; +const std::string OnyxToolParser::userRecipientTagChunk1 = "to"; +const std::string OnyxToolParser::userRecipientTagChunk2 = "=user"; +const std::string OnyxToolParser::messageTag = "<|message|>"; +const std::string OnyxToolParser::endOfToolTag = "<|eot|>"; // Static empty map the default-constructed impl binds its const-ref member to (used by the // direct-impl unit tests, which pass plain-string values that need no schema typing). diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index 3157449caf..0f6c2a3cff 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -132,6 +132,12 @@ class OnyxToolParser : public BaseOutputParser { static const std::string PARAMETER_NAME_TAG; // "" static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name + static const std::string startAssistantTurnTagChunk1; // "<|start|>" + static const std::string startAssistantTurnTagChunk2; // "assistant" + static const std::string userRecipientTagChunk1; // " to" + static const std::string userRecipientTagChunk2; // "=user" + static const std::string messageTag; // "<|message|>" + static const std::string endOfToolTag; // "<|eot|>" private: const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) @@ -166,6 +172,14 @@ class OnyxToolParser : public BaseOutputParser { bool requiresStreamingWithSpecialTokens() const override { return true; } + const std::vector& getTagSequenceToErase() const override { + static const std::vector tagSequenceToErase{startAssistantTurnTagChunk1, startAssistantTurnTagChunk2, userRecipientTagChunk1, userRecipientTagChunk2, messageTag}; + return tagSequenceToErase; + } + const std::vector& getSpecialTagsToErase() const override { + static const std::vector specialTagsToErase{endOfToolTag}; + return specialTagsToErase; + } }; } // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 8d39310533..90dd6b7f7c 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -116,16 +116,27 @@ const std::string& OutputParser::StreamOutputCache::getBuffer() const { return buffer; } +static void eraseTagsFromContent(std::string& content, const std::vector& tags) { + for (const auto& tag : tags) { + size_t pos = 0; + while ((pos = content.find(tag, pos)) != std::string::npos) { + content.erase(pos, tag.length()); + } + } +} + std::optional OutputParser::parseContentChunk(ProcessingPhase newPhase) { std::string chunkContent = streamOutputCache.getBuffer(); if (toolParser != nullptr) { + auto secquenceToErase = toolParser->getTagSequenceToErase(); + auto lookupResult = streamOutputCache.lookupTags(secquenceToErase); + if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { + eraseTagsFromContent(chunkContent, secquenceToErase); + } else if (lookupResult == TagLookupStatus::FOUND_INCOMPLETE) { + return std::nullopt; + } auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); - for (const auto& tag : specialTagsToErase) { - size_t pos = 0; - while ((pos = chunkContent.find(tag, pos)) != std::string::npos) { - chunkContent.erase(pos, tag.length()); - } - } + eraseTagsFromContent(chunkContent, specialTagsToErase); } if (chunkContent.empty() || chunkContent == "") { diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index ae109d51a6..cac8fb0b68 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -434,8 +434,15 @@ if __name__ == "__main__": int i = -1; std::vector>> chunkToDeltaVec{ // Content before any tool call -- OutputParser sees no start tag match, emits content. + {"<|start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"assistant ", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"JUST_SOME_STRING_BEFORE_SPECIAL_STARTING_TAG"}})"}, - // ATEM start tag "" split across several arbitrarily small chunks. + {"to=", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"get_weather", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"\n", ov::genai::GenerationFinishReason::NONE, std::nullopt}, // "" split mid-tag and mid-name -- name delta emitted @@ -645,6 +652,63 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { } } +// ============================================================================= +// Streaming reasoning followed by content (final answer). Verifies that: +// - reasoning_content deltas are emitted for reasoning body chunks +// - framing tags (to=self, <|message|>, <|eom|>) are swallowed (nullopt) +// - after reasoning ends, the content streams normally as content deltas +// - no reasoning leaks into content deltas +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingReasoningThenContent) { + int i = -1; + std::vector>> chunkToDeltaVec{ + // Reasoning start tag -- swallowed. + {"to=self", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // <|message|> separator -- swallowed. + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Reasoning body chunks -- emitted as reasoning_content deltas. + {"Let me think", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"Let me think"}})"}, + {" carefully.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" carefully."}})"}, + // Reasoning end tag -- swallowed. + {"<|eom|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Content envelope -- swallowed (harmony framing before the actual answer). + {"<|start|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"assistant ", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"to", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + // Content body chunks -- emitted as content deltas. + {"The weather in", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"The weather in"}})"}, + {" Paris is sunny.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Paris is sunny."}})"}, + {"<|eot|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() : "nullopt"); + } + } +} + // ============================================================================= // Proves the "unary is an edge case of streaming" property holds structurally, not // just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used From c2d3892e0abe546c2482e03799587557d5ba4780 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 5 Aug 2026 10:49:01 +0200 Subject: [PATCH 27/38] flow simplification --- src/llm/io_processing/onyx/onyx_tool_parser.cpp | 8 ++------ src/llm/io_processing/onyx/onyx_tool_parser.hpp | 14 +++----------- src/llm/io_processing/output_parser.cpp | 6 ++---- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index d3141d312a..b049886552 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -38,12 +38,8 @@ const std::string OnyxToolParser::FUNCTION_END_TAG = ""; const std::string OnyxToolParser::PARAMETER_NAME_TAG = ""; -const std::string OnyxToolParser::startAssistantTurnTagChunk1 = "<|start|>"; -const std::string OnyxToolParser::startAssistantTurnTagChunk2 = "assistant "; -const std::string OnyxToolParser::userRecipientTagChunk1 = "to"; -const std::string OnyxToolParser::userRecipientTagChunk2 = "=user"; -const std::string OnyxToolParser::messageTag = "<|message|>"; -const std::string OnyxToolParser::endOfToolTag = "<|eot|>"; +const std::string OnyxToolParser::CONTENT_START_INDICATOR = "<|start|>assistant to=user<|message|>"; +const std::string OnyxToolParser::END_OF_TURN_TAG = "<|eot|>"; // Static empty map the default-constructed impl binds its const-ref member to (used by the // direct-impl unit tests, which pass plain-string values that need no schema typing). diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index 0f6c2a3cff..fd83c755e6 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -132,12 +132,8 @@ class OnyxToolParser : public BaseOutputParser { static const std::string PARAMETER_NAME_TAG; // "" static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name - static const std::string startAssistantTurnTagChunk1; // "<|start|>" - static const std::string startAssistantTurnTagChunk2; // "assistant" - static const std::string userRecipientTagChunk1; // " to" - static const std::string userRecipientTagChunk2; // "=user" - static const std::string messageTag; // "<|message|>" - static const std::string endOfToolTag; // "<|eot|>" + static const std::string CONTENT_START_INDICATOR; // "<|start|>assistant to=user<|message|>" + static const std::string END_OF_TURN_TAG; // "<|eot|>" private: const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) @@ -172,12 +168,8 @@ class OnyxToolParser : public BaseOutputParser { bool requiresStreamingWithSpecialTokens() const override { return true; } - const std::vector& getTagSequenceToErase() const override { - static const std::vector tagSequenceToErase{startAssistantTurnTagChunk1, startAssistantTurnTagChunk2, userRecipientTagChunk1, userRecipientTagChunk2, messageTag}; - return tagSequenceToErase; - } const std::vector& getSpecialTagsToErase() const override { - static const std::vector specialTagsToErase{endOfToolTag}; + static const std::vector specialTagsToErase{CONTENT_START_INDICATOR, END_OF_TURN_TAG}; return specialTagsToErase; } }; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 90dd6b7f7c..b0ba5f129e 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -128,15 +128,13 @@ static void eraseTagsFromContent(std::string& content, const std::vector OutputParser::parseContentChunk(ProcessingPhase newPhase) { std::string chunkContent = streamOutputCache.getBuffer(); if (toolParser != nullptr) { - auto secquenceToErase = toolParser->getTagSequenceToErase(); + auto secquenceToErase = toolParser->getSpecialTagsToErase(); auto lookupResult = streamOutputCache.lookupTags(secquenceToErase); if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { eraseTagsFromContent(chunkContent, secquenceToErase); } else if (lookupResult == TagLookupStatus::FOUND_INCOMPLETE) { return std::nullopt; - } - auto& specialTagsToErase = toolParser->getSpecialTagsToErase(); - eraseTagsFromContent(chunkContent, specialTagsToErase); + } } if (chunkContent.empty() || chunkContent == "") { From 72bfdec4a3a1b8552db187bb3ba8ac7ec707fb97 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 5 Aug 2026 14:21:23 +0200 Subject: [PATCH 28/38] remove <|start|>assistant from content --- .../io_processing/onyx/onyx_tool_parser.cpp | 3 +- .../io_processing/onyx/onyx_tool_parser.hpp | 5 ++- .../onyx_output_parser_test.cpp | 45 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index b049886552..59b496c27c 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -38,7 +38,8 @@ const std::string OnyxToolParser::FUNCTION_END_TAG = ""; const std::string OnyxToolParser::PARAMETER_NAME_TAG = ""; -const std::string OnyxToolParser::CONTENT_START_INDICATOR = "<|start|>assistant to=user<|message|>"; +const std::string OnyxToolParser::ASSISTANT_PREFIX = "<|start|>assistant "; +const std::string OnyxToolParser::CONTENT_START_INDICATOR = "to=user<|message|>"; const std::string OnyxToolParser::END_OF_TURN_TAG = "<|eot|>"; // Static empty map the default-constructed impl binds its const-ref member to (used by the diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index fd83c755e6..4ce05311d8 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -132,7 +132,8 @@ class OnyxToolParser : public BaseOutputParser { static const std::string PARAMETER_NAME_TAG; // "" static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name - static const std::string CONTENT_START_INDICATOR; // "<|start|>assistant to=user<|message|>" + static const std::string ASSISTANT_PREFIX; // "<|start|>assistant " + static const std::string CONTENT_START_INDICATOR; // "to=...<|message|>" static const std::string END_OF_TURN_TAG; // "<|eot|>" private: @@ -169,7 +170,7 @@ class OnyxToolParser : public BaseOutputParser { return true; } const std::vector& getSpecialTagsToErase() const override { - static const std::vector specialTagsToErase{CONTENT_START_INDICATOR, END_OF_TURN_TAG}; + static const std::vector specialTagsToErase{ASSISTANT_PREFIX, CONTENT_START_INDICATOR, END_OF_TURN_TAG}; return specialTagsToErase; } }; diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index cac8fb0b68..c67e7a2941 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -709,6 +709,51 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenContent) { } } +TEST_F(OnyxOutputParserTest, StreamingContentOnly) { + int i = -1; + std::vector>> chunkToDeltaVec{ + {"to=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Your ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Your "}})"}, + {"tweet", ov::genai::GenerationFinishReason::NONE, std::nullopt}, // it starts with t, that means this chunk overlaps with the to=user + {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet has"}})"}, + {" been", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" been"}})"}, + {" posted.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" posted."}})"}, + {" Let me know", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Let me know"}})"}, + {" if you need", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" if you need"}})"}, + {" I can do", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" I can do"}})"}, + {" anything ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" anything "}})"}, + {"to ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"to "}})"}, + {"help.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"help."}})"}, + {"<|eot|>", ov::genai::GenerationFinishReason::STOP, std::nullopt}, + }; + + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + i++; + std::optional doc = outputParser->parseChunk(chunk, {}, /*toolsAvailable=*/true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + EXPECT_EQ(docStr, expected) << "Mismatch for chunk[" << i << "]: " << chunk; + } else { + EXPECT_TRUE(false) << "Mismatch for chunk[" << i << "]: " << chunk + << "\nexpectedDelta: " << (expectedDelta.has_value() ? expectedDelta.value() : "nullopt") + << "\nGot doc: " << (doc.has_value() ? [&]() { + rapidjson::StringBuffer b; + rapidjson::Writer w(b); + doc->Accept(w); + return std::string(b.GetString()); + }() : "nullopt"); + } + } +} + // ============================================================================= // Proves the "unary is an edge case of streaming" property holds structurally, not // just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used From 6057e68ee09596b1c52e990a8e5cf140c313b610 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Wed, 5 Aug 2026 16:26:15 +0200 Subject: [PATCH 29/38] potential fix for to=<|message|> content --- .../io_processing/onyx/onyx_tool_parser.cpp | 13 ++++-- .../io_processing/onyx/onyx_tool_parser.hpp | 10 ++++- src/llm/io_processing/output_parser.cpp | 8 ++-- src/stringutils.cpp | 4 +- src/stringutils.hpp | 2 +- .../onyx_output_parser_test.cpp | 41 ++++++++++++++++++- 6 files changed, 66 insertions(+), 12 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 59b496c27c..3a5b9568eb 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -256,9 +256,16 @@ OnyxToolParser::OnyxToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchem BaseOutputParser(tokenizer), toolSchemas(toolSchemas), streamParser(this->toolsParametersTypes) { - // Build dynamic start tags: "" (the ATEM block itself) plus - // "to=" for each tool in the schema — so the streaming framework can detect the - // harmony envelope prefix and route to this parser instead of leaking it as content. +} + +void OnyxToolParser::lazyFillParsingStartTags() const { + // toolSchemas is a reference that is empty when this object is constructed and only gets + // populated by the caller afterwards, once the current request's tools are known (and may + // hold a different tool set on every request if this parser instance is reused). Rebuild + // "to=" start tags from whatever toolSchemas currently holds on every call instead of + // hardcoding tool names or building the list once too early in the constructor. The schema + // map is small, so recomputing this each time is cheap. + parsingStartTags.clear(); parsingStartTags.push_back(TOOL_START_TAG); for (const auto& [name, _] : toolSchemas) { parsingStartTags.push_back("to=" + name); diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index 4ce05311d8..5b69eef897 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -144,11 +144,18 @@ class OnyxToolParser : public BaseOutputParser { int toolCallIndex{-1}; std::set returnedFirstDeltas; std::set returnedCompleteDeltas; - std::vector parsingStartTags; + // Mutable because it is lazily (re)built from toolSchemas inside the const getParsingStartTags() + // getter below. toolSchemas is a reference that is empty at construction time and only filled in + // by the caller afterwards (once the request's tools are known), so building this list once in the + // constructor would permanently miss every "to=" entry. Rebuilding it from scratch on every + // call keeps it in sync with whatever tools the current request declares, without hardcoding any + // tool name. + mutable std::vector parsingStartTags; std::optional sendFirstDeltaIfNeeded(const std::string& functionName); std::optional sendFullDelta(const ToolCalls_t& toolCalls); void lazyFillInitToolParametersTypesMap(); + void lazyFillParsingStartTags() const; public: OnyxToolParser() = delete; @@ -157,6 +164,7 @@ class OnyxToolParser : public BaseOutputParser { void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; const std::vector& getParsingStartTags() const override { + lazyFillParsingStartTags(); return parsingStartTags; } const std::vector& getSpecialParsingStartTags() const override { diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index b0ba5f129e..fb8634fe2b 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -45,13 +45,15 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s if (tag.empty()) { return TagLookupStatus::NOT_FOUND; } + // Require at least 2-char overlap to avoid false positives from single-character coincidences + static constexpr size_t MIN_OVERLAP = 2; if (tag.size() > buffer.size()) { /* If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (stringsOverlap(buffer, tag)) { + if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; @@ -66,7 +68,7 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s */ if (buffer.find(tag) != std::string::npos) { return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { + } else if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; @@ -81,7 +83,7 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s */ if (buffer == tag) { return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag)) { + } else if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; diff --git a/src/stringutils.cpp b/src/stringutils.cpp index 1f369902cc..33defa74d6 100644 --- a/src/stringutils.cpp +++ b/src/stringutils.cpp @@ -259,13 +259,13 @@ std::string toLower(const std::string& input) { return result; } -bool stringsOverlap(const std::string& lhs, const std::string& rhs) { +bool stringsOverlap(const std::string& lhs, const std::string& rhs, size_t minOverlap) { if (lhs.empty() && rhs.empty()) { return true; } size_t minLength = std::min(lhs.size(), rhs.size()); - for (size_t len = 1; len <= minLength; ++len) { + for (size_t len = minOverlap; len <= minLength; ++len) { if (lhs.compare(lhs.size() - len, len, rhs, 0, len) == 0) { return true; } diff --git a/src/stringutils.hpp b/src/stringutils.hpp index ac812702b9..569eb2c23a 100644 --- a/src/stringutils.hpp +++ b/src/stringutils.hpp @@ -127,7 +127,7 @@ bool isValidUtf8(const std::string& text); std::string toLower(const std::string& input); -bool stringsOverlap(const std::string& lhs, const std::string& rhs); +bool stringsOverlap(const std::string& lhs, const std::string& rhs, size_t minOverlap = 1); void escapeSpecialCharacters(std::string& text); diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index c67e7a2941..6e61b3f49b 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -715,8 +715,8 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { {"to=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"Your ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Your "}})"}, - {"tweet", ov::genai::GenerationFinishReason::NONE, std::nullopt}, // it starts with t, that means this chunk overlaps with the to=user - {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet has"}})"}, + {"tweet", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet"}})"}, // it starts with t, that means this chunk overlaps with the to=user + {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" has"}})"}, {" been", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" been"}})"}, {" posted.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" posted."}})"}, {" Let me know", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Let me know"}})"}, @@ -754,6 +754,43 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { } } +// ============================================================================= +// Regression test for an agentic-streaming bug: OutputParser/OnyxToolParser is often +// constructed BEFORE the request's tools are known (toolSchemas is a reference to a map +// that starts empty and is filled in by the caller afterwards -- see the constructor +// comment on OnyxToolParser::toolSchemas). getParsingStartTags() must lazily rebuild its +// "to=" tags from whatever toolSchemas holds at call time (not just what it held at +// construction time), otherwise the harmony envelope preceding a real tool call is never +// recognized as a start tag and leaks into content as raw " to=<|message|>" text. +// ============================================================================= +TEST_F(OnyxOutputParserTest, StreamingToolEnvelopeNotLeakedWhenSchemasFilledAfterConstruction) { + ToolsSchemas_t lateSchemas; // empty when OutputParser/OnyxToolParser are constructed + OutputParser parser(*opt125mTokenizer, "onyx", "onyx", lateSchemas); + + // Populate the SAME map object only now -- OnyxToolParser keeps a reference to it, so + // this mirrors production code filling request.toolNameSchemaMap after construction. + lateSchemas = toolsSchemas; + + // Harmony envelope for a tool call, split the way a real generation streams it. If + // getParsingStartTags() were still frozen at the empty set captured at construction + // time, none of these chunks would match a start tag and they would be flushed as content. + auto doc = parser.parseChunk(" to=get_weather", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(doc.has_value()) << "envelope prefix must not be flushed as content"; + + doc = parser.parseChunk("<|message|>", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(doc.has_value()) << "\"<|message|>\" separator must not be flushed as content"; + + doc = parser.parseChunk("\n\n", {}, /*toolsAvailable=*/true, ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(doc.has_value()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + EXPECT_NE(docStr.find(R"("tool_calls")"), std::string::npos) << docStr; + EXPECT_NE(docStr.find(R"("name":"get_weather")"), std::string::npos) << docStr; + EXPECT_EQ(docStr.find(R"("content")"), std::string::npos) << "envelope leaked into content: " << docStr; +} + // ============================================================================= // Proves the "unary is an edge case of streaming" property holds structurally, not // just by coincidence: OnyxToolParser::parse() drives the same OnyxToolParserImpl used From bb0a408d82a043e317ca43ea605674a12b6a52ce Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Thu, 6 Aug 2026 14:31:46 +0200 Subject: [PATCH 30/38] copilot's review --- src/llm/io_processing/base_output_parser.hpp | 9 --------- src/llm/io_processing/output_parser.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 4064fcf1a3..0b83ea3839 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -134,14 +134,5 @@ class BaseOutputParser { static const std::vector emptyVector; return emptyVector; } - - // Get the vector of tags that should be erased from the content before parsing. - // This is useful for cleaning up the content from tags that are necessary for parsing - // but should not be present in the final output. - // This is temporary solution until we have a content parser for that kind of scenarios. - virtual const std::vector& getTagSequenceToErase() const { - static const std::vector emptyVector; - return emptyVector; - } }; } // namespace ovms diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index fb8634fe2b..5ebb689a5b 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -53,7 +53,7 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { + if (stringsOverlap(buffer, tag, std::min(buffer.size(), MIN_OVERLAP))) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; @@ -130,10 +130,10 @@ static void eraseTagsFromContent(std::string& content, const std::vector OutputParser::parseContentChunk(ProcessingPhase newPhase) { std::string chunkContent = streamOutputCache.getBuffer(); if (toolParser != nullptr) { - auto secquenceToErase = toolParser->getSpecialTagsToErase(); - auto lookupResult = streamOutputCache.lookupTags(secquenceToErase); + auto tagsToErase = toolParser->getSpecialTagsToErase(); + auto lookupResult = streamOutputCache.lookupTags(tagsToErase); if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { - eraseTagsFromContent(chunkContent, secquenceToErase); + eraseTagsFromContent(chunkContent, tagsToErase); } else if (lookupResult == TagLookupStatus::FOUND_INCOMPLETE) { return std::nullopt; } From 39ea1b7fd55734933c15c2ebaabdeb380be27b69 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 10 Aug 2026 14:27:39 +0200 Subject: [PATCH 31/38] outdated comments, code review, revert in content normalization processor --- .../text_content_normalization_processor.cpp | 13 ------ .../text_content_normalization_processor.hpp | 14 +++---- .../onyx/onyx_reasoning_parser.cpp | 4 -- .../onyx/onyx_reasoning_parser.hpp | 16 -------- .../io_processing/onyx/onyx_tool_parser.cpp | 15 ++----- src/llm/io_processing/output_parser.cpp | 4 +- ...emplate_and_parser_onyx_roundtrip_test.cpp | 10 +---- .../onyx_output_parser_test.cpp | 41 ------------------- 8 files changed, 12 insertions(+), 105 deletions(-) diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 871a0f5f06..2180d5faae 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -29,19 +29,6 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { ov::genai::ChatHistory& chatHistory = std::get(req.input); for (size_t i = 0; i < chatHistory.size(); i++) { const auto content = chatHistory[i]["content"]; - if (content.is_null()) { - // TODO @atobiszei to check if really needed when we have IR - // Standard OpenAI shape for e.g. an assistant message that only carries - // "tool_calls" sets "content": null (openai_completions.cpp stores this - // verbatim -- only a *missing* content field is defaulted to ""). Some - // chat templates (e.g. Onyx's) unconditionally render content for every - // message regardless of role/tool_calls and are not written to expect - // null there, which raises a template error instead of just omitting - // the text. Normalize null the same way a missing field is already - // defaulted, so every template sees a plain string as before. - chatHistory[i]["content"] = std::string(""); - continue; - } if (!content.is_array()) { continue; } diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index b6292cb302..e68d892490 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,15 +19,11 @@ namespace ovms { -// Flattens text-only content arrays in ChatHistory messages to plain strings, and -// normalizes an explicit "content": null (the standard OpenAI shape for e.g. an -// assistant message that only carries tool_calls) to "" -- some chat templates -// (e.g. Onyx's) unconditionally render content for every message and are not -// written to expect null there. Parts/null are joined/replaced for backward -// compatibility with chat templates. Runs for both LM and VLM chat paths: arrays -// that contain images (or other non-text modalities) are left untouched for -// ImageDecodingProcessor. -// Must run before ChatTemplateProcessor. +// Flattens text-only content arrays in ChatHistory messages to plain strings. +// Parts are joined with "\n" for backward compatibility with chat templates. +// Runs for both LM and VLM chat paths: arrays that contain images (or other +// non-text modalities) are left untouched. +// Must run before ChatTemplateProcessor and after Image/Audio decoding processors. class TextContentNormalizationProcessor : public BaseInputProcessor { public: absl::Status process(InputRequest& req) override; diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp index 4a7d7fad63..5291be615c 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.cpp @@ -26,10 +26,6 @@ namespace ovms { void OnyxReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { - // With EOS suppression the model may produce multiple interleaved turns in one - // generation (reasoning → tool call → reasoning → tool call → ... → answer). - // We must extract ALL reasoning segments and strip ALL turn boundaries/envelopes. - // Step 1: Extract and remove ALL "to=self<|message|>...<|eom|>" reasoning segments. for (;;) { size_t selfPos = parsedOutput.content.find(selfRecipientTag); diff --git a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp index 4dc54046d9..e7b7d415a2 100644 --- a/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_reasoning_parser.hpp @@ -27,22 +27,6 @@ namespace ovms { -// Onyx (early preview model) framing: -// TODO @atobiszei simplify comment. tag naming convention. no need to define all tags here -// <|start|>assistant[ to=]<|message|>{content}{<|eom|>|<|eot|>} -// The chat template never emits a ""-style dedicated reasoning tag: private -// chain-of-thought is just an assistant turn routed with recipient="self", ending in -// the continuation marker "<|eom|>" (never "<|eot|>", which is reserved for turns that -// end the whole assistant turn -- i.e. the final answer). -// -// Because generation stops at the first "<|eom|>"/"<|eot|>"/"<|end_of_text|>" (see -// generation_config.json's eos_token_id list in the Onyx HF conversion script), a single -// generate() call only ever produces ONE such framed segment. This parser is therefore -// also responsible for stripping the generic " to="+"<|message|>"+terminator -// envelope from plain final-answer turns (recipient="user" or absent) -- this class runs -// before the tool parser (see OutputParser::parse()), so it must NOT touch content when -// the envelope routes to a function call (recipient="functions."); it leaves that -// segment untouched so OnyxToolParser can find and parse it afterwards. class OnyxReasoningParser : public BaseOutputParser { protected: // Marks a private chain-of-thought turn (recipient="self"). diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 3a5b9568eb..1736eea0c0 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -52,11 +52,6 @@ OnyxToolParserImpl::OnyxToolParserImpl() : OnyxToolParserImpl::OnyxToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap) : toolsParametersTypeMap(toolsParametersTypeMap) {} -// parseToolSchema / createToolsParametersTypesMap now live in base_output_parser -// (shared with Qwen3CoderToolParser, Minicpm5ToolParser, ...); trimNewline, -// jsonTypeOf and enforceStringValue come from io_processing/utils. This parser -// reuses them instead of keeping its own copies. - void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameterValueAsString) { if (this->removeNewlineAroundParameters) trimNewline(parameterValueAsString); @@ -70,11 +65,7 @@ void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameter if (paramIt != this->toolsParametersTypeMap.end()) { auto paramJt = paramIt->second.find(currentParameterName); if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::BOOLEAN)) { - if (parameterValueAsString == "True" || parameterValueAsString == "TRUE") { - parameterValueAsString = "true"; - } else if (parameterValueAsString == "False" || parameterValueAsString == "FALSE") { - parameterValueAsString = "false"; - } + std::transform(parameterValueAsString.begin(), parameterValueAsString.end(), parameterValueAsString.begin(), ::tolower); } } temp.Parse(parameterValueAsString.c_str()); @@ -88,7 +79,7 @@ void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameter if (!currentFunctionArgsDoc.HasMember(keyVal)) { currentFunctionArgsDoc.AddMember(keyVal, v, allocator); } else { - SPDLOG_DEBUG("Parameter: {} already exists in document", key); + SPDLOG_TRACE("Parameter: {} already exists in document", key); } } else { rapidjson::Value valueCopy; @@ -103,7 +94,7 @@ void OnyxToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameter SPDLOG_TRACE("Will add key:{} val:{} type:{}", key, parameterValueAsString, jsonTypeOf(valueCopy)); currentFunctionArgsDoc.AddMember(keyVal, valueCopy, allocator); } else { - SPDLOG_DEBUG("Parameter: {} already exists in document.", key); + SPDLOG_TRACE("Parameter: {} already exists in document.", key); } } } diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 5ebb689a5b..a42a1ca26f 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -118,6 +118,8 @@ const std::string& OutputParser::StreamOutputCache::getBuffer() const { return buffer; } +// TODO: @przepeck We should consider moving this and +// similar workarounds to a content parser class static void eraseTagsFromContent(std::string& content, const std::vector& tags) { for (const auto& tag : tags) { size_t pos = 0; @@ -130,7 +132,7 @@ static void eraseTagsFromContent(std::string& content, const std::vector OutputParser::parseContentChunk(ProcessingPhase newPhase) { std::string chunkContent = streamOutputCache.getBuffer(); if (toolParser != nullptr) { - auto tagsToErase = toolParser->getSpecialTagsToErase(); + auto& tagsToErase = toolParser->getSpecialTagsToErase(); auto lookupResult = streamOutputCache.lookupTags(tagsToErase); if (lookupResult == TagLookupStatus::FOUND_COMPLETE) { eraseTagsFromContent(chunkContent, tagsToErase); diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp index 1f1f92c702..004bc6b97e 100644 --- a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -104,10 +104,7 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { } }; -// ============================================================================= -// Turn 1 of the muse/README.md "get_weather" example: user asks a question, the -// prompt is rendered, and the model's tool-call continuation is parsed. -// ============================================================================= + TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { ov::genai::ChatHistory chatHistory; chatHistory.push_back(ov::genai::JsonContainer::from_json_string( @@ -136,11 +133,6 @@ TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"SF"})"); } -// ============================================================================= -// Turn 2 of the same example: tool result fed back into history (Onyx's own -// "name" + role="tool" shape, NOT OpenAI's tool_call_id), then the model's final -// answer continuation is parsed. -// ============================================================================= TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinalAnswer) { ov::genai::ChatHistory chatHistory; chatHistory.push_back(ov::genai::JsonContainer::from_json_string( diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index 6e61b3f49b..b35d703e0f 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -33,47 +33,6 @@ using namespace ovms; -// ============================================================================= -// NEW Onyx tool-call format (ATEM), captured live from the running model -// (muse/onyx_live_withargs_1000_raw.txt, muse/onyx_live_nargs_1500_raw.txt) and -// matching `render_atem` in muse/onyx-ov-int4-v2/chat_template.jinja: -// -// to=<|message|> -// -// -// ... -// -// {<|eom|>|<|eot|>} -// -// Key differences from the previous Onyx drop (which these tests used to cover): -// - The recipient is the BARE tool name `to=get_weather`, NOT `to=functions.get_weather` -// (the "functions." prefix only appears if the tool itself is namespaced). The -// authoritative function name is therefore read from ``, -// not from the `to=` recipient. -// - Arguments are an ATEM XML block, essentially qwen3coder with -// `atem:` tags -- NOT a single raw JSON blob. Parameter values are rendered -// UNQUOTED (e.g. 37.7749,-122.4194), -// so arguments must be typed via the tool JSON schema exactly like -// Qwen3CoderToolParser (string->quoted, integer/number->numeric, -// bool/array/object->parsed, fall back to string). The toolsSchemas fixture below -// is therefore load-bearing now. -// -// Reasoning framing is UNCHANGED (" to=self<|message|>...<|eom|>") -- those tests are -// carried over verbatim. -// -// These tests define the TARGET contract; the parser implementation -// (src/llm/io_processing/onyx/*) is rewritten later against them. Until then the -// tool-call tests are expected to FAIL (the current parser still looks for the old -// "to=functions."/raw-JSON framing) while the reasoning tests still pass. -// ============================================================================= - -// Onyx does not ship a converted HF tokenizer in this early preview, and none of the -// segments the parser looks for ("", "", "<|eom|>", "<|eot|>") are real special tokens of the model this parser is -// designed for -- they are plain text sequences that must round-trip losslessly through -// encode()+decode() on ANY tokenizer. facebook/opt-125m is already used the same way for -// chat-template testing (see ChatTemplateEndToEndMinjaTest), so it is reused here to avoid -// pulling in a new model fixture. // TODO @atobiszei replace when tokenizer is available #ifdef _WIN32 const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\facebook\\opt-125m"; From 1ccaafd5f99944b8e55d641ebcf7bf3365d628a4 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 10 Aug 2026 14:36:32 +0200 Subject: [PATCH 32/38] revert build workarounds --- Dockerfile.ubuntu | 8 ++------ Makefile | 9 +-------- versions.mk | 6 +++--- windows_install_build_dependencies.bat | 6 +++--- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 40048f51db..fef0ed40e0 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -156,14 +156,10 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" -# onyx-support patches (temporary, one-off - see patches/*/readme.md for the -# commits they apply to) -COPY patches /patches/ - ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ ARG SDL_OPS="-Wl,-z,relro,-z,now,-z,noexecstack -fPIE -pie -fstack-protector-strong -fexceptions -fasynchronous-unwind-tables -fcf-protection -fpic -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fno-strict-overflow -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wuninitialized" # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git apply /patches/openvino/0001-openvino.patch && git apply /patches/openvino/0002-openvino.patch && git apply /patches/openvino/0003-openvino.patch && git apply /patches/openvino/0004-openvino.patch && git apply /patches/openvino/0005-openvino.patch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive WORKDIR /openvino/build RUN if [ "$ov_use_binary" == "0" ]; then \ if [[ $debug_bazel_flags == *"py_off"* ]]; then \ @@ -232,7 +228,7 @@ ARG ov_genai_org=openvinotoolkit WORKDIR /openvino_genai/ # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then \ - git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git apply /patches/openvino.genai/0001-openvino.genai.patch && git submodule update --init --recursive && \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive && \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} " -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ && \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ && \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ && \ diff --git a/Makefile b/Makefile index ff11a83ac4..2fcc40b417 100644 --- a/Makefile +++ b/Makefile @@ -133,14 +133,7 @@ endif ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" - # ubuntu24 defaults to building OpenVINO/GenAI from source (see versions.mk) - # so that the onyx-support patches in ./patches can be applied. Other ubuntu - # flavors keep using the prebuilt binary package by default. - ifeq ($(BASE_OS),ubuntu24) - OV_USE_BINARY ?= 0 - else - OV_USE_BINARY ?= 1 - endif + OV_USE_BINARY ?= 1 ifeq ($(findstring ubuntu22,$(BASE_OS)),ubuntu22) ifeq ($(OV_USE_BINARY),0) $(error OV_USE_BINARY = 0 not supported on Ubuntu22 OS) diff --git a/versions.mk b/versions.mk index c3df6eb564..d111954e0b 100644 --- a/versions.mk +++ b/versions.mk @@ -19,9 +19,9 @@ # Any variable can be overridden by the environment or command-line. # Source repository git commits / branches (used for source builds) -OV_SOURCE_BRANCH ?= muse_onyx -OV_TOKENIZERS_BRANCH ?= master -OV_GENAI_BRANCH ?= muse_onyx +OV_SOURCE_BRANCH ?= 894ddbcccf0755986b5a6bab807e7d9dd5cb8bb1 +OV_TOKENIZERS_BRANCH ?= 2eca683e943db7aa2ba784d19f7d62399bf3ae03 +OV_GENAI_BRANCH ?= fbbeb800ff5e6c6029b3747f87c1231a8b424d5e # Source repository organizations OV_SOURCE_ORG ?= openvinotoolkit diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index c02afd1e86..3ac79c2920 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -139,7 +139,7 @@ IF /I EXIST %bash_path% ( :: Set default OV_USE_BINARY if not set if "%OV_USE_BINARY%"=="" ( - set "OV_USE_BINARY=0" + set "OV_USE_BINARY=1" ) set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" @@ -248,7 +248,7 @@ IF /I EXIST %BAZEL_SHORT_PATH%\openvino_src ( ) IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( - git clone https://github.com/intel-sandbox/openvino.private %BAZEL_SHORT_PATH%\openvino_src + git clone https://github.com/%OV_SOURCE_ORG%/openvino %BAZEL_SHORT_PATH%\openvino_src ) set "BACK_CWD=%cd%" @@ -298,7 +298,7 @@ if !errorlevel! neq 0 exit /b !errorlevel! ::::::::::::::::::::::: OpenVINO GenAI IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( - git clone https://github.com/intel-sandbox/openvino.genai.private %BAZEL_SHORT_PATH%\openvino_genai_src + git clone https://github.com/%OV_GENAI_ORG%/openvino.genai.git %BAZEL_SHORT_PATH%\openvino_genai_src ) cd %BAZEL_SHORT_PATH%\openvino_genai_src git fetch origin From 6d75d64bf3f480e193420d6f8bc9f968f3145c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Rzepecki?= Date: Mon, 10 Aug 2026 13:55:58 +0100 Subject: [PATCH 33/38] remove outdated comment --- src/test/llm/output_parsers/onyx_output_parser_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index b35d703e0f..9c4c2da6e3 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -674,7 +674,7 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { {"to=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"Your ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Your "}})"}, - {"tweet", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet"}})"}, // it starts with t, that means this chunk overlaps with the to=user + {"tweet", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet"}})"}, {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" has"}})"}, {" been", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" been"}})"}, {" posted.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" posted."}})"}, From 355ab7ba7fee25457404f98af3f725961dffd325 Mon Sep 17 00:00:00 2001 From: Pawel Rzepecki Date: Mon, 10 Aug 2026 15:11:32 +0200 Subject: [PATCH 34/38] revert stringOverlap parameter, tests fixed --- src/llm/io_processing/output_parser.cpp | 8 +++----- src/stringutils.cpp | 4 ++-- src/stringutils.hpp | 2 +- ...t_content_normalization_processor_test.cpp | 20 ------------------- .../onyx_output_parser_test.cpp | 4 ++-- 5 files changed, 8 insertions(+), 30 deletions(-) diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index a42a1ca26f..4c80ab2b1a 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -45,15 +45,13 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s if (tag.empty()) { return TagLookupStatus::NOT_FOUND; } - // Require at least 2-char overlap to avoid false positives from single-character coincidences - static constexpr size_t MIN_OVERLAP = 2; if (tag.size() > buffer.size()) { /* If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE otherwise we return NOT_FOUND */ - if (stringsOverlap(buffer, tag, std::min(buffer.size(), MIN_OVERLAP))) { + if (stringsOverlap(buffer, tag)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; @@ -68,7 +66,7 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s */ if (buffer.find(tag) != std::string::npos) { return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { + } else if (stringsOverlap(buffer, tag)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; @@ -83,7 +81,7 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s */ if (buffer == tag) { return TagLookupStatus::FOUND_COMPLETE; - } else if (stringsOverlap(buffer, tag, MIN_OVERLAP)) { + } else if (stringsOverlap(buffer, tag)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; diff --git a/src/stringutils.cpp b/src/stringutils.cpp index 33defa74d6..1f369902cc 100644 --- a/src/stringutils.cpp +++ b/src/stringutils.cpp @@ -259,13 +259,13 @@ std::string toLower(const std::string& input) { return result; } -bool stringsOverlap(const std::string& lhs, const std::string& rhs, size_t minOverlap) { +bool stringsOverlap(const std::string& lhs, const std::string& rhs) { if (lhs.empty() && rhs.empty()) { return true; } size_t minLength = std::min(lhs.size(), rhs.size()); - for (size_t len = minOverlap; len <= minLength; ++len) { + for (size_t len = 1; len <= minLength; ++len) { if (lhs.compare(lhs.size() - len, len, rhs, 0, len) == 0) { return true; } diff --git a/src/stringutils.hpp b/src/stringutils.hpp index 569eb2c23a..ac812702b9 100644 --- a/src/stringutils.hpp +++ b/src/stringutils.hpp @@ -127,7 +127,7 @@ bool isValidUtf8(const std::string& text); std::string toLower(const std::string& input); -bool stringsOverlap(const std::string& lhs, const std::string& rhs, size_t minOverlap = 1); +bool stringsOverlap(const std::string& lhs, const std::string& rhs); void escapeSpecialCharacters(std::string& text); diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index 43ae0a6e1b..2b1840f0de 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -99,23 +99,3 @@ TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { ASSERT_TRUE(result[0]["content"].is_array()); EXPECT_EQ(result[0]["content"].size(), 2u); } - -TEST(TextContentNormalizationProcessorTest, NullContentNormalizedToEmptyString) { - // Standard OpenAI shape for e.g. an assistant message that only carries - // "tool_calls": content is explicitly null (not just absent). Some templates - // (e.g. Onyx's) unconditionally render content for every message and error out - // on null, so this must be normalized to "" the same way a missing field is. - ov::genai::ChatHistory history; - ov::AnyMap msg = {{"role", std::string("assistant")}}; - msg["content"] = ov::genai::JsonContainer(nullptr); - history.push_back(msg); - - InputRequest req = makeChatRequest(history); - TextContentNormalizationProcessor processor; - const auto status = processor.process(req); - - EXPECT_TRUE(status.ok()); - const auto& result = std::get(req.input); - ASSERT_TRUE(result[0]["content"].is_string()); - EXPECT_EQ(result[0]["content"].as_string().value_or("__unset__"), ""); -} diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index 9c4c2da6e3..28d0023306 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -674,8 +674,8 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { {"to=user", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"<|message|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, {"Your ", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Your "}})"}, - {"tweet", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet"}})"}, - {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" has"}})"}, + {"tweet", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {" has", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"tweet has"}})"}, {" been", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" been"}})"}, {" posted.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" posted."}})"}, {" Let me know", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" Let me know"}})"}, From 2c9c95dc899f3243cd456e4ccac7513bf8b8f929 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 10 Aug 2026 18:18:12 +0200 Subject: [PATCH 35/38] spelling --- spelling-whitelist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 81f083e9b5..175473be80 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -39,3 +39,5 @@ windows_parse_tests.bat:141: SEH ==> SHE windows_parse_tests.bat:144: SEH ==> SHE src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this +extras/chat_template_examples/chat_template_onyx.jinja +src/test/llm/chat_templates/chat_template_onyx.jinja From 889c7502e9747a33fe9d15b3d9449e1aaa0825d5 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 10 Aug 2026 19:11:32 +0200 Subject: [PATCH 36/38] style --- .../io_processing/onyx/onyx_tool_parser.hpp | 18 +++++++++--------- src/llm/io_processing/output_parser.hpp | 2 +- ...template_and_parser_onyx_roundtrip_test.cpp | 3 +-- .../chat_template_end_to_end_minja_test.cpp | 2 +- .../output_parsers/onyx_output_parser_test.cpp | 9 ++++++--- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.hpp b/src/llm/io_processing/onyx/onyx_tool_parser.hpp index 5b69eef897..26868b8e28 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.hpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.hpp @@ -125,16 +125,16 @@ struct OnyxToolParserImpl { class OnyxToolParser : public BaseOutputParser { public: - static const std::string TOOL_START_TAG; // "" - static const std::string TOOL_END_TAG; // "" - static const std::string FUNCTION_NAME_TAG; // "" - static const std::string PARAMETER_NAME_TAG; // "" - static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name - static const std::string ASSISTANT_PREFIX; // "<|start|>assistant " + static const std::string TOOL_START_TAG; // "" + static const std::string TOOL_END_TAG; // "" + static const std::string FUNCTION_NAME_TAG; // "" + static const std::string PARAMETER_NAME_TAG; // "" + static const std::string NAME_ATTR_END_TAG; // "\">" -- closes an invoke/parameter name + static const std::string ASSISTANT_PREFIX; // "<|start|>assistant " static const std::string CONTENT_START_INDICATOR; // "to=...<|message|>" - static const std::string END_OF_TURN_TAG; // "<|eot|>" + static const std::string END_OF_TURN_TAG; // "<|eot|>" private: const ToolsSchemas_t& toolSchemas; // filled outside; kept as reference (may change) diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index a6d0f7bbc5..991cd4902a 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -58,7 +58,7 @@ class OutputParser { ov::genai::Tokenizer tokenizer; std::unique_ptr toolParser = nullptr; // Tool parser for extracting tool calls std::unique_ptr reasoningParser = nullptr; // Reasoning parser for extracting reasoning content - bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) + bool decodeWithSpecialTokens = false; // Onyx parsers match on special token text (e.g. <|message|>, <|eom|>) // Streaming related members ProcessingPhase processingPhase = UNKNOWN; diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp index 004bc6b97e..f80391d6e8 100644 --- a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -70,7 +70,7 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { std::string rendered = tokenizer.apply_chat_template(chatHistory, /*add_generation_prompt=*/true); static const std::string generationPromptTail = "<|start|>assistant"; EXPECT_TRUE(rendered.size() >= generationPromptTail.size() && - rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) + rendered.compare(rendered.size() - generationPromptTail.size(), generationPromptTail.size(), generationPromptTail) == 0) << "Generation prompt tail changed, Onyx parser assumptions may be stale: " << rendered; return rendered; } @@ -104,7 +104,6 @@ class OnyxChatTemplateAndParserRoundtripTest : public ::testing::Test { } }; - TEST_F(OnyxChatTemplateAndParserRoundtripTest, UserQuestion_ModelEmitsToolCall) { ov::genai::ChatHistory chatHistory; chatHistory.push_back(ov::genai::JsonContainer::from_json_string( diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index 0157bcab0e..4c0cae3bcb 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -687,7 +687,7 @@ TEST_F(ChatTemplateEndToEndMinjaTest, Onyx_FullMultiTurnToolCallRoundTrip) { EXPECT_TRUE(caps.supportsToolCalls); std::string expectedOutput = -R"(# Valid recipients: "self", "user".<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>tool get_weather<|message|> + R"(# Valid recipients: "self", "user".<|eot|><|start|>user<|message|>What's the weather in Paris?<|eot|><|start|>assistant to=functions.get_weather<|message|>{"location":"Paris","unit":"celsius"}<|eom|><|start|>tool get_weather<|message|> {"temperature":15,"unit":"celsius"} <|eot|><|start|>assistant to=user<|message|>It's 15C in Paris.<|eot|><|start|>assistant)"; EXPECT_NE(appliedOutput.find(expectedOutput), std::string::npos) << appliedOutput; diff --git a/src/test/llm/output_parsers/onyx_output_parser_test.cpp b/src/test/llm/output_parsers/onyx_output_parser_test.cpp index 28d0023306..15f5cef99e 100644 --- a/src/test/llm/output_parsers/onyx_output_parser_test.cpp +++ b/src/test/llm/output_parsers/onyx_output_parser_test.cpp @@ -606,7 +606,8 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenToolCall) { rapidjson::Writer w(b); doc->Accept(w); return std::string(b.GetString()); - }() : "nullopt"); + }() + : "nullopt"); } } } @@ -663,7 +664,8 @@ TEST_F(OnyxOutputParserTest, StreamingReasoningThenContent) { rapidjson::Writer w(b); doc->Accept(w); return std::string(b.GetString()); - }() : "nullopt"); + }() + : "nullopt"); } } } @@ -708,7 +710,8 @@ TEST_F(OnyxOutputParserTest, StreamingContentOnly) { rapidjson::Writer w(b); doc->Accept(w); return std::string(b.GetString()); - }() : "nullopt"); + }() + : "nullopt"); } } } From 87cfc7e364a5c74d483df36482f5c54886fe9fa4 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 10 Aug 2026 19:52:50 +0200 Subject: [PATCH 37/38] style --- src/llm/io_processing/onyx/onyx_tool_parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llm/io_processing/onyx/onyx_tool_parser.cpp b/src/llm/io_processing/onyx/onyx_tool_parser.cpp index 1736eea0c0..cae334d5fc 100644 --- a/src/llm/io_processing/onyx/onyx_tool_parser.cpp +++ b/src/llm/io_processing/onyx/onyx_tool_parser.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "rapidjson/error/en.h" From 7921bd1a41d48655f67b8a16304a398ced8b22b0 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Tue, 11 Aug 2026 00:06:33 +0200 Subject: [PATCH 38/38] fix test for onyx parser --- ...emplate_and_parser_onyx_roundtrip_test.cpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp index f80391d6e8..0dfb17e44f 100644 --- a/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp +++ b/src/test/llm/chat_template_and_parser_onyx_roundtrip_test.cpp @@ -147,19 +147,12 @@ TEST_F(OnyxChatTemplateAndParserRoundtripTest, ToolResultFedBack_ModelEmitsFinal // Assistant tool-call history uses the recipient/content path (not `tool_calls`), which the // new template still renders via the plain "to=<|message|>...<|eom|>" envelope. EXPECT_NE(prompt.find(R"(<|start|>assistant to=functions.get_weather<|message|>{"city": "SF"}<|eom|>)"), std::string::npos) << prompt; - // NOTE (verified against the new template's actual minja render): OpenVINO GenAI STILL - // rewrites role="tool" into a role="user" message wrapping a "tool_response" JSON object - // (caps.supportsToolCalls == false), even though the new template DOES read `tools`/render - // tool defs -- so Onyx's own `elif role == 'tool'` template branch remains DEAD CODE on the - // minja path. What changed vs the previous drop: the wrapper no longer carries a "tool" - // field (only "content"). The tool output content appears with backslash-escaped quotes - // inside that JSON string, e.g.: - // <|start|>user<|message|>{ - // "tool_response": { - // "content": "{\"temp\": 65}" - // }<|eot|> - EXPECT_NE(prompt.find(R"("tool_response")"), std::string::npos) << prompt; - EXPECT_NE(prompt.find(R"({\"temp\": 65})"), std::string::npos) << prompt; + // NOTE: the Onyx template's `elif role == 'tool'` branch IS active on the minja path + // (caps.supportsToolCalls == true for this template), so the tool response is rendered as: + // <|start|>tool functions.get_weather<|message|> + // ...content...<|eot|> + EXPECT_NE(prompt.find(R"(<|start|>tool functions.get_weather<|message|>)"), std::string::npos) << prompt; + EXPECT_NE(prompt.find(R"({"temp": 65})"), std::string::npos) << prompt; std::string modelContinuation = R"( to=user<|message|>It's 65F in SF.<|eot|>)"; ParsedOutput parsedOutput = parseModelContinuation(modelContinuation);