From 3514b7085c46058c2830df8908bb8ce07324d57c Mon Sep 17 00:00:00 2001 From: Jean-Noel Vouilloz Date: Wed, 19 Aug 2026 15:32:40 +0200 Subject: [PATCH] Don't assume hybrid-thinking by default in streaming chat completions Streaming chat completions defaulted `thinking_enabled` to True whenever a request didn't explicitly pass `chat_template_kwargs.enable_thinking` (the common case - clients like Cline never send it). That flag controls whether the server's ReasoningSplitter treats the stream as starting inside a block and holds text back as `reasoning_content` until a `` close tag shows up. For models that don't do hybrid step-by-step reasoning at all (e.g. Qwen3-Coder), that close tag never arrives, so the splitter never exits reasoning mode: every token of the real answer - plain content and tool calls alike - gets streamed out as `reasoning_content` for the whole response, while `content`/`tool_calls` stay empty. Reported symptom: Cline shows a perfectly sensible answer in its "thinking" panel, then errors with "Invalid API Response: the provider returned an empty or unparsable response" once the stream ends with nothing usable in the actual message. The non-streaming path already avoids this - it checks the completed text for "" before deciding whether to split reasoning out at all. Streaming can't do that (it has to decide before generation starts), so it must not guess reasoning-mode by default; only enable it when the request actually asked for it. Flipping that default exposed a second, previously-masked bug in QwenXmlToolCallParser: with thinking wrongly enabled, non-Qwen-XML `` bodies (e.g. Hermes-style raw JSON) used to get silently absorbed as reasoning_content and never reach the tool parser at all - a separate end-of-stream fallback that regexes the raw accumulated text for Hermes tool calls was quietly doing all the work. With reasoning no longer swallowing that text, it now reaches QwenXmlToolCallParser live, which only understands Qwen's own XML: fed raw JSON instead, it errored character-by-character but still emitted one bogus, half-empty tool-call-start fragment the moment it saw . That lone fragment was enough to flip `tool_call_sent = True` in the route, which skips the very fallback that used to rescue this case - turning a clean two-delta Hermes tool call into one garbage delta. Fixed by not registering a call (and not emitting its start fragment) the instant appears. QwenXmlToolCallParser now only registers a call once it actually confirms Qwen's syntax follows; a block in some other format never registers, so it can't surface as a bogus fragment - it's left for the accumulated-text Hermes fallback to handle, exactly as before. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01JvRzxr1m91BJhVbaz3FjM9 --- src/engine/ov_genai/qwen_tool_parser.py | 23 ++++++++++++++--------- src/server/routes/openai.py | 9 ++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/engine/ov_genai/qwen_tool_parser.py b/src/engine/ov_genai/qwen_tool_parser.py index 9169110..6c52378 100644 --- a/src/engine/ov_genai/qwen_tool_parser.py +++ b/src/engine/ov_genai/qwen_tool_parser.py @@ -221,7 +221,7 @@ def _step(self, content_out: list, fragments: list) -> bool: self.status = StreamingStatus.TOOL_CALL_STOP return True self._buf = buf[len(TOOL_OPEN):] - self._start_call(fragments) + self._start_call() self._state = IN_TOOL_CALL return True @@ -238,11 +238,7 @@ def _step(self, content_out: list, fragments: list) -> bool: return False name = buf[:gt].strip() self._buf = buf[gt + 1:] - self._cur["function"]["name"] = name - fragments.append({ - "index": len(self._calls) - 1, - "function": {"name": name}, - }) + self._register_call(name, fragments) self._state = IN_FUNCTION return True @@ -332,19 +328,28 @@ def _frag(self, fragments: list, arguments: str): }) self._cur["function"]["arguments"] += arguments - def _start_call(self, fragments: list): + def _start_call(self): + # Pending only: not registered in `_calls` (and no fragment emitted) + # until `_register_call` confirms this is really Qwen's + # syntax. A block using a different + # format (e.g. Hermes-style raw JSON) never registers, so it can't + # surface as a bogus half-empty tool call - it's left for the + # accumulated-text Hermes fallback in openai.py to pick up instead. self._cur = { "id": f"call_{next(self._ids):024x}", "type": "function", "function": {"name": "", "arguments": ""}, } - self._calls.append(self._cur) self._param_index = 0 + + def _register_call(self, name: str, fragments: list): + self._cur["function"]["name"] = name + self._calls.append(self._cur) fragments.append({ "index": len(self._calls) - 1, "id": self._cur["id"], "type": "function", - "function": {"name": "", "arguments": ""}, + "function": {"name": name, "arguments": ""}, }) def _close_call(self, fragments: list): diff --git a/src/server/routes/openai.py b/src/server/routes/openai.py index cd923dd..da01f86 100644 --- a/src/server/routes/openai.py +++ b/src/server/routes/openai.py @@ -292,11 +292,10 @@ async def openai_chat_completions( created_ts = int(time.time()) request_id = f"ov-{uuid.uuid4().hex[:24]}" - thinking_enabled = True - if chat_template_kwargs: - thinking_enabled = chat_template_kwargs.get( - "enable_thinking", True - ) + # Don't assume hybrid-thinking mode unless told to expect it (see PR + # description): models without a closing tag would otherwise + # have their entire streamed answer misclassified as reasoning_content. + thinking_enabled = bool(chat_template_kwargs.get("enable_thinking", False)) if generation_config.stream: