From 3185adabba370b4136fd63d7d3d50530fb5739a5 Mon Sep 17 00:00:00 2001 From: EmBista Date: Sun, 16 Aug 2026 23:47:53 +1000 Subject: [PATCH] Support structured outputs on the chat routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Responses API carries structured outputs at `text.format`, but neither chat-compat route ever built one. `response_format` (and Ollama's `format`) were read off the request and dropped. Upstream never saw a schema, answered 200, and the model returned whatever shape it liked — silently, since the reply is still valid JSON, just not the requested one. - map a json_schema `response_format` to `text.format`, accepting both the nested Chat Completions spelling and the flat Responses one - map Ollama's `format` when it holds a schema - fall back to `legacy` reasoning-compat when the caller asks for JSON, since think-tags mode prepends `…` to the very content that was asked to be JSON. `legacy` keeps content untouched and puts the reasoning in sibling string fields, the shape openai-compatible clients already read. `json_object`, and Ollama's bare "json", are deliberately not forwarded: upstream refuses that format unless the input mentions "json", so sending it would turn requests that pass today into 400s. They still get the compat fallback, so their content is JSON a client can actually parse — the override keys off what the caller asked for, not off what was forwarded. The fallback applies only when the server is in think-tags mode. The other compat modes already keep reasoning out of content, and forcing legacy over `--reasoning-compat o3` would change the type of `message.reasoning`. Requests that send neither field are unaffected: no `text` is added to the upstream payload and reasoning still rides in think tags. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 1 + chatmock/routes_ollama.py | 22 ++- chatmock/routes_openai.py | 13 ++ chatmock/transform.py | 17 ++ chatmock/upstream.py | 3 + chatmock/utils.py | 40 +++++ tests/test_routes.py | 319 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 412 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 94b78e6..232ffff 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ account. The current catalog commonly includes: ## Features - Tool / function calling +- Structured outputs via JSON schema (`response_format`, or `format` on the Ollama routes) - Vision / image input - Thinking summaries (via think tags) - Configurable thinking effort diff --git a/chatmock/routes_ollama.py b/chatmock/routes_ollama.py index 70981ee..258016e 100644 --- a/chatmock/routes_ollama.py +++ b/chatmock/routes_ollama.py @@ -16,7 +16,12 @@ build_reasoning_param, extract_reasoning_from_model_name, ) -from .transform import convert_ollama_messages, normalize_ollama_tools +from .transform import ( + convert_ollama_format_to_text_format, + convert_ollama_messages, + normalize_ollama_tools, + ollama_format_requests_json, +) from .upstream import normalize_model_name, start_upstream_request from .utils import convert_chat_messages_to_responses_input, convert_tools_chat_to_responses @@ -198,6 +203,15 @@ def ollama_chat() -> Response: tool_choice = payload.get("tool_choice", "auto") parallel_tool_calls = bool(payload.get("parallel_tool_calls", False)) + text_format = convert_ollama_format_to_text_format(payload.get("format")) + + if ( + ollama_format_requests_json(payload.get("format")) + and (reasoning_compat or "").strip().lower() == "think-tags" + ): + # think tags would be prepended to content the caller asked to be json + reasoning_compat = "legacy" + # Passthrough Responses API tools (web_search) via ChatMock extension fields extra_tools: List[Dict[str, Any]] = [] had_responses_tools = False @@ -271,6 +285,7 @@ def ollama_chat() -> Response: allowed_efforts=allowed_efforts_for_model(model), ), service_tier=service_tier_resolution.service_tier, + text_format=text_format, ) if error_resp is not None: if verbose: @@ -311,6 +326,7 @@ def ollama_chat() -> Response: allowed_efforts=allowed_efforts_for_model(model), ), service_tier=service_tier_resolution.service_tier, + text_format=text_format, ) record_rate_limits_from_response(upstream2) if err2 is None and upstream2 is not None and upstream2.status_code < 400: @@ -333,7 +349,7 @@ def ollama_chat() -> Response: if stream_req: def _gen(): - compat = (current_app.config.get("REASONING_COMPAT", "think-tags") or "think-tags").strip().lower() + compat = (reasoning_compat or "think-tags").strip().lower() think_open = False think_closed = False saw_any_summary = False @@ -551,7 +567,7 @@ def _gen(): finally: upstream.close() - if (current_app.config.get("REASONING_COMPAT", "think-tags") or "think-tags").strip().lower() == "think-tags": + if (reasoning_compat or "think-tags").strip().lower() == "think-tags": rtxt_parts = [] if isinstance(reasoning_summary_text, str) and reasoning_summary_text.strip(): rtxt_parts.append(reasoning_summary_text) diff --git a/chatmock/routes_openai.py b/chatmock/routes_openai.py index 673e22f..c2213d6 100644 --- a/chatmock/routes_openai.py +++ b/chatmock/routes_openai.py @@ -32,6 +32,8 @@ from .upstream import normalize_model_name, start_upstream_raw_request, start_upstream_request from .utils import ( convert_chat_messages_to_responses_input, + convert_response_format_to_text_format, + response_format_requests_json, convert_tools_chat_to_responses, sse_translate_chat, sse_translate_text, @@ -210,6 +212,15 @@ def chat_completions() -> Response: if tier_error is not None: return tier_error + text_format = convert_response_format_to_text_format(payload.get("response_format")) + + if ( + response_format_requests_json(payload.get("response_format")) + and (reasoning_compat or "").strip().lower() == "think-tags" + ): + # think tags would be prepended to content the caller asked to be json + reasoning_compat = "legacy" + upstream, error_resp = start_upstream_request( model, input_items, @@ -218,6 +229,7 @@ def chat_completions() -> Response: parallel_tool_calls=parallel_tool_calls, reasoning_param=reasoning_param, service_tier=service_tier, + text_format=text_format, ) if error_resp is not None: if verbose: @@ -255,6 +267,7 @@ def chat_completions() -> Response: parallel_tool_calls=parallel_tool_calls, reasoning_param=reasoning_param, service_tier=service_tier, + text_format=text_format, ) record_rate_limits_from_response(upstream2) if err2 is None and upstream2 is not None and upstream2.status_code < 400: diff --git a/chatmock/transform.py b/chatmock/transform.py index 7c611fb..525c1e6 100644 --- a/chatmock/transform.py +++ b/chatmock/transform.py @@ -147,3 +147,20 @@ def normalize_ollama_tools(tools: List[Dict[str, Any]] | None) -> List[Dict[str, ) return out + +def ollama_format_requests_json(fmt: Any) -> bool: + if isinstance(fmt, str): + return fmt.strip().lower() == "json" + return isinstance(fmt, dict) and bool(fmt) + + +def convert_ollama_format_to_text_format(fmt: Any) -> Dict[str, Any] | None: + """Map Ollama's format, when it holds a schema, onto a Responses text.format. + + The bare "json" string is left alone: upstream refuses json_object unless + the input mentions "json". + """ + + if not isinstance(fmt, dict) or not fmt: + return None + return {"type": "json_schema", "name": "response", "schema": fmt, "strict": False} diff --git a/chatmock/upstream.py b/chatmock/upstream.py index 9ad7941..38f7273 100644 --- a/chatmock/upstream.py +++ b/chatmock/upstream.py @@ -35,6 +35,7 @@ def start_upstream_request( parallel_tool_calls: bool = False, reasoning_param: Dict[str, Any] | None = None, service_tier: str | None = None, + text_format: Dict[str, Any] | None = None, ): access_token, account_id = get_effective_chatgpt_auth() if not access_token or not account_id: @@ -86,6 +87,8 @@ def start_upstream_request( responses_payload["reasoning"] = reasoning_param if isinstance(service_tier, str) and service_tier.strip(): responses_payload["service_tier"] = service_tier.strip().lower() + if isinstance(text_format, dict) and text_format: + responses_payload["text"] = {"format": text_format} return start_upstream_raw_request( responses_payload, diff --git a/chatmock/utils.py b/chatmock/utils.py index 96dd314..66eb2ca 100644 --- a/chatmock/utils.py +++ b/chatmock/utils.py @@ -247,6 +247,46 @@ def convert_tools_chat_to_responses(tools: Any) -> List[Dict[str, Any]]: return out +def convert_response_format_to_text_format(response_format: Any) -> Dict[str, Any] | None: + """Map a Chat Completions response_format onto a Responses text.format.""" + + if not isinstance(response_format, dict): + return None + + # json_object is deliberately not mapped: upstream rejects it unless the + # input mentions "json", so honouring it would 400 requests that pass today. + if response_format.get("type") != "json_schema": + return None + + spec = response_format.get("json_schema") + if not isinstance(spec, dict): + spec = response_format + schema = spec.get("schema") + if not isinstance(schema, dict) or not schema: + return None + + name = spec.get("name") + text_format: Dict[str, Any] = { + "type": "json_schema", + "name": name.strip() if isinstance(name, str) and name.strip() else "response", + "schema": schema, + "strict": bool(spec.get("strict")), + } + description = spec.get("description") + if isinstance(description, str) and description.strip(): + text_format["description"] = description + return text_format + + +def response_format_requests_json(response_format: Any) -> bool: + """Whether the caller asked for JSON content, mapped upstream or not.""" + + return isinstance(response_format, dict) and response_format.get("type") in ( + "json_schema", + "json_object", + ) + + def load_chatgpt_tokens( ensure_fresh: bool = True, *, diff --git a/tests/test_routes.py b/tests/test_routes.py index a490670..bbb950f 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -660,5 +660,324 @@ def close(self) -> None: ) +class ResponseFormatTests(unittest.TestCase): + SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["summary"], + "properties": {"summary": {"type": "string"}}, + } + + def setUp(self) -> None: + reset_session_state() + self.app = create_app(model_sync=False) + self.client = self.app.test_client() + + def _post(self, mock_start, **extra) -> dict: + mock_start.return_value = ( + FakeUpstream( + [ + {"type": "response.output_text.delta", "delta": "{}"}, + {"type": "response.completed", "response": {"id": "resp-fmt"}}, + ] + ), + None, + ) + response = self.client.post( + "/v1/chat/completions", + json={"model": "gpt-5.6-sol", "messages": [{"role": "user", "content": "hi"}], **extra}, + ) + self.assertEqual(response.status_code, 200) + return mock_start.call_args.kwargs + + + + + @patch("chatmock.routes_openai.start_upstream_request") + def test_formats_that_are_not_forwarded(self, mock_start) -> None: + self.assertIsNone(self._post(mock_start)["text_format"]) + self.assertIsNone(self._post(mock_start, response_format={"type": "text"})["text_format"]) + self.assertIsNone( + self._post(mock_start, response_format={"type": "json_object"})["text_format"] + ) + self.assertIsNone( + self._post( + mock_start, response_format={"type": "json_schema", "json_schema": {"schema": {}}} + )["text_format"] + ) + + + @patch("chatmock.routes_openai.start_upstream_request") + def test_json_content_is_not_corrupted_by_think_tags(self, mock_start) -> None: + events = [ + {"type": "response.reasoning_summary_text.delta", "delta": "**Weighing it up**"}, + {"type": "response.reasoning_text.delta", "delta": "the full trace"}, + {"type": "response.output_text.delta", "delta": '{"summary":"ok"}'}, + {"type": "response.completed", "response": {"id": "resp-fmt"}}, + ] + + def post(**extra): + mock_start.return_value = (FakeUpstream(list(events)), None) + return self.client.post( + "/v1/chat/completions", + json={ + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + **extra, + }, + ).get_json()["choices"][0]["message"] + + self.assertTrue(post()["content"].startswith("")) + + message = post( + response_format={ + "type": "json_schema", + "json_schema": {"name": "r", "schema": self.SCHEMA}, + } + ) + self.assertEqual(json.loads(message["content"]), {"summary": "ok"}) + self.assertEqual(message["reasoning_summary"], "**Weighing it up**") + self.assertEqual(message["reasoning"], "the full trace") + + @patch("chatmock.routes_openai.start_upstream_request") + def test_non_think_tags_compat_is_left_alone(self, mock_start) -> None: + client = create_app(model_sync=False, reasoning_compat="o3").test_client() + mock_start.return_value = ( + FakeUpstream( + [ + {"type": "response.reasoning_summary_text.delta", "delta": "**Weighing it up**"}, + {"type": "response.output_text.delta", "delta": '{"summary":"ok"}'}, + {"type": "response.completed", "response": {"id": "resp-fmt"}}, + ] + ), + None, + ) + message = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "r", "schema": self.SCHEMA}, + }, + }, + ).get_json()["choices"][0]["message"] + self.assertEqual( + message["reasoning"], {"content": [{"type": "text", "text": "**Weighing it up**"}]} + ) + self.assertEqual(json.loads(message["content"]), {"summary": "ok"}) + + def _post_ollama(self, mock_start, **extra) -> dict: + mock_start.return_value = ( + FakeUpstream( + [ + {"type": "response.reasoning_summary_text.delta", "delta": "**Weighing it up**"}, + {"type": "response.output_text.delta", "delta": '{"summary":"ok"}'}, + {"type": "response.completed", "response": {"id": "resp-fmt"}}, + ] + ), + None, + ) + response = self.client.post( + "/api/chat", + json={ + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + "stream": False, + **extra, + }, + ) + self.assertEqual(response.status_code, 200) + return response.get_json() + + + @patch("chatmock.routes_ollama.start_upstream_request") + def test_ollama_bare_json_is_uncorrupted_but_not_forwarded(self, mock_start) -> None: + body = self._post_ollama(mock_start, format="json") + self.assertIsNone(mock_start.call_args.kwargs["text_format"]) + self.assertEqual(json.loads(body["message"]["content"]), {"summary": "ok"}) + + + +class StructuredOutputWireTests(unittest.TestCase): + """Asserts past the start_upstream_request boundary, onto the actual payload. + + The route tests mock that call, so they pass whether or not the schema is + ever put on the wire. + """ + + SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["summary"], + "properties": {"summary": {"type": "string"}}, + } + + EVENTS = [ + {"type": "response.reasoning_summary_text.delta", "delta": "**T**"}, + {"type": "response.output_text.delta", "delta": '{"summary":"ok"}'}, + {"type": "response.completed", "response": {"id": "resp-wire"}}, + ] + + def setUp(self) -> None: + reset_session_state() + self.client = create_app(model_sync=False).test_client() + + def _send(self, mock_post, path: str, body: dict) -> tuple[dict, str]: + mock_post.return_value = FakeUpstream(list(self.EVENTS)) + response = self.client.post(path, json=body) + self.assertEqual(response.status_code, 200) + sent = mock_post.call_args.kwargs["json"] + return sent, response.get_data(as_text=True) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_schema_reaches_the_upstream_payload(self, mock_post, _auth) -> None: + sent, _ = self._send( + mock_post, + "/v1/chat/completions", + { + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": self.SCHEMA}, + }, + }, + ) + self.assertEqual( + sent["text"], + { + "format": { + "type": "json_schema", + "name": "weather", + "schema": self.SCHEMA, + "strict": False, + } + }, + ) + + flat, _ = self._send( + mock_post, + "/v1/chat/completions", + { + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + "response_format": { + "type": "json_schema", + "name": "weather", + "schema": self.SCHEMA, + "strict": True, + }, + }, + ) + self.assertEqual(flat["text"]["format"]["strict"], True) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_ollama_schema_reaches_the_upstream_payload(self, mock_post, _auth) -> None: + sent, _ = self._send( + mock_post, + "/api/chat", + { + "model": "gpt-5.6-sol", + "stream": False, + "messages": [{"role": "user", "content": "hi"}], + "format": self.SCHEMA, + }, + ) + self.assertEqual(sent["text"]["format"]["schema"], self.SCHEMA) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_no_text_key_without_a_format(self, mock_post, _auth) -> None: + sent, _ = self._send( + mock_post, + "/v1/chat/completions", + {"model": "gpt-5.6-sol", "messages": [{"role": "user", "content": "hi"}]}, + ) + self.assertNotIn("text", sent) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_streaming_content_is_uncorrupted(self, mock_post, _auth) -> None: + for path, body, extract in ( + ( + "/v1/chat/completions", + { + "model": "gpt-5.6-sol", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "r", "schema": self.SCHEMA}, + }, + }, + lambda line: json.loads(line[len("data: ") :])["choices"][0]["delta"].get("content"), + ), + ( + "/api/chat", + { + "model": "gpt-5.6-sol", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + "format": self.SCHEMA, + }, + lambda line: json.loads(line)["message"].get("content"), + ), + ): + with self.subTest(path=path): + _, raw = self._send(mock_post, path, body) + self.assertNotIn("", raw) + chunks = [] + for line in raw.splitlines(): + if not line.strip() or line.startswith("data: [DONE]"): + continue + try: + chunks.append(extract(line) or "") + except (ValueError, KeyError, IndexError): + continue + self.assertEqual(json.loads("".join(chunks)), {"summary": "ok"}) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_schema_survives_the_rejected_tools_retry(self, mock_post, _auth) -> None: + mock_post.side_effect = [ + FakeUpstream(status_code=400, content=b'{"error":{"message":"nope"}}'), + FakeUpstream(list(self.EVENTS)), + ] + response = self.client.post( + "/v1/chat/completions", + json={ + "model": "gpt-5.6-sol", + "messages": [{"role": "user", "content": "hi"}], + "responses_tools": [{"type": "web_search"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "r", "schema": self.SCHEMA}, + }, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(mock_post.call_count, 2) + retried = mock_post.call_args_list[1].kwargs["json"] + self.assertEqual(retried["text"]["format"]["schema"], self.SCHEMA) + + @patch("chatmock.upstream.get_effective_chatgpt_auth", return_value=("token", "acct")) + @patch("chatmock.upstream.requests.post") + def test_streaming_keeps_think_tags_without_a_format(self, mock_post, _auth) -> None: + _, raw = self._send( + mock_post, + "/v1/chat/completions", + { + "model": "gpt-5.6-sol", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + }, + ) + self.assertIn("", raw) + + if __name__ == "__main__": unittest.main()