diff --git a/CHANGELOG.md b/CHANGELOG.md index bc704e063..8a5554863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -230,6 +230,11 @@ All notable changes to SkillOpt are documented here. This project adheres to instead of treating them as missing, and load integral floats such as `1776.0` as `1776`, so a correct integer answer is no longer scored wrong (thanks @Shizoqua). +- Pass `--json-schema` instead of `--schema` in the `claude_chat` backend and + read the CLI's `structured_output` field. Structured calls (OfficeQA and + SpreadsheetBench rollouts) failed with `unknown option '--schema'`, and the + old `result` read would have returned the reply as raw JSON text with no + tool calls. ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/model/claude_backend.py b/skillopt/model/claude_backend.py index 373811af3..50ba91342 100644 --- a/skillopt/model/claude_backend.py +++ b/skillopt/model/claude_backend.py @@ -269,7 +269,7 @@ def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[ cmd.extend(["--effort", effort]) structured_output = bool(return_message) if structured_output: - cmd.extend(["--schema", _assistant_message_schema_wrapper()]) + cmd.extend(["--json-schema", _assistant_message_schema_wrapper()]) # Feed the prompt via stdin (and the system prompt via a file, above), not # argv: on Windows the whole command line is capped at ~32 KB and large # optimizer prompts / grown skills overflow it → [WinError 206]. Pin UTF-8 @@ -294,6 +294,15 @@ def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[ return raw_text, result_event or {}, usage_info +def _structured_payload(result_event: dict[str, Any]) -> Any: + # With --json-schema the CLI returns the validated object in + # `structured_output`; `result` only repeats it as JSON text. + structured = result_event.get("structured_output") + if isinstance(structured, dict): + return structured + return result_event.get("result", result_event) + + def _compat_message_from_payload(payload: Any) -> CompatAssistantMessage: if not isinstance(payload, dict): return CompatAssistantMessage(content=str(payload or ""), tool_calls=[]) @@ -316,7 +325,7 @@ def _call_messages(messages: list[dict[str, Any]], max_completion_tokens: int, r raw_text, payload, usage_info = _run_claude_print(system=system, prompt=prompt, model=model, tools=tools, tool_choice=tool_choice, return_message=return_message, timeout=timeout, attachments=attachments) tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) if return_message: - return _compat_message_from_payload(payload.get("result", payload)), usage_info + return _compat_message_from_payload(_structured_payload(payload)), usage_info return raw_text, usage_info except Exception as e: # noqa: BLE001 last_err = e diff --git a/tests/test_claude_backend_structured_output.py b/tests/test_claude_backend_structured_output.py new file mode 100644 index 000000000..ec98e8f90 --- /dev/null +++ b/tests/test_claude_backend_structured_output.py @@ -0,0 +1,93 @@ +"""The claude_chat structured-output path must match the Claude CLI's flags and result shape.""" +from __future__ import annotations + +import json +from types import SimpleNamespace + +from skillopt.model import claude_backend + +_TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up a value.", + "parameters": {"type": "object", "properties": {"key": {"type": "string"}}}, + }, + } +] +_MESSAGES = [ + {"role": "system", "content": "You answer questions."}, + {"role": "user", "content": "What year was it signed?"}, +] + + +def _install_fake_cli(monkeypatch, envelope: dict) -> dict: + captured: dict = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = list(cmd) + return SimpleNamespace(returncode=0, stdout=json.dumps(envelope) + "\n", stderr="") + + monkeypatch.setattr(claude_backend.subprocess, "run", fake_run) + return captured + + +def test_structured_call_passes_json_schema_and_reads_structured_output(monkeypatch) -> None: + # Shape of a Claude CLI 2.1.296 `-p --output-format json --json-schema ...` result: + # the validated object is in `structured_output`, and `result` holds the same JSON as text. + structured = { + "content": "Looking it up.", + "tool_calls": [{"name": "lookup", "arguments": '{"key": "year"}'}], + } + captured = _install_fake_cli( + monkeypatch, + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": json.dumps(structured), + "structured_output": structured, + "usage": {"input_tokens": 7, "output_tokens": 5}, + }, + ) + + message, usage = claude_backend.chat_target_messages( + _MESSAGES, tools=_TOOLS, return_message=True, retries=1 + ) + + cmd = captured["cmd"] + # The CLI rejects `--schema` with "error: unknown option '--schema'". + assert "--schema" not in cmd + schema = json.loads(cmd[cmd.index("--json-schema") + 1]) + assert schema == claude_backend._assistant_message_schema() + assert message.content == "Looking it up." + assert [(call.function.name, call.function.arguments) for call in message.tool_calls] == [ + ("lookup", '{"key": "year"}') + ] + assert usage == {"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 12} + + +def test_structured_call_without_structured_output_keeps_the_text_reply(monkeypatch) -> None: + _install_fake_cli( + monkeypatch, + {"type": "result", "subtype": "success", "result": "1776", "usage": {}}, + ) + + message, _ = claude_backend.chat_target_messages(_MESSAGES, return_message=True, retries=1) + + assert message.content == "1776" + assert message.tool_calls == [] + + +def test_plain_call_sends_no_schema(monkeypatch) -> None: + captured = _install_fake_cli( + monkeypatch, + {"type": "result", "subtype": "success", "result": "1776", "usage": {}}, + ) + + text, _ = claude_backend.chat_target("You answer questions.", "What year was it signed?", retries=1) + + assert text == "1776" + assert "--json-schema" not in captured["cmd"] + assert "--schema" not in captured["cmd"]