Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,11 @@ All notable changes to SkillOpt are documented here. This project adheres to
instead of treating them as missing, and load integral floats such as
`1776.0` as `1776`, so a correct integer answer is no longer scored wrong
(thanks @Shizoqua).
- Pass `--json-schema` instead of `--schema` in the `claude_chat` backend and
read the CLI's `structured_output` field. Structured calls (OfficeQA and
SpreadsheetBench rollouts) failed with `unknown option '--schema'`, and the
old `result` read would have returned the reply as raw JSON text with no
tool calls.

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
13 changes: 11 additions & 2 deletions skillopt/model/claude_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[
cmd.extend(["--effort", effort])
structured_output = bool(return_message)
if structured_output:
cmd.extend(["--schema", _assistant_message_schema_wrapper()])
cmd.extend(["--json-schema", _assistant_message_schema_wrapper()])
# Feed the prompt via stdin (and the system prompt via a file, above), not
# argv: on Windows the whole command line is capped at ~32 KB and large
# optimizer prompts / grown skills overflow it → [WinError 206]. Pin UTF-8
Expand All @@ -294,6 +294,15 @@ def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[
return raw_text, result_event or {}, usage_info


def _structured_payload(result_event: dict[str, Any]) -> Any:
# With --json-schema the CLI returns the validated object in
# `structured_output`; `result` only repeats it as JSON text.
structured = result_event.get("structured_output")
if isinstance(structured, dict):
return structured
return result_event.get("result", result_event)


def _compat_message_from_payload(payload: Any) -> CompatAssistantMessage:
if not isinstance(payload, dict):
return CompatAssistantMessage(content=str(payload or ""), tool_calls=[])
Expand All @@ -316,7 +325,7 @@ def _call_messages(messages: list[dict[str, Any]], max_completion_tokens: int, r
raw_text, payload, usage_info = _run_claude_print(system=system, prompt=prompt, model=model, tools=tools, tool_choice=tool_choice, return_message=return_message, timeout=timeout, attachments=attachments)
tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"])
if return_message:
return _compat_message_from_payload(payload.get("result", payload)), usage_info
return _compat_message_from_payload(_structured_payload(payload)), usage_info
return raw_text, usage_info
except Exception as e: # noqa: BLE001
last_err = e
Expand Down
93 changes: 93 additions & 0 deletions tests/test_claude_backend_structured_output.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
"""The claude_chat structured-output path must match the Claude CLI's flags and result shape."""
from __future__ import annotations

import json
from types import SimpleNamespace

from skillopt.model import claude_backend

_TOOLS = [
{
"type": "function",
"function": {
"name": "lookup",
"description": "Look up a value.",
"parameters": {"type": "object", "properties": {"key": {"type": "string"}}},
},
}
]
_MESSAGES = [
{"role": "system", "content": "You answer questions."},
{"role": "user", "content": "What year was it signed?"},
]


def _install_fake_cli(monkeypatch, envelope: dict) -> dict:
captured: dict = {}

def fake_run(cmd, **kwargs):
captured["cmd"] = list(cmd)
return SimpleNamespace(returncode=0, stdout=json.dumps(envelope) + "\n", stderr="")

monkeypatch.setattr(claude_backend.subprocess, "run", fake_run)
return captured


def test_structured_call_passes_json_schema_and_reads_structured_output(monkeypatch) -> None:
# Shape of a Claude CLI 2.1.296 `-p --output-format json --json-schema ...` result:
# the validated object is in `structured_output`, and `result` holds the same JSON as text.
structured = {
"content": "Looking it up.",
"tool_calls": [{"name": "lookup", "arguments": '{"key": "year"}'}],
}
captured = _install_fake_cli(
monkeypatch,
{
"type": "result",
"subtype": "success",
"is_error": False,
"result": json.dumps(structured),
"structured_output": structured,
"usage": {"input_tokens": 7, "output_tokens": 5},
},
)

message, usage = claude_backend.chat_target_messages(
_MESSAGES, tools=_TOOLS, return_message=True, retries=1
)

cmd = captured["cmd"]
# The CLI rejects `--schema` with "error: unknown option '--schema'".
assert "--schema" not in cmd
schema = json.loads(cmd[cmd.index("--json-schema") + 1])
assert schema == claude_backend._assistant_message_schema()
assert message.content == "Looking it up."
assert [(call.function.name, call.function.arguments) for call in message.tool_calls] == [
("lookup", '{"key": "year"}')
]
assert usage == {"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 12}


def test_structured_call_without_structured_output_keeps_the_text_reply(monkeypatch) -> None:
_install_fake_cli(
monkeypatch,
{"type": "result", "subtype": "success", "result": "1776", "usage": {}},
)

message, _ = claude_backend.chat_target_messages(_MESSAGES, return_message=True, retries=1)

assert message.content == "1776"
assert message.tool_calls == []


def test_plain_call_sends_no_schema(monkeypatch) -> None:
captured = _install_fake_cli(
monkeypatch,
{"type": "result", "subtype": "success", "result": "1776", "usage": {}},
)

text, _ = claude_backend.chat_target("You answer questions.", "What year was it signed?", retries=1)

assert text == "1776"
assert "--json-schema" not in captured["cmd"]
assert "--schema" not in captured["cmd"]