diff --git a/CHANGELOG.md b/CHANGELOG.md index bc704e063..c784b25c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,15 @@ All notable changes to SkillOpt are documented here. This project adheres to normalized name also keys the harvest checkpoint and the tasks document, so `"Codex"` no longer reads a different checkpoint than `"codex"`. A non-string value is rejected with a clear error. +- The `claude_chat` backend sends a single user request to the Claude CLI as + written. It used to wrap it as `History: - User: ...` with every line after + the first indented by two spaces, followed by `Answer the latest user + request.`, so the target saw altered text in a transcript frame instead of + the plain request it gets in deployment. Multi-turn, tool and + structured-output calls keep the transcript form, as does a request that + starts with `/`, which `claude -p` could run as a slash command. The + `claude_code_exec` chat path shares this prompt builder. Scores from earlier + `claude_chat` runs are not directly comparable. ### Fixed - Repeated rollouts of a SkillOpt-Sleep tool task are separate samples again. diff --git a/skillopt/model/claude_backend.py b/skillopt/model/claude_backend.py index 373811af3..4ab139173 100644 --- a/skillopt/model/claude_backend.py +++ b/skillopt/model/claude_backend.py @@ -85,6 +85,7 @@ def _build_prompt_from_messages(messages: list[dict[str, Any]], *, tools: list[d history_parts: list[str] = [] attachments: list[dict[str, Any]] = [] image_counter = 0 + turns: list[tuple[str, str]] = [] def _history_line(label: str, body: str) -> str: stripped = body.strip() @@ -100,6 +101,7 @@ def _history_line(label: str, body: str) -> str: if text.strip(): system_parts.append(text.strip()) continue + turns.append((role, text)) if role == "assistant": block = _history_line("Assistant", text) tool_calls = message.get("tool_calls") or [] @@ -120,6 +122,17 @@ def _history_line(label: str, body: str) -> str: continue history_parts.append(_history_line(role.capitalize(), text)) + system_text = "\n\n".join(part for part in system_parts if part).strip() + # One plain user request goes to the CLI as written, the way the target + # sees it in deployment. The transcript form below re-indents every line + # after the first and adds "History:" and "Answer the latest user request." + # A request starting with "/" stays wrapped: `claude -p` runs a known slash + # command such as /help instead of sending it to the model. + if not tools and not structured_output and len(turns) == 1: + role, text = turns[0] + if role == "user" and text.strip() and not text.lstrip().startswith("/"): + return system_text, text, attachments + prompt_parts: list[str] = [] if tools: simplified_tools = _simplify_tool_schemas(tools) @@ -139,7 +152,7 @@ def _history_line(label: str, body: str) -> str: prompt_parts.append("Set `content` to the assistant-visible reply. Set `tool_calls` to an empty array when no compatibility tool is needed.") else: prompt_parts.append("Answer the latest user request.") - return "\n\n".join(part for part in system_parts if part).strip(), "\n\n".join(prompt_parts), attachments + return system_text, "\n\n".join(prompt_parts), attachments def _copy_attachments_to_temp(attachments: list[dict[str, Any]], temp_dir: str) -> list[dict[str, str]]: diff --git a/tests/test_claude_backend_prompt.py b/tests/test_claude_backend_prompt.py new file mode 100644 index 000000000..14f964fc8 --- /dev/null +++ b/tests/test_claude_backend_prompt.py @@ -0,0 +1,88 @@ +"""A single user request must reach the Claude CLI as written, not as a transcript.""" +from __future__ import annotations + +import json +from types import SimpleNamespace + +from skillopt.model import claude_backend +from skillopt.model.claude_backend import _build_prompt_from_messages + +_SYSTEM = "You review Python code." +_REQUEST = "Fix the bug:\n\ndef total(xs):\n return sum(xs) + 1\n" +_TOOLS = [ + { + "type": "function", + "function": {"name": "run", "description": "Run code.", "parameters": {"type": "object"}}, + } +] + + +def test_single_user_turn_is_sent_verbatim() -> None: + system, prompt, attachments = _build_prompt_from_messages( + [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _REQUEST}] + ) + + assert system == _SYSTEM + # Previously: "History:\n- User: Fix the bug:\n \n def total(xs):\n return ..." + # with every line after the first indented and "Answer the latest user request." appended. + assert prompt == _REQUEST + assert attachments == [] + + +def test_chat_target_feeds_the_request_to_the_cli_unchanged(monkeypatch) -> None: + captured: dict = {} + + def fake_run(cmd, **kwargs): + captured["stdin"] = kwargs["input"] + system_file = cmd[cmd.index("--append-system-prompt-file") + 1] + with open(system_file, encoding="utf-8") as fh: + captured["system"] = fh.read() + payload = {"type": "result", "result": "ok", "usage": {}} + return SimpleNamespace(returncode=0, stdout=json.dumps(payload) + "\n", stderr="") + + monkeypatch.setattr(claude_backend.subprocess, "run", fake_run) + + text, _ = claude_backend.chat_target(_SYSTEM, _REQUEST, retries=1) + + assert text == "ok" + assert captured["stdin"] == _REQUEST + assert captured["system"] == _SYSTEM + + +def test_multi_turn_history_keeps_the_transcript_form() -> None: + _, prompt, _ = _build_prompt_from_messages( + [ + {"role": "system", "content": _SYSTEM}, + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second question"}, + ] + ) + + assert prompt.startswith("History:\n- User: First question\n- Assistant: First answer") + assert prompt.endswith("Answer the latest user request.") + + +def test_tool_and_structured_calls_keep_their_instructions() -> None: + messages = [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _REQUEST}] + + _, with_tools, _ = _build_prompt_from_messages(messages, tools=_TOOLS, structured_output=True) + _, structured, _ = _build_prompt_from_messages(messages, structured_output=True) + + assert with_tools.startswith("Available compatibility tools:") + assert "History:\n- User: Fix the bug:" in with_tools + assert structured.endswith("Return only JSON matching the provided schema.") + + +def test_request_that_starts_with_a_slash_keeps_the_transcript_form() -> None: + # `claude -p` runs a prompt such as "/help" as a slash command without + # calling the model, so a request starting with "/" stays wrapped. + _, prompt, _ = _build_prompt_from_messages([{"role": "user", "content": "/help me read this log"}]) + + assert prompt == "History:\n- User: /help me read this log\n\nAnswer the latest user request." + + +def test_empty_user_turn_never_yields_an_empty_prompt() -> None: + _, prompt, _ = _build_prompt_from_messages([{"role": "user", "content": " "}]) + + assert prompt.strip()