Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,15 @@ All notable changes to SkillOpt are documented here. This project adheres to
normalized name also keys the harvest checkpoint and the tasks document, so
`"Codex"` no longer reads a different checkpoint than `"codex"`. A non-string
value is rejected with a clear error.
- The `claude_chat` backend sends a single user request to the Claude CLI as
written. It used to wrap it as `History: - User: ...` with every line after
the first indented by two spaces, followed by `Answer the latest user
request.`, so the target saw altered text in a transcript frame instead of
the plain request it gets in deployment. Multi-turn, tool and
structured-output calls keep the transcript form, as does a request that
starts with `/`, which `claude -p` could run as a slash command. The
`claude_code_exec` chat path shares this prompt builder. Scores from earlier
`claude_chat` runs are not directly comparable.

### Fixed
- Repeated rollouts of a SkillOpt-Sleep tool task are separate samples again.
Expand Down
15 changes: 14 additions & 1 deletion skillopt/model/claude_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ def _build_prompt_from_messages(messages: list[dict[str, Any]], *, tools: list[d
history_parts: list[str] = []
attachments: list[dict[str, Any]] = []
image_counter = 0
turns: list[tuple[str, str]] = []

def _history_line(label: str, body: str) -> str:
stripped = body.strip()
Expand All @@ -100,6 +101,7 @@ def _history_line(label: str, body: str) -> str:
if text.strip():
system_parts.append(text.strip())
continue
turns.append((role, text))
if role == "assistant":
block = _history_line("Assistant", text)
tool_calls = message.get("tool_calls") or []
Expand All @@ -120,6 +122,17 @@ def _history_line(label: str, body: str) -> str:
continue
history_parts.append(_history_line(role.capitalize(), text))

system_text = "\n\n".join(part for part in system_parts if part).strip()
# One plain user request goes to the CLI as written, the way the target
# sees it in deployment. The transcript form below re-indents every line
# after the first and adds "History:" and "Answer the latest user request."
# A request starting with "/" stays wrapped: `claude -p` runs a known slash
# command such as /help instead of sending it to the model.
if not tools and not structured_output and len(turns) == 1:
role, text = turns[0]
if role == "user" and text.strip() and not text.lstrip().startswith("/"):
return system_text, text, attachments

prompt_parts: list[str] = []
if tools:
simplified_tools = _simplify_tool_schemas(tools)
Expand All @@ -139,7 +152,7 @@ def _history_line(label: str, body: str) -> str:
prompt_parts.append("Set `content` to the assistant-visible reply. Set `tool_calls` to an empty array when no compatibility tool is needed.")
else:
prompt_parts.append("Answer the latest user request.")
return "\n\n".join(part for part in system_parts if part).strip(), "\n\n".join(prompt_parts), attachments
return system_text, "\n\n".join(prompt_parts), attachments


def _copy_attachments_to_temp(attachments: list[dict[str, Any]], temp_dir: str) -> list[dict[str, str]]:
Expand Down
88 changes: 88 additions & 0 deletions tests/test_claude_backend_prompt.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
"""A single user request must reach the Claude CLI as written, not as a transcript."""
from __future__ import annotations

import json
from types import SimpleNamespace

from skillopt.model import claude_backend
from skillopt.model.claude_backend import _build_prompt_from_messages

_SYSTEM = "You review Python code."
_REQUEST = "Fix the bug:\n\ndef total(xs):\n return sum(xs) + 1\n"
_TOOLS = [
{
"type": "function",
"function": {"name": "run", "description": "Run code.", "parameters": {"type": "object"}},
}
]


def test_single_user_turn_is_sent_verbatim() -> None:
system, prompt, attachments = _build_prompt_from_messages(
[{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _REQUEST}]
)

assert system == _SYSTEM
# Previously: "History:\n- User: Fix the bug:\n \n def total(xs):\n return ..."
# with every line after the first indented and "Answer the latest user request." appended.
assert prompt == _REQUEST
assert attachments == []


def test_chat_target_feeds_the_request_to_the_cli_unchanged(monkeypatch) -> None:
captured: dict = {}

def fake_run(cmd, **kwargs):
captured["stdin"] = kwargs["input"]
system_file = cmd[cmd.index("--append-system-prompt-file") + 1]
with open(system_file, encoding="utf-8") as fh:
captured["system"] = fh.read()
payload = {"type": "result", "result": "ok", "usage": {}}
return SimpleNamespace(returncode=0, stdout=json.dumps(payload) + "\n", stderr="")

monkeypatch.setattr(claude_backend.subprocess, "run", fake_run)

text, _ = claude_backend.chat_target(_SYSTEM, _REQUEST, retries=1)

assert text == "ok"
assert captured["stdin"] == _REQUEST
assert captured["system"] == _SYSTEM


def test_multi_turn_history_keeps_the_transcript_form() -> None:
_, prompt, _ = _build_prompt_from_messages(
[
{"role": "system", "content": _SYSTEM},
{"role": "user", "content": "First question"},
{"role": "assistant", "content": "First answer"},
{"role": "user", "content": "Second question"},
]
)

assert prompt.startswith("History:\n- User: First question\n- Assistant: First answer")
assert prompt.endswith("Answer the latest user request.")


def test_tool_and_structured_calls_keep_their_instructions() -> None:
messages = [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _REQUEST}]

_, with_tools, _ = _build_prompt_from_messages(messages, tools=_TOOLS, structured_output=True)
_, structured, _ = _build_prompt_from_messages(messages, structured_output=True)

assert with_tools.startswith("Available compatibility tools:")
assert "History:\n- User: Fix the bug:" in with_tools
assert structured.endswith("Return only JSON matching the provided schema.")


def test_request_that_starts_with_a_slash_keeps_the_transcript_form() -> None:
# `claude -p` runs a prompt such as "/help" as a slash command without
# calling the model, so a request starting with "/" stays wrapped.
_, prompt, _ = _build_prompt_from_messages([{"role": "user", "content": "/help me read this log"}])

assert prompt == "History:\n- User: /help me read this log\n\nAnswer the latest user request."


def test_empty_user_turn_never_yields_an_empty_prompt() -> None:
_, prompt, _ = _build_prompt_from_messages([{"role": "user", "content": " "}])

assert prompt.strip()