Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ All notable changes to SkillOpt are documented here. This project adheres to
normalized name also keys the harvest checkpoint and the tasks document, so
`"Codex"` no longer reads a different checkpoint than `"codex"`. A non-string
value is rejected with a clear error.
- The `claude_chat` and `claude_code_exec` backends default to the Claude
CLI's `sonnet` alias, which resolves to the latest Sonnet, instead of
`claude-sonnet-4-6`. `--backend claude` without `model.optimizer` or
`model.target` ran both roles on that model after Sonnet 5 and Sonnet 5.5
shipped. Set either key to a full model ID to pin one.

### Fixed
- Repeated rollouts of a SkillOpt-Sleep tool task are separate samples again.
Expand Down
4 changes: 2 additions & 2 deletions skillopt/model/claude_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
CLAUDE_SETTING_SOURCES = os.environ.get("CLAUDE_SETTING_SOURCES", "user,project")
CLAUDE_ALLOW_ATTACHMENT_READ = os.environ.get("CLAUDE_ALLOW_ATTACHMENT_READ", "1").strip().lower() not in {"0", "false", "no"}

OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6")
TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6")
OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", default_model_for_backend("claude"))
TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", default_model_for_backend("claude"))
REASONING_EFFORT: str | None = None
_VALID_EFFORTS = {"low", "medium", "high", "xhigh", "max"}

Expand Down
6 changes: 3 additions & 3 deletions skillopt/model/claude_code_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
from skillopt.model import codex_backend as _codex
from skillopt.model.claude_backend import _build_prompt_from_messages
from skillopt.model.codex_harness import run_claude_code_chat
from skillopt.model.common import TokenTracker
from skillopt.model.common import TokenTracker, default_model_for_backend

OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6")
TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6")
OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", default_model_for_backend("claude_code_exec"))
TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", default_model_for_backend("claude_code_exec"))
REASONING_EFFORT: str | None = None
tracker = TokenTracker()

Expand Down
8 changes: 5 additions & 3 deletions skillopt/model/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
"openai_chat": "gpt-4o",
"codex": "gpt-4o",
"codex_exec": "gpt-4o",
"claude": "claude-sonnet-4-6",
"claude_chat": "claude-sonnet-4-6",
"claude_code_exec": "claude-sonnet-4-6",
# The Claude CLI resolves `sonnet` to the latest Sonnet the account can
# use, so this default cannot go stale the way a dated model ID does.
"claude": "sonnet",
"claude_chat": "sonnet",
"claude_code_exec": "sonnet",
"cursor_exec": "composer-2.5",
# copilot_chat / copilot_exec deliberately use the Azure default fallback
# from default_model_for_backend(). That fallback configures shared role
Expand Down
12 changes: 6 additions & 6 deletions tests/test_claude_code_exec_resolution.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def test_train_claude_code_exec_defaults_target_only(monkeypatch) -> None:
assert cfg["optimizer_backend"] == "openai_chat"
assert cfg["optimizer_model"] == "gpt-5.5"
assert cfg["target_backend"] == "claude_code_exec"
assert cfg["target_model"] == "claude-sonnet-4-6"
assert cfg["target_model"] == "sonnet"


def test_train_claude_code_exec_optimizer_opt_in_normalizes_model(monkeypatch) -> None:
Expand All @@ -52,9 +52,9 @@ def test_train_claude_code_exec_optimizer_opt_in_normalizes_model(monkeypatch) -
)

assert cfg["optimizer_backend"] == "claude_code_exec"
assert cfg["optimizer_model"] == "claude-sonnet-4-6"
assert cfg["optimizer_model"] == "sonnet"
assert cfg["target_backend"] == "claude_code_exec"
assert cfg["target_model"] == "claude-sonnet-4-6"
assert cfg["target_model"] == "sonnet"


class _StopAfterResolution(Exception):
Expand Down Expand Up @@ -122,7 +122,7 @@ def test_eval_claude_code_exec_defaults_target_only(monkeypatch, tmp_path) -> No
"optimizer_backend": "openai_chat",
"target_backend": "claude_code_exec",
"optimizer_model": "gpt-5.5",
"target_model": "claude-sonnet-4-6",
"target_model": "sonnet",
}


Expand All @@ -136,6 +136,6 @@ def test_eval_claude_code_exec_optimizer_opt_in_normalizes_model(
assert observed == {
"optimizer_backend": "claude_code_exec",
"target_backend": "claude_code_exec",
"optimizer_model": "claude-sonnet-4-6",
"target_model": "claude-sonnet-4-6",
"optimizer_model": "sonnet",
"target_model": "sonnet",
}
38 changes: 38 additions & 0 deletions tests/test_claude_default_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
"""Claude backends default to the Claude CLI's `sonnet` alias, from one source."""
from __future__ import annotations

import importlib.util

import pytest

from skillopt.model import claude_backend, claude_code_backend
from skillopt.model.common import default_model_for_backend


@pytest.mark.parametrize("backend", ["claude", "claude_chat", "claude_code_exec", "anthropic"])
def test_claude_backends_default_to_the_cli_sonnet_alias(backend: str) -> None:
# `claude --help`: "Provide an alias for the latest model (e.g. 'fable', 'opus',
# or 'sonnet') or a model's full name." A dated ID such as claude-sonnet-4-6
# keeps every unconfigured run on that model after newer ones ship.
assert default_model_for_backend(backend) == "sonnet"


def _fresh_copy(module):
spec = importlib.util.spec_from_file_location(f"_fresh_{module.__name__}", module.__file__)
fresh = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fresh)
return fresh


@pytest.mark.parametrize(
("module", "backend"),
[(claude_backend, "claude"), (claude_code_backend, "claude_code_exec")],
)
def test_module_deployments_fall_back_to_the_backend_default(monkeypatch, module, backend) -> None:
monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False)
monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False)

fresh = _fresh_copy(module)

assert fresh.OPTIMIZER_DEPLOYMENT == default_model_for_backend(backend)
assert fresh.TARGET_DEPLOYMENT == default_model_for_backend(backend)