diff --git a/CHANGELOG.md b/CHANGELOG.md index bc704e063..3990e862d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,11 @@ All notable changes to SkillOpt are documented here. This project adheres to normalized name also keys the harvest checkpoint and the tasks document, so `"Codex"` no longer reads a different checkpoint than `"codex"`. A non-string value is rejected with a clear error. +- The `claude_chat` and `claude_code_exec` backends default to the Claude + CLI's `sonnet` alias, which resolves to the latest Sonnet, instead of + `claude-sonnet-4-6`. `--backend claude` without `model.optimizer` or + `model.target` ran both roles on that model after Sonnet 5 and Sonnet 5.5 + shipped. Set either key to a full model ID to pin one. ### Fixed - Repeated rollouts of a SkillOpt-Sleep tool task are separate samples again. diff --git a/skillopt/model/claude_backend.py b/skillopt/model/claude_backend.py index 373811af3..f0cea93f8 100644 --- a/skillopt/model/claude_backend.py +++ b/skillopt/model/claude_backend.py @@ -19,8 +19,8 @@ CLAUDE_SETTING_SOURCES = os.environ.get("CLAUDE_SETTING_SOURCES", "user,project") CLAUDE_ALLOW_ATTACHMENT_READ = os.environ.get("CLAUDE_ALLOW_ATTACHMENT_READ", "1").strip().lower() not in {"0", "false", "no"} -OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6") -TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6") +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", default_model_for_backend("claude")) +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", default_model_for_backend("claude")) REASONING_EFFORT: str | None = None _VALID_EFFORTS = {"low", "medium", "high", "xhigh", "max"} diff --git a/skillopt/model/claude_code_backend.py b/skillopt/model/claude_code_backend.py index 7cc626d2f..0949199ee 100644 --- a/skillopt/model/claude_code_backend.py +++ b/skillopt/model/claude_code_backend.py @@ -15,10 +15,10 @@ from skillopt.model import codex_backend as _codex from skillopt.model.claude_backend import _build_prompt_from_messages from skillopt.model.codex_harness import run_claude_code_chat -from skillopt.model.common import TokenTracker +from skillopt.model.common import TokenTracker, default_model_for_backend -OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6") -TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6") +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", default_model_for_backend("claude_code_exec")) +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", default_model_for_backend("claude_code_exec")) REASONING_EFFORT: str | None = None tracker = TokenTracker() diff --git a/skillopt/model/common.py b/skillopt/model/common.py index cb91f415b..ebf40dfec 100644 --- a/skillopt/model/common.py +++ b/skillopt/model/common.py @@ -20,9 +20,11 @@ "openai_chat": "gpt-4o", "codex": "gpt-4o", "codex_exec": "gpt-4o", - "claude": "claude-sonnet-4-6", - "claude_chat": "claude-sonnet-4-6", - "claude_code_exec": "claude-sonnet-4-6", + # The Claude CLI resolves `sonnet` to the latest Sonnet the account can + # use, so this default cannot go stale the way a dated model ID does. + "claude": "sonnet", + "claude_chat": "sonnet", + "claude_code_exec": "sonnet", "cursor_exec": "composer-2.5", # copilot_chat / copilot_exec deliberately use the Azure default fallback # from default_model_for_backend(). That fallback configures shared role diff --git a/tests/test_claude_code_exec_resolution.py b/tests/test_claude_code_exec_resolution.py index b283b2643..4c6da8a59 100644 --- a/tests/test_claude_code_exec_resolution.py +++ b/tests/test_claude_code_exec_resolution.py @@ -39,7 +39,7 @@ def test_train_claude_code_exec_defaults_target_only(monkeypatch) -> None: assert cfg["optimizer_backend"] == "openai_chat" assert cfg["optimizer_model"] == "gpt-5.5" assert cfg["target_backend"] == "claude_code_exec" - assert cfg["target_model"] == "claude-sonnet-4-6" + assert cfg["target_model"] == "sonnet" def test_train_claude_code_exec_optimizer_opt_in_normalizes_model(monkeypatch) -> None: @@ -52,9 +52,9 @@ def test_train_claude_code_exec_optimizer_opt_in_normalizes_model(monkeypatch) - ) assert cfg["optimizer_backend"] == "claude_code_exec" - assert cfg["optimizer_model"] == "claude-sonnet-4-6" + assert cfg["optimizer_model"] == "sonnet" assert cfg["target_backend"] == "claude_code_exec" - assert cfg["target_model"] == "claude-sonnet-4-6" + assert cfg["target_model"] == "sonnet" class _StopAfterResolution(Exception): @@ -122,7 +122,7 @@ def test_eval_claude_code_exec_defaults_target_only(monkeypatch, tmp_path) -> No "optimizer_backend": "openai_chat", "target_backend": "claude_code_exec", "optimizer_model": "gpt-5.5", - "target_model": "claude-sonnet-4-6", + "target_model": "sonnet", } @@ -136,6 +136,6 @@ def test_eval_claude_code_exec_optimizer_opt_in_normalizes_model( assert observed == { "optimizer_backend": "claude_code_exec", "target_backend": "claude_code_exec", - "optimizer_model": "claude-sonnet-4-6", - "target_model": "claude-sonnet-4-6", + "optimizer_model": "sonnet", + "target_model": "sonnet", } diff --git a/tests/test_claude_default_model.py b/tests/test_claude_default_model.py new file mode 100644 index 000000000..292544735 --- /dev/null +++ b/tests/test_claude_default_model.py @@ -0,0 +1,38 @@ +"""Claude backends default to the Claude CLI's `sonnet` alias, from one source.""" +from __future__ import annotations + +import importlib.util + +import pytest + +from skillopt.model import claude_backend, claude_code_backend +from skillopt.model.common import default_model_for_backend + + +@pytest.mark.parametrize("backend", ["claude", "claude_chat", "claude_code_exec", "anthropic"]) +def test_claude_backends_default_to_the_cli_sonnet_alias(backend: str) -> None: + # `claude --help`: "Provide an alias for the latest model (e.g. 'fable', 'opus', + # or 'sonnet') or a model's full name." A dated ID such as claude-sonnet-4-6 + # keeps every unconfigured run on that model after newer ones ship. + assert default_model_for_backend(backend) == "sonnet" + + +def _fresh_copy(module): + spec = importlib.util.spec_from_file_location(f"_fresh_{module.__name__}", module.__file__) + fresh = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fresh) + return fresh + + +@pytest.mark.parametrize( + ("module", "backend"), + [(claude_backend, "claude"), (claude_code_backend, "claude_code_exec")], +) +def test_module_deployments_fall_back_to_the_backend_default(monkeypatch, module, backend) -> None: + monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) + monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) + + fresh = _fresh_copy(module) + + assert fresh.OPTIMIZER_DEPLOYMENT == default_model_for_backend(backend) + assert fresh.TARGET_DEPLOYMENT == default_model_for_backend(backend)