Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ All notable changes to SkillOpt are documented here. This project adheres to
instead of treating them as missing, and load integral floats such as
`1776.0` as `1776`, so a correct integer answer is no longer scored wrong
(thanks @Shizoqua).
- Treat an optimizer rewrite whose `new_skill` is `null`, a list, or another
non-string value as no usable rewrite, in both `rewrite_from_suggestions`
and `full_rewrite_minibatch` modes, instead of writing `None` or the value's
Python repr as the new skill (thanks @Shizoqua).

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
6 changes: 3 additions & 3 deletions skillopt/engine/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
from skillopt.optimizer.clip import rank_and_select
from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate
from skillopt.optimizer.meta_skill import run_meta_skill
from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions
from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions, usable_new_skill
from skillopt.optimizer.scheduler import build_scheduler
from skillopt.optimizer.skill import apply_patch_with_report
from skillopt.optimizer.skill_aware import (
Expand Down Expand Up @@ -1474,12 +1474,12 @@ def _persist_runtime_state(last_completed_step: int) -> None:
selected_candidate = next(
(
item for item in skill_candidates
if isinstance(item, dict) and str(item.get("new_skill", "")).strip()
if isinstance(item, dict) and usable_new_skill(item.get("new_skill"))
),
None,
)
if selected_candidate:
candidate_skill = str(selected_candidate["new_skill"]).rstrip() + "\n"
candidate_skill = usable_new_skill(selected_candidate["new_skill"])
apply_report = []
rewrite_result = {
"reasoning": ranked_patch.get("reasoning", ""),
Expand Down
16 changes: 14 additions & 2 deletions skillopt/optimizer/rewrite.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@
from skillopt.utils import extract_json


def usable_new_skill(value: object) -> str:
"""Return a model-provided ``new_skill`` as skill text, or ``""`` if unusable.

Only a non-blank string is a skill document. ``str()`` of ``None``, a list or
a dict would otherwise become the literal skill text (for example ``"None"``).
"""
if not isinstance(value, str) or not value.strip():
return ""
return value.rstrip() + "\n"


def rewrite_skill_from_suggestions(
skill_content: str,
patch: dict,
Expand Down Expand Up @@ -49,8 +60,9 @@ def rewrite_skill_from_suggestions(
reasoning_effort=reasoning_effort,
)
result = extract_json(response)
if result and str(result.get("new_skill", "")).strip():
result["new_skill"] = str(result["new_skill"]).rstrip() + "\n"
new_skill = usable_new_skill(result.get("new_skill")) if result else ""
if new_skill:
result["new_skill"] = new_skill
if "change_summary" not in result or not isinstance(result["change_summary"], list):
result["change_summary"] = []
return result
Expand Down
48 changes: 48 additions & 0 deletions tests/test_rewrite_new_skill.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Optimizer rewrites must not turn a non-string ``new_skill`` into the skill text."""

from __future__ import annotations

import json

import pytest

from skillopt.optimizer import rewrite
from skillopt.optimizer.rewrite import usable_new_skill

PATCH = {"revise_suggestions": [{"type": "add", "title": "Check units", "instruction": "Check units first."}]}


def _rewrite(monkeypatch, payload: dict):
monkeypatch.setattr(rewrite, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {}))
return rewrite.rewrite_skill_from_suggestions("# Skill\n- Rule one.\n", PATCH, system_prompt="rewrite")


@pytest.mark.parametrize("new_skill", [None, ["# Skill", "- Rule one."], {"text": "# Skill"}, 42, " "])
def test_unusable_new_skill_gives_no_rewrite(monkeypatch, new_skill):
assert _rewrite(monkeypatch, {"new_skill": new_skill, "change_summary": ["x"]}) is None


def test_string_new_skill_is_returned(monkeypatch):
result = _rewrite(
monkeypatch, {"new_skill": "# Skill\n- Rule one.\n- Check units first. ", "change_summary": ["x"]}
)

assert result["new_skill"] == "# Skill\n- Rule one.\n- Check units first.\n"
assert result["change_summary"] == ["x"]


@pytest.mark.parametrize(
("value", "expected"),
[
("# Skill\n- Rule. ", "# Skill\n- Rule.\n"),
(None, ""),
("", ""),
(" \n ", ""),
(["# Skill"], ""),
({"new_skill": "# Skill"}, ""),
(0, ""),
],
)
def test_usable_new_skill(value, expected):
# The trainer uses this to pick a full_rewrite_minibatch candidate.
assert usable_new_skill(value) == expected