From f43e02faa16544f82083d41fe714e0c39e371b13 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:55:26 +0000 Subject: [PATCH] fix(optimizer): ignore a non-string new_skill from an optimizer rewrite rewrite_skill_from_suggestions accepted any new_skill whose str() was non-blank. If the optimizer returned "new_skill": null, str(None) is "None", so the candidate skill became the literal text "None"; a list or dict became its Python repr. The trainer's full_rewrite_minibatch path selected candidates with the same str(...).strip() check. Add usable_new_skill(), which returns the skill text only for a non-blank string, and use it in both places. A non-string new_skill now counts as no usable rewrite, and the trainer keeps the current skill as it already does for an empty one. --- CHANGELOG.md | 4 +++ skillopt/engine/trainer.py | 6 ++--- skillopt/optimizer/rewrite.py | 16 +++++++++-- tests/test_rewrite_new_skill.py | 48 +++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 tests/test_rewrite_new_skill.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd0fffee..11d377473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,6 +209,10 @@ All notable changes to SkillOpt are documented here. This project adheres to instead of treating them as missing, and load integral floats such as `1776.0` as `1776`, so a correct integer answer is no longer scored wrong (thanks @Shizoqua). +- Treat an optimizer rewrite whose `new_skill` is `null`, a list, or another + non-string value as no usable rewrite, in both `rewrite_from_suggestions` + and `full_rewrite_minibatch` modes, instead of writing `None` or the value's + Python repr as the new skill (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index d96026c3e..acbcdbdc9 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -57,7 +57,7 @@ from skillopt.optimizer.clip import rank_and_select from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate from skillopt.optimizer.meta_skill import run_meta_skill -from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions +from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions, usable_new_skill from skillopt.optimizer.scheduler import build_scheduler from skillopt.optimizer.skill import apply_patch_with_report from skillopt.optimizer.skill_aware import ( @@ -1474,12 +1474,12 @@ def _persist_runtime_state(last_completed_step: int) -> None: selected_candidate = next( ( item for item in skill_candidates - if isinstance(item, dict) and str(item.get("new_skill", "")).strip() + if isinstance(item, dict) and usable_new_skill(item.get("new_skill")) ), None, ) if selected_candidate: - candidate_skill = str(selected_candidate["new_skill"]).rstrip() + "\n" + candidate_skill = usable_new_skill(selected_candidate["new_skill"]) apply_report = [] rewrite_result = { "reasoning": ranked_patch.get("reasoning", ""), diff --git a/skillopt/optimizer/rewrite.py b/skillopt/optimizer/rewrite.py index f8b062baa..af16503d9 100644 --- a/skillopt/optimizer/rewrite.py +++ b/skillopt/optimizer/rewrite.py @@ -9,6 +9,17 @@ from skillopt.utils import extract_json +def usable_new_skill(value: object) -> str: + """Return a model-provided ``new_skill`` as skill text, or ``""`` if unusable. + + Only a non-blank string is a skill document. ``str()`` of ``None``, a list or + a dict would otherwise become the literal skill text (for example ``"None"``). + """ + if not isinstance(value, str) or not value.strip(): + return "" + return value.rstrip() + "\n" + + def rewrite_skill_from_suggestions( skill_content: str, patch: dict, @@ -49,8 +60,9 @@ def rewrite_skill_from_suggestions( reasoning_effort=reasoning_effort, ) result = extract_json(response) - if result and str(result.get("new_skill", "")).strip(): - result["new_skill"] = str(result["new_skill"]).rstrip() + "\n" + new_skill = usable_new_skill(result.get("new_skill")) if result else "" + if new_skill: + result["new_skill"] = new_skill if "change_summary" not in result or not isinstance(result["change_summary"], list): result["change_summary"] = [] return result diff --git a/tests/test_rewrite_new_skill.py b/tests/test_rewrite_new_skill.py new file mode 100644 index 000000000..5ba1b182e --- /dev/null +++ b/tests/test_rewrite_new_skill.py @@ -0,0 +1,48 @@ +"""Optimizer rewrites must not turn a non-string ``new_skill`` into the skill text.""" + +from __future__ import annotations + +import json + +import pytest + +from skillopt.optimizer import rewrite +from skillopt.optimizer.rewrite import usable_new_skill + +PATCH = {"revise_suggestions": [{"type": "add", "title": "Check units", "instruction": "Check units first."}]} + + +def _rewrite(monkeypatch, payload: dict): + monkeypatch.setattr(rewrite, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {})) + return rewrite.rewrite_skill_from_suggestions("# Skill\n- Rule one.\n", PATCH, system_prompt="rewrite") + + +@pytest.mark.parametrize("new_skill", [None, ["# Skill", "- Rule one."], {"text": "# Skill"}, 42, " "]) +def test_unusable_new_skill_gives_no_rewrite(monkeypatch, new_skill): + assert _rewrite(monkeypatch, {"new_skill": new_skill, "change_summary": ["x"]}) is None + + +def test_string_new_skill_is_returned(monkeypatch): + result = _rewrite( + monkeypatch, {"new_skill": "# Skill\n- Rule one.\n- Check units first. ", "change_summary": ["x"]} + ) + + assert result["new_skill"] == "# Skill\n- Rule one.\n- Check units first.\n" + assert result["change_summary"] == ["x"] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("# Skill\n- Rule. ", "# Skill\n- Rule.\n"), + (None, ""), + ("", ""), + (" \n ", ""), + (["# Skill"], ""), + ({"new_skill": "# Skill"}, ""), + (0, ""), + ], +) +def test_usable_new_skill(value, expected): + # The trainer uses this to pick a full_rewrite_minibatch candidate. + assert usable_new_skill(value) == expected