diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd0fffe..5e466e0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,6 +209,10 @@ All notable changes to SkillOpt are documented here. This project adheres to instead of treating them as missing, and load integral floats such as `1776.0` as `1776`, so a correct integer answer is no longer scored wrong (thanks @Shizoqua). +- Keep slow-update guidance and optimizer meta-skill memory as text when the + optimizer returns them as a list of strings, instead of writing the list's + Python repr into the skill. Non-text or blank content is treated as no + update (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/optimizer/meta_skill.py b/skillopt/optimizer/meta_skill.py index 6e34ff10..dd3579e9 100644 --- a/skillopt/optimizer/meta_skill.py +++ b/skillopt/optimizer/meta_skill.py @@ -10,7 +10,7 @@ import traceback from skillopt.model import chat_optimizer -from skillopt.optimizer.slow_update import format_comparison_text +from skillopt.optimizer.slow_update import content_text, format_comparison_text from skillopt.prompts import load_prompt from skillopt.utils import extract_json @@ -68,10 +68,11 @@ def run_meta_skill( stage="meta_skill", ) result = extract_json(response) - if result and result.get("meta_skill_content"): + content = content_text(result.get("meta_skill_content")) if result else "" + if content: return { - "reasoning": str(result.get("reasoning", "")).strip(), - "meta_skill_content": str(result["meta_skill_content"]).strip(), + "reasoning": content_text(result.get("reasoning")), + "meta_skill_content": content, } except Exception: # noqa: BLE001 traceback.print_exc() diff --git a/skillopt/optimizer/slow_update.py b/skillopt/optimizer/slow_update.py index ba05b750..b4c37094 100644 --- a/skillopt/optimizer/slow_update.py +++ b/skillopt/optimizer/slow_update.py @@ -39,6 +39,20 @@ # ── Field manipulation helpers ────────────────────────────────────────────── +def content_text(value: object) -> str: + """Return a model-provided text field as stripped text, or ``""`` if unusable. + + A list of strings (a common model deviation for bullet-style guidance) is + joined one item per line. Anything else that is not a string, such as + ``None``, a dict or a number, is unusable rather than written as its repr. + """ + if isinstance(value, str): + return value.strip() + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return "\n".join(item.strip() for item in value if item.strip()) + return "" + + def has_slow_update_field(skill: str) -> bool: return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill @@ -432,10 +446,11 @@ def run_slow_update( stage="slow_update", ) result = extract_json(response) - if result and result.get("slow_update_content"): + content = content_text(result.get("slow_update_content")) if result else "" + if content: return { - "reasoning": str(result.get("reasoning", "")).strip(), - "slow_update_content": str(result["slow_update_content"]).strip(), + "reasoning": content_text(result.get("reasoning")), + "slow_update_content": content, } except Exception: # noqa: BLE001 traceback.print_exc() diff --git a/tests/test_slow_update_content.py b/tests/test_slow_update_content.py new file mode 100644 index 00000000..c5cef257 --- /dev/null +++ b/tests/test_slow_update_content.py @@ -0,0 +1,52 @@ +"""Slow-update and meta-skill outputs must be text, never a Python repr of a list.""" + +from __future__ import annotations + +import json + +import pytest + +from skillopt.optimizer import meta_skill, slow_update + + +def _slow_update(monkeypatch, payload: dict): + monkeypatch.setattr(slow_update, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {})) + return slow_update.run_slow_update("# Skill", [], [], [], comparison_pairs=[], system_prompt="slow") + + +def _meta_skill(monkeypatch, payload: dict): + monkeypatch.setattr(meta_skill, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {})) + return meta_skill.run_meta_skill("# Prev", "# Curr", [], system_prompt="meta") + + +def test_slow_update_list_content_is_joined_into_lines(monkeypatch): + result = _slow_update( + monkeypatch, {"reasoning": "r", "slow_update_content": ["Check units first.", " Prefer exact spans. "]} + ) + + assert result["slow_update_content"] == "Check units first.\nPrefer exact spans." + skill = slow_update.replace_slow_update_field("# Skill", result["slow_update_content"]) + assert "['" not in skill + assert slow_update.extract_slow_update_field(skill) == "Check units first.\nPrefer exact spans." + + +def test_meta_skill_list_content_is_joined_into_lines(monkeypatch): + result = _meta_skill( + monkeypatch, {"reasoning": "r", "meta_skill_content": ["Rank by support.", "Merge duplicates."]} + ) + + assert result["meta_skill_content"] == "Rank by support.\nMerge duplicates." + + +@pytest.mark.parametrize("content", [{"text": "x"}, 42, ["ok", 3], [], ["", " "], " "]) +def test_unusable_content_gives_no_update(monkeypatch, content): + assert _slow_update(monkeypatch, {"reasoning": "r", "slow_update_content": content}) is None + assert _meta_skill(monkeypatch, {"reasoning": "r", "meta_skill_content": content}) is None + + +def test_string_content_and_null_reasoning(monkeypatch): + slow = _slow_update(monkeypatch, {"reasoning": None, "slow_update_content": " Check units first. "}) + meta = _meta_skill(monkeypatch, {"reasoning": None, "meta_skill_content": "Rank by support."}) + + assert slow == {"reasoning": "", "slow_update_content": "Check units first."} + assert meta == {"reasoning": "", "meta_skill_content": "Rank by support."}