From 664f9ca4c62c57dc1cb16e36cdb0c0fe78ef7a12 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:56:47 +0000 Subject: [PATCH] fix(optimizer): keep slow-update and meta-skill content as text run_slow_update and run_meta_skill accepted any truthy content field and passed it through str(). When the optimizer returned the guidance as a JSON list of strings, the list's Python repr was written into the skill's slow-update region or the optimizer memory, for example "['Check units first.', 'Prefer exact spans.']". A dict or number was written the same way, and whitespace-only content was accepted as an empty update. A null reasoning field became the text "None". Add content_text(): a string is stripped, a list of strings is joined one item per line, and anything else is unusable. Both parsers use it for the content and the reasoning, and return None when the content is empty, as they already do for a parse failure. --- CHANGELOG.md | 4 +++ skillopt/optimizer/meta_skill.py | 9 +++--- skillopt/optimizer/slow_update.py | 21 +++++++++++-- tests/test_slow_update_content.py | 52 +++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/test_slow_update_content.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd0fffee..5e466e0a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,6 +209,10 @@ All notable changes to SkillOpt are documented here. This project adheres to instead of treating them as missing, and load integral floats such as `1776.0` as `1776`, so a correct integer answer is no longer scored wrong (thanks @Shizoqua). +- Keep slow-update guidance and optimizer meta-skill memory as text when the + optimizer returns them as a list of strings, instead of writing the list's + Python repr into the skill. Non-text or blank content is treated as no + update (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/optimizer/meta_skill.py b/skillopt/optimizer/meta_skill.py index 6e34ff10d..dd3579e90 100644 --- a/skillopt/optimizer/meta_skill.py +++ b/skillopt/optimizer/meta_skill.py @@ -10,7 +10,7 @@ import traceback from skillopt.model import chat_optimizer -from skillopt.optimizer.slow_update import format_comparison_text +from skillopt.optimizer.slow_update import content_text, format_comparison_text from skillopt.prompts import load_prompt from skillopt.utils import extract_json @@ -68,10 +68,11 @@ def run_meta_skill( stage="meta_skill", ) result = extract_json(response) - if result and result.get("meta_skill_content"): + content = content_text(result.get("meta_skill_content")) if result else "" + if content: return { - "reasoning": str(result.get("reasoning", "")).strip(), - "meta_skill_content": str(result["meta_skill_content"]).strip(), + "reasoning": content_text(result.get("reasoning")), + "meta_skill_content": content, } except Exception: # noqa: BLE001 traceback.print_exc() diff --git a/skillopt/optimizer/slow_update.py b/skillopt/optimizer/slow_update.py index ba05b7507..b4c370948 100644 --- a/skillopt/optimizer/slow_update.py +++ b/skillopt/optimizer/slow_update.py @@ -39,6 +39,20 @@ # ── Field manipulation helpers ────────────────────────────────────────────── +def content_text(value: object) -> str: + """Return a model-provided text field as stripped text, or ``""`` if unusable. + + A list of strings (a common model deviation for bullet-style guidance) is + joined one item per line. Anything else that is not a string, such as + ``None``, a dict or a number, is unusable rather than written as its repr. + """ + if isinstance(value, str): + return value.strip() + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return "\n".join(item.strip() for item in value if item.strip()) + return "" + + def has_slow_update_field(skill: str) -> bool: return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill @@ -432,10 +446,11 @@ def run_slow_update( stage="slow_update", ) result = extract_json(response) - if result and result.get("slow_update_content"): + content = content_text(result.get("slow_update_content")) if result else "" + if content: return { - "reasoning": str(result.get("reasoning", "")).strip(), - "slow_update_content": str(result["slow_update_content"]).strip(), + "reasoning": content_text(result.get("reasoning")), + "slow_update_content": content, } except Exception: # noqa: BLE001 traceback.print_exc() diff --git a/tests/test_slow_update_content.py b/tests/test_slow_update_content.py new file mode 100644 index 000000000..c5cef2574 --- /dev/null +++ b/tests/test_slow_update_content.py @@ -0,0 +1,52 @@ +"""Slow-update and meta-skill outputs must be text, never a Python repr of a list.""" + +from __future__ import annotations + +import json + +import pytest + +from skillopt.optimizer import meta_skill, slow_update + + +def _slow_update(monkeypatch, payload: dict): + monkeypatch.setattr(slow_update, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {})) + return slow_update.run_slow_update("# Skill", [], [], [], comparison_pairs=[], system_prompt="slow") + + +def _meta_skill(monkeypatch, payload: dict): + monkeypatch.setattr(meta_skill, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {})) + return meta_skill.run_meta_skill("# Prev", "# Curr", [], system_prompt="meta") + + +def test_slow_update_list_content_is_joined_into_lines(monkeypatch): + result = _slow_update( + monkeypatch, {"reasoning": "r", "slow_update_content": ["Check units first.", " Prefer exact spans. "]} + ) + + assert result["slow_update_content"] == "Check units first.\nPrefer exact spans." + skill = slow_update.replace_slow_update_field("# Skill", result["slow_update_content"]) + assert "['" not in skill + assert slow_update.extract_slow_update_field(skill) == "Check units first.\nPrefer exact spans." + + +def test_meta_skill_list_content_is_joined_into_lines(monkeypatch): + result = _meta_skill( + monkeypatch, {"reasoning": "r", "meta_skill_content": ["Rank by support.", "Merge duplicates."]} + ) + + assert result["meta_skill_content"] == "Rank by support.\nMerge duplicates." + + +@pytest.mark.parametrize("content", [{"text": "x"}, 42, ["ok", 3], [], ["", " "], " "]) +def test_unusable_content_gives_no_update(monkeypatch, content): + assert _slow_update(monkeypatch, {"reasoning": "r", "slow_update_content": content}) is None + assert _meta_skill(monkeypatch, {"reasoning": "r", "meta_skill_content": content}) is None + + +def test_string_content_and_null_reasoning(monkeypatch): + slow = _slow_update(monkeypatch, {"reasoning": None, "slow_update_content": " Check units first. "}) + meta = _meta_skill(monkeypatch, {"reasoning": None, "meta_skill_content": "Rank by support."}) + + assert slow == {"reasoning": "", "slow_update_content": "Check units first."} + assert meta == {"reasoning": "", "meta_skill_content": "Rank by support."}