Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ All notable changes to SkillOpt are documented here. This project adheres to
instead of treating them as missing, and load integral floats such as
`1776.0` as `1776`, so a correct integer answer is no longer scored wrong
(thanks @Shizoqua).
- Keep slow-update guidance and optimizer meta-skill memory as text when the
optimizer returns them as a list of strings, instead of writing the list's
Python repr into the skill. Non-text or blank content is treated as no
update (thanks @Shizoqua).

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
9 changes: 5 additions & 4 deletions skillopt/optimizer/meta_skill.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import traceback

from skillopt.model import chat_optimizer
from skillopt.optimizer.slow_update import format_comparison_text
from skillopt.optimizer.slow_update import content_text, format_comparison_text
from skillopt.prompts import load_prompt
from skillopt.utils import extract_json

Expand Down Expand Up @@ -68,10 +68,11 @@ def run_meta_skill(
stage="meta_skill",
)
result = extract_json(response)
if result and result.get("meta_skill_content"):
content = content_text(result.get("meta_skill_content")) if result else ""
if content:
return {
"reasoning": str(result.get("reasoning", "")).strip(),
"meta_skill_content": str(result["meta_skill_content"]).strip(),
"reasoning": content_text(result.get("reasoning")),
"meta_skill_content": content,
}
except Exception: # noqa: BLE001
traceback.print_exc()
Expand Down
21 changes: 18 additions & 3 deletions skillopt/optimizer/slow_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,20 @@
# ── Field manipulation helpers ──────────────────────────────────────────────


def content_text(value: object) -> str:
"""Return a model-provided text field as stripped text, or ``""`` if unusable.

A list of strings (a common model deviation for bullet-style guidance) is
joined one item per line. Anything else that is not a string, such as
``None``, a dict or a number, is unusable rather than written as its repr.
"""
if isinstance(value, str):
return value.strip()
if isinstance(value, list) and all(isinstance(item, str) for item in value):
return "\n".join(item.strip() for item in value if item.strip())
return ""


def has_slow_update_field(skill: str) -> bool:
return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill

Expand Down Expand Up @@ -432,10 +446,11 @@ def run_slow_update(
stage="slow_update",
)
result = extract_json(response)
if result and result.get("slow_update_content"):
content = content_text(result.get("slow_update_content")) if result else ""
if content:
return {
"reasoning": str(result.get("reasoning", "")).strip(),
"slow_update_content": str(result["slow_update_content"]).strip(),
"reasoning": content_text(result.get("reasoning")),
"slow_update_content": content,
}
except Exception: # noqa: BLE001
traceback.print_exc()
Expand Down
52 changes: 52 additions & 0 deletions tests/test_slow_update_content.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Slow-update and meta-skill outputs must be text, never a Python repr of a list."""

from __future__ import annotations

import json

import pytest

from skillopt.optimizer import meta_skill, slow_update


def _slow_update(monkeypatch, payload: dict):
monkeypatch.setattr(slow_update, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {}))
return slow_update.run_slow_update("# Skill", [], [], [], comparison_pairs=[], system_prompt="slow")


def _meta_skill(monkeypatch, payload: dict):
monkeypatch.setattr(meta_skill, "chat_optimizer", lambda **kwargs: (json.dumps(payload), {}))
return meta_skill.run_meta_skill("# Prev", "# Curr", [], system_prompt="meta")


def test_slow_update_list_content_is_joined_into_lines(monkeypatch):
result = _slow_update(
monkeypatch, {"reasoning": "r", "slow_update_content": ["Check units first.", " Prefer exact spans. "]}
)

assert result["slow_update_content"] == "Check units first.\nPrefer exact spans."
skill = slow_update.replace_slow_update_field("# Skill", result["slow_update_content"])
assert "['" not in skill
assert slow_update.extract_slow_update_field(skill) == "Check units first.\nPrefer exact spans."


def test_meta_skill_list_content_is_joined_into_lines(monkeypatch):
result = _meta_skill(
monkeypatch, {"reasoning": "r", "meta_skill_content": ["Rank by support.", "Merge duplicates."]}
)

assert result["meta_skill_content"] == "Rank by support.\nMerge duplicates."


@pytest.mark.parametrize("content", [{"text": "x"}, 42, ["ok", 3], [], ["", " "], " "])
def test_unusable_content_gives_no_update(monkeypatch, content):
assert _slow_update(monkeypatch, {"reasoning": "r", "slow_update_content": content}) is None
assert _meta_skill(monkeypatch, {"reasoning": "r", "meta_skill_content": content}) is None


def test_string_content_and_null_reasoning(monkeypatch):
slow = _slow_update(monkeypatch, {"reasoning": None, "slow_update_content": " Check units first. "})
meta = _meta_skill(monkeypatch, {"reasoning": None, "meta_skill_content": "Rank by support."})

assert slow == {"reasoning": "", "slow_update_content": "Check units first."}
assert meta == {"reasoning": "", "meta_skill_content": "Rank by support."}