diff --git a/CHANGELOG.md b/CHANGELOG.md index bc704e06..a63a8c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,11 @@ All notable changes to SkillOpt are documented here. This project adheres to decisions and accepted edits are unchanged even when the token budget binds. A single-edit candidate needs no extra replay; an N-edit candidate costs up to N validation replays, which are included in `tokens used` and checked - against `max_tokens_per_night` one replay at a time. Follow-up to #174. + against `max_tokens_per_night` one replay at a time. An ablation replay that + fails leaves its rows `unmeasured` and is logged as an + `edit_ablation_failed` event instead of failing the night, and each ablation + event names its fan-out group (`skill`), `night` and `trial_index`. + Follow-up to #174. - **OfficeQA Parallel Search MCP** as an opt-in `env.search_mode=parallel` provider for `openai_chat`/`qwen_chat`. The default stays `offline` and no config selects Parallel automatically. Selecting it sends the generated diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 726dffc9..336bb4f3 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -69,6 +69,21 @@ the key has no effect. Rows are included in `report.md`, `report.json`, `diagnostics.json`, the CLI's `--json` output, and the evidence log; in multi-skill fan-out, each group's rows are in the evidence log. +An ablation replay that fails does not fail the night, because every decision +is already final when it runs. Its row is `unmeasured` with the error in its +note, the ablation replays still to come in that consolidation do not start +and say so in their notes, and a `gate` / `edit_ablation_failed` event is +written to the evidence log. The night is staged as it would be with the key +off. Errors that pause or stop the whole cycle (pending handoff calls, the +Cursor backend's fail-closed errors, staging errors) still do. + +Ablation events are written after every decision, so in multi-skill fan-out +they sit together at the end of the evidence log instead of next to their +group's other events. Each `edit_ablation` and `edit_ablation_failed` event +therefore carries `skill` (the fan-out group's skill name, empty for the +night's aggregate run), `night`, and `trial_index` (the position of its trial +among that run's gate trials). + Rule-based heading checks are intentionally additive. `section_present` keeps its legacy strict behavior for existing task sets: an ATX heading cannot append text after the requested name. Use `section_contains` when a numbered, diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index e85928af..f594180d 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -12,13 +12,15 @@ from dataclasses import dataclass, field from typing import Callable, List, Optional, Tuple -from skillopt_sleep.backend import Backend +from skillopt_sleep.backend import Backend, CursorBackendError # Self-contained validation gate (vendored from SkillOpt; zero dependency on the # research package, so this open-source tool stays decoupled from the paper code). from skillopt_sleep.gate import evaluate_gate, select_gate_score +from skillopt_sleep.handoff_backend import PendingCalls from skillopt_sleep.memory import apply_edits_detailed from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.staging import StagingError from skillopt_sleep.types import ( EditRecord, ReplayResult, @@ -280,6 +282,7 @@ def consolidate( night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, edit_ablation_queue: Optional[List[Callable[[], bool]]] = None, + skill_name: str = "", ) -> ConsolidationResult: """Run one consolidation epoch: reflect -> bounded edit -> gate. @@ -313,6 +316,20 @@ def consolidate( ``edit_ablation_queue``: this epoch's ablation is appended to it instead of being run here, and the caller runs it with :func:`run_edit_ablations` once every consolidation has decided. + + An ablation that raises does not cost the epoch its result, on either + path: the decision is already made. The row being measured is + ``unmeasured`` with a note, a ``gate`` / ``edit_ablation_failed`` event is + logged, and no further ablation replay of this epoch starts. Only + ``PendingCalls``, ``CursorBackendError`` and ``StagingError`` propagate, + as they do from a group in ``consolidate_groups``. + + ``skill_name`` labels this epoch's ablation events. They are logged after + every decision, so in fan-out they sit at the end of the evidence log, + away from their group's other events. Each one carries ``skill``, + ``night`` and ``trial_index`` (the trial's position in ``gate_trials``), + so it can be matched to its trial without comparing edit content. Leave + ``skill_name`` empty for a run that is not a per-skill group. """ from skillopt_sleep import evidence as evlog ev = evlog.get(backend) @@ -387,7 +404,9 @@ def _edits_payload(edits: List[EditRecord]) -> List[dict]: # memory step, the final validation replay or a later epoch still checks. pending_ablations: List[dict] = [] - def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: + def _ablate_edits( + plan: dict, budget_spent: bool, failure: str + ) -> Tuple[List[dict], bool, str]: """Leave-one-out replay of one accepted trial. For each applied edit, rebuild the candidate from the pre-trial @@ -399,10 +418,24 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: The budget is checked before every replay. Returns the rows and whether the budget is spent, so a later trial does not start a replay the budget no longer covers. + + The epoch has already decided, so an error here must not cost it its + result. A row whose replay or scoring raises is ``unmeasured``, the + error is logged, and no further replay starts: ``failure`` carries the + error to the rows and trials that follow and is returned with the + rows. ``PendingCalls``, ``CursorBackendError`` and ``StagingError`` + still propagate, as the cycle-level control flow they are. """ which = plan["which"] doc, new_doc, applied = plan["doc"], plan["new_doc"], plan["applied"] with_pairs, with_score = plan["with_pairs"], plan["with_score"] + # Where this trial sits in the epoch's gate_trials. Ablation events are + # logged after every decision (in fan-out, after every group), so they + # name their trial instead of relying on their place in the log. + trial_index = next( + (i for i, trial in enumerate(gate_trials) if trial is plan["trial"]), + None, + ) rows: List[dict] = [] for index, edit in enumerate(applied): remaining = [e for j, e in enumerate(applied) if j != index] @@ -419,43 +452,60 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: ] replayed = False note = "" - without_pairs: Optional[List[Tuple[TaskRecord, ReplayResult]]] - if not still_applied: - without_pairs = plan["prior_pairs"] - note = "scored by the replay that preceded this trial" - elif without_doc == new_doc: - without_pairs = with_pairs - note = "document is the same without this edit" - else: - if not budget_spent and budget_exhausted is not None: - budget_spent = bool(budget_exhausted()) - if budget_spent and ev is not None: - ev.log("budget", "skip", step="edit ablation", - night=night) - if budget_spent: + without_score: Optional[float] = None + task_rows: List[dict] = [] + try: + without_pairs: Optional[List[Tuple[TaskRecord, ReplayResult]]] + if not still_applied: + without_pairs = plan["prior_pairs"] + note = "scored by the replay that preceded this trial" + elif without_doc == new_doc: + without_pairs = with_pairs + note = "document is the same without this edit" + elif failure: without_pairs = None - note = "not replayed: token budget spent" + note = "not replayed: ablation stopped after a failure" else: - evlog.set_phase(backend, f"gate_ablation:{which}") - without_pairs = replay_batch( - backend, - val_tasks, - without_doc if which == "skill" else plan["skill"], - without_doc if which == "memory" else plan["memory"], + if not budget_spent and budget_exhausted is not None: + budget_spent = bool(budget_exhausted()) + if budget_spent and ev is not None: + ev.log("budget", "skip", step="edit ablation", + night=night) + if budget_spent: + without_pairs = None + note = "not replayed: token budget spent" + else: + evlog.set_phase(backend, f"gate_ablation:{which}") + without_pairs = replay_batch( + backend, + val_tasks, + without_doc if which == "skill" else plan["skill"], + without_doc if which == "memory" else plan["memory"], + ) + replayed = True + if without_pairs is not None: + task_rows = _task_deltas( + val_tasks, without_pairs, with_pairs, + gate_metric, gate_mixed_weight, ) - replayed = True - if without_pairs is None: - without_score = None - task_rows: List[dict] = [] - else: - h, s = aggregate_scores(without_pairs) - without_score = select_gate_score( - h, s, gate_metric, gate_mixed_weight - ) - task_rows = _task_deltas( - val_tasks, without_pairs, with_pairs, - gate_metric, gate_mixed_weight, - ) + # Assigned last: a row whose scoring raised has no score, + # which is what reports it as unmeasured below. + h, s = aggregate_scores(without_pairs) + without_score = select_gate_score( + h, s, gate_metric, gate_mixed_weight + ) + except (PendingCalls, CursorBackendError, StagingError): + # Cycle-level control flow (a paused handoff night, Cursor's + # fail-closed path), not evidence about one edit. + raise + except Exception as exc: + failure = f"{type(exc).__name__}: {exc}"[:300] + note = f"ablation failed: {failure}" + if ev is not None: + ev.log("gate", "edit_ablation_failed", target=which, + skill=skill_name, night=night, + trial_index=trial_index, edit_index=index, + error=failure) observed = _ablation_observed_change( with_score, without_score, task_rows ) @@ -488,19 +538,25 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: }) if ev is not None: ev.log("gate", "edit_ablation", target=which, + skill=skill_name, night=night, trial_index=trial_index, n_edits=len(applied), n_replays=sum(1 for row in rows if row["replayed"]), rows=rows) - return rows, budget_spent + return rows, budget_spent, failure def _run_pending_ablations() -> bool: """Ablate every accepted trial of this epoch, in trial order. Returns True when a replay was skipped because the budget was spent. + A failed ablation is not a budget skip and is not raised: it is + reported on its rows and in the evidence log. """ budget_spent = False + failure = "" for plan in pending_ablations: - rows, budget_spent = _ablate_edits(plan, budget_spent) + rows, budget_spent, failure = _ablate_edits( + plan, budget_spent, failure + ) plan["trial"]["edit_ablation"] = rows return budget_spent diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index 8446038a..800ca6f3 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -1202,6 +1202,9 @@ def budget_exhausted() -> bool: # Every decision of the night is final here. The replays below are # ordinary model calls: their tokens are part of "tokens used" and count # toward max_tokens_per_night, which is checked before each of them. + # A replay that fails is contained by the queued run itself: its rows are + # unmeasured and the night is staged as decided. Only cycle-level errors + # reach this handler. if ablation_queue: _progress(cfg, "edit ablation start") try: diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 84f48caa..6d9d1c04 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -148,6 +148,7 @@ def dream_consolidate( night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, edit_ablation_queue: Optional[List[Callable[[], bool]]] = None, + skill_name: str = "", ) -> ConsolidationResult: """Recall similar past experience + dream synthetic variants, then run one gated consolidation epoch over the enlarged training pool. @@ -156,8 +157,8 @@ def dream_consolidate( augmentation only enlarge the TRAIN split, so the val slice the gate scores on is never polluted. With ``recall_k=0`` and ``dream_rollouts=1`` (the defaults) this is exactly the previous single-shot ``consolidate``. - ``budget_exhausted`` and ``edit_ablation_queue`` are passed through to - ``consolidate``. + ``budget_exhausted``, ``edit_ablation_queue`` and ``skill_name`` are passed + through to ``consolidate``. """ train = [t for t in tasks if t.split == "train"] enlarged = list(tasks) @@ -189,4 +190,5 @@ def dream_consolidate( evolve_memory=evolve_memory, night=night, budget_exhausted=budget_exhausted, edit_ablation_queue=edit_ablation_queue, + skill_name=skill_name, ) diff --git a/skillopt_sleep/multi_skill.py b/skillopt_sleep/multi_skill.py index 5645592d..41cba37a 100644 --- a/skillopt_sleep/multi_skill.py +++ b/skillopt_sleep/multi_skill.py @@ -91,8 +91,10 @@ def consolidate_groups( last group has decided, so its replays cannot use budget a later group still needs. A caller with further budget-dependent work passes its own ``edit_ablation_queue`` and runs it afterwards; otherwise the ablations run - here, after the loop. Either way a skipped ablation never changes a - group's status or reason. + here, after the loop. Either way a skipped or failed ablation never changes + a group's status or reason. Each group's name is passed to + ``consolidate_fn`` as ``skill_name``, so its ablation events can be told + apart from the other groups'. """ # This wrapper's contract is stricter than consolidate(): shared memory is # always read-only. Override a caller-supplied value instead of passing a @@ -134,6 +136,11 @@ def consolidate_groups( continue try: group_kwargs = dict(consolidate_kwargs) + if consolidate_kwargs.get("gate_edit_ablation"): + # Ablation events are logged once every group has decided, at + # the end of the evidence log; the name ties each one back to + # its group. + group_kwargs["skill_name"] = name if group_kwargs_fn is not None: group_kwargs.update(group_kwargs_fn(group)) # A per-group factory cannot weaken the shared-memory invariant. diff --git a/tests/test_gate_edit_ablation.py b/tests/test_gate_edit_ablation.py index d1b2f31d..d381f45d 100644 --- a/tests/test_gate_edit_ablation.py +++ b/tests/test_gate_edit_ablation.py @@ -9,15 +9,18 @@ import pytest +from skillopt_sleep import evidence as evidence_log from skillopt_sleep.__main__ import _report_payload -from skillopt_sleep.backend import Backend, MockBackend +from skillopt_sleep.backend import Backend, CursorBackendError, MockBackend from skillopt_sleep.config import DEFAULTS, load_config from skillopt_sleep.consolidate import run_edit_ablations from skillopt_sleep.cycle import _render_report_md, run_sleep_cycle from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona +from skillopt_sleep.handoff_backend import PendingCalls from skillopt_sleep.memory import set_learned from skillopt_sleep.mine import assign_splits -from skillopt_sleep.multi_skill import SkillGroup, consolidate_groups +from skillopt_sleep.multi_skill import CONSOLIDATED, SkillGroup, consolidate_groups +from skillopt_sleep.staging import StagingError from skillopt_sleep.types import EditRecord, ReplayResult, SleepReport, TaskRecord RULE_A = "Rule A: answer the question that was asked." @@ -1199,3 +1202,533 @@ def test_held_out_test_replay_keeps_its_budget(tmp_path) -> None: ("gate", "edit_ablation") ) assert free.report.tokens_used > spend + + +# ── a failed ablation never costs a decided night its result ───────────────── + +FAILED_NOTE = "ablation failed: RuntimeError: backend down" +STOPPED_NOTE = "not replayed: ablation stopped after a failure" + + +def _ablation_replays_raise(error: BaseException, scorer=_free_rider_scores): + """Score like ``scorer``, but raise ``error`` in every ablation replay. + + An ablation replay is the only validation replay that scores a skill with + exactly one of Rule A and Rule B: the baseline has neither, and the trial + and final replays have both. + """ + + def scores(task_id: str, skill: str, memory: str) -> float: + if task_id != "train" and (RULE_A in skill) != (RULE_B in skill): + raise error + return scorer(task_id, skill, memory) + + return scores + + +def test_failed_ablation_replay_leaves_the_decision_and_marks_the_rows( + monkeypatch, +) -> None: + _patch_replay(monkeypatch, _free_rider_scores) + off = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=False) + calls = _patch_replay( + monkeypatch, _ablation_replays_raise(RuntimeError("backend down")) + ) + + # No queue: the epoch runs its own ablation, after its decision. + on = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + assert on.accepted is True + _assert_same_decisions(on, off) + failed, stopped = _skill_trial(on)["edit_ablation"] + assert failed["note"] == FAILED_NOTE + assert stopped["note"] == STOPPED_NOTE + for row in (failed, stopped): + assert row["observed_change"] == "unmeasured" + assert row["replayed"] is False + assert row["score_without"] is None + assert row["task_changes"] == [] + # Baseline, trial, final, and the one ablation replay that raised: the + # second ablation replay never started. + assert len(_val_replays(calls)) == 4 + # A failure is not a budget skip. + assert on.budget_skipped == [] + json.dumps(on.gate_trials, allow_nan=False) + + +def test_queued_ablation_failure_is_not_raised_to_the_caller(monkeypatch) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("backend down"))) + queue: list = [] + result = _consolidate( + [_add(RULE_A), _add(RULE_B)], + gate_edit_ablation=True, + edit_ablation_queue=queue, + ) + + # The caller's queue run neither raises nor reports a budget skip. + assert run_edit_ablations(queue) is False + + assert queue == [] + assert result.accepted is True + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + + +def test_rows_that_need_no_replay_are_still_measured_after_a_failure( + monkeypatch, +) -> None: + _patch_replay( + monkeypatch, + _ablation_replays_raise( + RuntimeError("backend down"), scorer=_skill_and_memory_scores + ), + ) + + result = _consolidate( + [_add(RULE_A), _add(RULE_B), _add(MEMORY_RULE, target="memory")], + evolve_memory=True, + gate_edit_ablation=True, + ) + + assert result.accepted is True and MEMORY_RULE in result.new_memory + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + memory_trial = next(t for t in result.gate_trials if t["target"] == "memory") + (memory_row,) = memory_trial["edit_ablation"] + # The failure stops further replays, not the rows that never needed one. + assert memory_row["replayed"] is False + assert memory_row["observed_change"] == "higher_with_edit" + + +def test_failure_in_one_trial_stops_the_replays_of_the_next(monkeypatch) -> None: + second_memory_rule = "Memory rule: note which sources were checked." + + def scores(task_id: str, skill: str, memory: str) -> float: + if task_id == "essay": + return 1.0 if RULE_A in skill else 0.0 + if task_id == "router": + return 0.9 if MEMORY_RULE in memory else 0.8 + return 0.0 + + calls = _patch_replay( + monkeypatch, + _ablation_replays_raise(RuntimeError("backend down"), scorer=scores), + ) + + result = _consolidate( + [ + _add(RULE_A), + _add(RULE_B), + _add(MEMORY_RULE, target="memory"), + _add(second_memory_rule, target="memory"), + ], + evolve_memory=True, + gate_edit_ablation=True, + ) + + assert result.accepted is True + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + memory_trial = next(t for t in result.gate_trials if t["target"] == "memory") + memory_rows = memory_trial["edit_ablation"] + # The memory trial is ablated after the skill trial, whose replay failed: + # neither of its two replays starts. + assert [row["note"] for row in memory_rows] == [STOPPED_NOTE, STOPPED_NOTE] + assert all(row["replayed"] is False for row in memory_rows) + assert not [ + call for call in calls + if (MEMORY_RULE in call["memory"]) != (second_memory_rule in call["memory"]) + ] + + +def test_failed_ablation_note_is_bounded(monkeypatch) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("x" * 5000))) + + result = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + failed = _skill_trial(result)["edit_ablation"][0] + assert failed["note"].startswith("ablation failed: RuntimeError: xxx") + assert len(failed["note"]) < 400 + + +def test_failure_stops_the_ablation_replays_that_follow(monkeypatch) -> None: + ablation_replays = {"count": 0} + + def scores(task_id: str, skill: str, memory: str) -> float: + rules = [rule in skill for rule in (RULE_A, RULE_B, RULE_C, RULE_D)] + if task_id == "essay" and any(rules) and not all(rules): + ablation_replays["count"] += 1 + if ablation_replays["count"] == 2: + raise RuntimeError("backend down") + return _free_rider_scores(task_id, skill, memory) + + calls = _patch_replay(monkeypatch, scores) + + result = _consolidate( + [_add(RULE_A), _add(RULE_B), _add(RULE_C), _add(RULE_D)], + gate_edit_ablation=True, + ) + + assert result.accepted is True + rows = _skill_trial(result)["edit_ablation"] + # The first replay finished and keeps its row; the second raised; the + # third and fourth never started. + assert [row["replayed"] for row in rows] == [True, False, False, False] + assert [row["observed_change"] for row in rows] == [ + "higher_with_edit", + "unmeasured", + "unmeasured", + "unmeasured", + ] + assert [row["note"] for row in rows] == ["", FAILED_NOTE, STOPPED_NOTE, STOPPED_NOTE] + assert len(_val_replays(calls)) == 5 + + +@pytest.mark.parametrize("missing", ["hard", "soft"]) +def test_ablation_result_without_a_numeric_score_is_unmeasured( + monkeypatch, missing +) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + + def fake_replay_batch(backend, tasks, skill, memory): + ablation = (RULE_A in skill) != (RULE_B in skill) + pairs = [] + for task in tasks: + score = _free_rider_scores(task.id, skill, memory) + result = ReplayResult(id=task.id, hard=score, soft=score, response="r") + if ablation and task.id == "router": + # Before a decision, an absent numeric score aborts the + # evaluation. After it, the row is all it can cost. A missing + # hard score fails the per-task comparison, a missing soft + # score the aggregate that follows it. + setattr(result, missing, None) + pairs.append((task, result)) + return pairs + + monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch) + + result = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + assert result.accepted is True + failed, stopped = _skill_trial(result)["edit_ablation"] + # The replay itself finished; scoring its result is what raised. + assert failed["replayed"] is True + assert failed["observed_change"] == "unmeasured" + assert failed["score_without"] is None + assert failed["task_changes"] == [] + assert failed["note"].startswith("ablation failed: TypeError") + assert stopped["note"] == STOPPED_NOTE + json.dumps(result.gate_trials, allow_nan=False) + + +@pytest.mark.parametrize( + "error", + [ + PendingCalls({"call-1": {"prompt": "replay"}}), + CursorBackendError("cursor refused"), + StagingError("staging refused"), + KeyboardInterrupt(), + ], + ids=lambda error: type(error).__name__, +) +def test_control_flow_errors_still_leave_the_ablation(monkeypatch, error) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(error)) + + with pytest.raises(type(error)): + _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + +def test_one_group_failing_its_ablation_leaves_every_group_decided( + monkeypatch, +) -> None: + off, _off_calls = _consolidate_two_groups(monkeypatch, ablation=False) + calls = _patch_replay( + monkeypatch, _ablation_replays_raise(RuntimeError("backend down")) + ) + + on = consolidate_groups( + ScriptedBackend([_add(RULE_A), _add(RULE_B)]), + _two_groups(), + "", + gate_metric="hard", + gate_edit_ablation=True, + ) + + assert list(on) == list(off) + for name, expected in off.items(): + outcome = on[name] + assert outcome.status == expected.status == CONSOLIDATED + assert outcome.reason == expected.reason == "" + _assert_same_decisions(outcome.result, expected.result) + assert [ + row["note"] for row in _skill_trial(outcome.result)["edit_ablation"] + ] == [FAILED_NOTE, STOPPED_NOTE] + # Each group stops at its own first failure: one attempt per group. + assert sum(_is_ablation_replay(call) for call in calls) == 2 + + +class AblationFailingMock(MockBackend): + """MockBackend whose model calls raise during edit-ablation replays only.""" + + def attempt(self, *args, **kwargs): + if evidence_log.phase(self).startswith("gate_ablation:"): + raise RuntimeError("backend down") + return super().attempt(*args, **kwargs) + + +def _failing_night(tmp_path, name: str, *, ablation: bool, fanout: bool = False): + root = tmp_path / name + project = root / "project" + project.mkdir(parents=True) + claude_home = root / ".claude" + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + fanout_config = {} + if fanout: + for skill_name in ("research-skill", "programming-skill"): + skill_path = claude_home / "skills" / skill_name / "SKILL.md" + skill_path.parent.mkdir(parents=True) + skill_path.write_text(f"# {skill_name}\n", encoding="utf-8") + programming = assign_splits( + programmer_persona(), holdout_fraction=0.34, seed=1 + ) + tasks = [replace(task, skill_hint="research-skill") for task in tasks] + tasks += [ + replace(task, id=f"prog-{task.id}", skill_hint="programming-skill") + for task in programming + ] + fanout_config = { + "managed_skill_name": "skillopt-sleep-learned", + "multi_skill_fanout": True, + } + config = load_config( + invoked_project=str(project), + projects="invoked", + backend="mock", + state_dir=str(root / "state"), + claude_home=str(claude_home), + gate_edit_ablation=ablation, + auto_adopt=False, + **fanout_config, + ) + return run_sleep_cycle(config, seed_tasks=tasks, backend=AblationFailingMock()) + + +def _staged_proposals(outcome) -> dict: + proposals = {} + for file_name in sorted(os.listdir(outcome.staging_dir)): + if file_name.startswith("proposed_"): + with open( + os.path.join(outcome.staging_dir, file_name), encoding="utf-8" + ) as handle: + proposals[file_name] = handle.read() + return proposals + + +def _events(outcome, name: str) -> list[dict]: + return [ + event + for event in evidence_log.read_events( + os.path.join(outcome.staging_dir, "evidence.jsonl") + ) + if event["event"] == name + ] + + +def test_night_is_staged_when_an_ablation_replay_fails(tmp_path) -> None: + off = _failing_night(tmp_path, "off", ablation=False) + on = _failing_night(tmp_path, "on", ablation=True) + + # The night used to raise here and stage nothing but its evidence log. + assert off.report.accepted is True + assert _night_decisions(on) == _night_decisions(off) + assert _staged_proposals(off) + assert _staged_proposals(on) == _staged_proposals(off) + assert ("stage", "staged") in _evidence_events(on) + accepted_trial = next( + trial for trial in on.report.gate_trials + if trial["target"] != "final" and trial["accepted"] + ) + notes = [row["note"] for row in accepted_trial["edit_ablation"]] + assert notes[0] == FAILED_NOTE + assert notes[1:] and set(notes[1:]) == {STOPPED_NOTE} + (failure,) = _events(on, "edit_ablation_failed") + assert failure["stage"] == "gate" + assert failure["error"] == "RuntimeError: backend down" + assert failure["edit_index"] == 0 + # A failure is reported on its rows and in the evidence log, never as a + # step the token budget skipped. + assert _budget_notes(on) == [] + with open(os.path.join(on.staging_dir, "report.md"), encoding="utf-8") as handle: + assert FAILED_NOTE in handle.read() + + +def test_fanout_night_is_staged_when_ablation_replays_fail(tmp_path) -> None: + off = _failing_night(tmp_path, "off", ablation=False, fanout=True) + on = _failing_night(tmp_path, "on", ablation=True, fanout=True) + + assert [group.accepted for group in off.report.skill_groups] == [True, True] + assert _night_decisions(on) == _night_decisions(off) + assert len(_staged_proposals(off)) == 3 + assert _staged_proposals(on) == _staged_proposals(off) + # One failure per consolidation that had a replay to run, each named. + failures = _events(on, "edit_ablation_failed") + assert failures + assert failures[0]["skill"] == "" + group_names = {group.skill_name for group in on.report.skill_groups} + assert {failure["skill"] for failure in failures[1:]} <= group_names + assert all( + failure["error"] == "RuntimeError: backend down" for failure in failures + ) + + +# ── ablation events name the trial they describe ───────────────────────────── + + +def test_ablation_event_names_its_skill_night_and_trial(monkeypatch, tmp_path) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _skill_and_memory_scores) + backend = ScriptedBackend( + [_add(RULE_A), _add(RULE_B), _add(MEMORY_RULE, target="memory")] + ) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + result = consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + set_learned("# Memory\n", []), + gate_metric="hard", + gate_edit_ablation=True, + night=7, + skill_name="alpha", + ) + + events = [ + event + for event in evidence_log.read_events(str(log_path)) + if event["event"] == "edit_ablation" + ] + assert [event["target"] for event in events] == ["skill", "memory"] + for event in events: + assert event["skill"] == "alpha" + assert event["night"] == 7 + # trial_index points at the trial that holds these rows. + trial = result.gate_trials[event["trial_index"]] + assert trial["target"] == event["target"] + assert [row["content"] for row in trial["edit_ablation"]] == [ + row["content"] for row in event["rows"] + ] + + +def test_ablation_event_of_an_unnamed_run_has_an_empty_skill( + monkeypatch, tmp_path +) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _free_rider_scores) + backend = ScriptedBackend([_add(RULE_A), _add(RULE_B)]) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + "", + gate_metric="hard", + evolve_memory=False, + gate_edit_ablation=True, + ) + + (event,) = [ + event + for event in evidence_log.read_events(str(log_path)) + if event["event"] == "edit_ablation" + ] + assert event["skill"] == "" + assert event["night"] == 1 + assert event["trial_index"] == 0 + + +def test_failed_ablation_event_names_its_trial_and_edit(monkeypatch, tmp_path) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("backend down"))) + backend = ScriptedBackend([_add(RULE_A), _add(RULE_B)]) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + "", + gate_metric="hard", + evolve_memory=False, + gate_edit_ablation=True, + night=3, + skill_name="alpha", + ) + + events = evidence_log.read_events(str(log_path)) + (failure,) = [e for e in events if e["event"] == "edit_ablation_failed"] + assert failure["stage"] == "gate" + assert failure["target"] == "skill" + assert failure["skill"] == "alpha" + assert failure["night"] == 3 + assert failure["trial_index"] == 0 + assert failure["edit_index"] == 0 + assert failure["error"] == "RuntimeError: backend down" + # The trial still gets its one edit_ablation event, with every row. + (rows_event,) = [e for e in events if e["event"] == "edit_ablation"] + assert rows_event["n_replays"] == 0 + assert [row["note"] for row in rows_event["rows"]] == [FAILED_NOTE, STOPPED_NOTE] + + +def test_groups_pass_their_name_only_with_ablation_on() -> None: + seen: list[dict] = [] + + def record(backend, tasks, skill, memory, **kwargs): + seen.append(kwargs) + raise RuntimeError("stop after recording") + + for ablation in (False, True): + consolidate_groups( + ScriptedBackend([]), + _two_groups(), + "", + consolidate_fn=record, + gate_edit_ablation=ablation, + ) + + off_alpha, off_beta, on_alpha, on_beta = seen + # Off by default: a custom consolidate_fn sees the arguments it always saw. + assert "skill_name" not in off_alpha and "skill_name" not in off_beta + assert on_alpha["skill_name"] == "alpha" + assert on_beta["skill_name"] == "beta" + + +def test_fanout_ablation_events_name_their_group(tmp_path) -> None: + outcome = _fanout_night(tmp_path, "night", ablation=True, max_tokens=0) + + accepted_groups = [ + group.skill_name for group in outcome.report.skill_groups if group.accepted + ] + assert len(accepted_groups) == 2 + events = _events(outcome, "edit_ablation") + # The aggregate run is not a per-skill group; each group's event carries + # the group's name, so it no longer has to be matched by edit content. + assert [event["skill"] for event in events] == ["", *accepted_groups] + assert all(event["night"] == outcome.report.night for event in events) + assert all(isinstance(event["trial_index"], int) for event in events)