From a0d3c4181995f19168b018ee23e50d6e048a8368 Mon Sep 17 00:00:00 2001 From: Intelligence Research Project <52513440+sunghunkwag@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:14:23 +0900 Subject: [PATCH] fix(sleep): keep a decided night when an edit-ablation replay fails Follow-up to #390, for the two points left open in its review. A failed ablation replay could abort a night whose decisions were final. `consolidate_groups` swallows ordinary errors from its own ablation queue, but `run_sleep_cycle` passes its own queue and step 4c only handles `CursorBackendError`, and `consolidate()` without a queue raised after deciding. With a backend that raises `RuntimeError` only during `gate_ablation:*` calls, the night raised and staged nothing but its evidence log, while the same night with the option off staged an accepted skill. The failure is now contained where the rows are built, in the ablation run that the queue path, the no-queue path and `consolidate_groups` share: - A row whose replay or scoring raises is `unmeasured`, with the error in its note. Rows that finished keep their result. - No further ablation replay of that consolidation starts, and those rows say so in their note. Rows that need no replay are still filled in. - A `gate` / `edit_ablation_failed` event is logged. - `PendingCalls`, `CursorBackendError` and `StagingError` propagate as before. `KeyboardInterrupt` is not an `Exception` and is not caught. - A failure is not a budget skip: it adds nothing to `budget_skipped`. Ablation events now say which trial they describe. They are logged after every decision, so in fan-out each group's `gate` / `edit_ablation` event sits at the end of the log and could only be matched to its group by edit content. Both ablation events carry `skill` (the group's name, empty for the night's aggregate run), `night` and `trial_index`. `consolidate` and `dream_consolidate` take the name as `skill_name`; `consolidate_groups` passes it only when `gate_edit_ablation` is on, so a custom `consolidate_fn` is otherwise called with the arguments it always was. With the option off nothing changes. With it on and no failure, report.md, report.json, diagnostics.json, the proposed skills and the token total are identical to main. --- CHANGELOG.md | 6 +- docs/sleep/README.md | 15 + skillopt_sleep/consolidate.py | 132 +++++--- skillopt_sleep/cycle.py | 3 + skillopt_sleep/dream.py | 6 +- skillopt_sleep/multi_skill.py | 11 +- tests/test_gate_edit_ablation.py | 537 ++++++++++++++++++++++++++++++- 7 files changed, 665 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc704e06..a63a8c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,11 @@ All notable changes to SkillOpt are documented here. This project adheres to decisions and accepted edits are unchanged even when the token budget binds. A single-edit candidate needs no extra replay; an N-edit candidate costs up to N validation replays, which are included in `tokens used` and checked - against `max_tokens_per_night` one replay at a time. Follow-up to #174. + against `max_tokens_per_night` one replay at a time. An ablation replay that + fails leaves its rows `unmeasured` and is logged as an + `edit_ablation_failed` event instead of failing the night, and each ablation + event names its fan-out group (`skill`), `night` and `trial_index`. + Follow-up to #174. - **OfficeQA Parallel Search MCP** as an opt-in `env.search_mode=parallel` provider for `openai_chat`/`qwen_chat`. The default stays `offline` and no config selects Parallel automatically. Selecting it sends the generated diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 726dffc9..336bb4f3 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -69,6 +69,21 @@ the key has no effect. Rows are included in `report.md`, `report.json`, `diagnostics.json`, the CLI's `--json` output, and the evidence log; in multi-skill fan-out, each group's rows are in the evidence log. +An ablation replay that fails does not fail the night, because every decision +is already final when it runs. Its row is `unmeasured` with the error in its +note, the ablation replays still to come in that consolidation do not start +and say so in their notes, and a `gate` / `edit_ablation_failed` event is +written to the evidence log. The night is staged as it would be with the key +off. Errors that pause or stop the whole cycle (pending handoff calls, the +Cursor backend's fail-closed errors, staging errors) still do. + +Ablation events are written after every decision, so in multi-skill fan-out +they sit together at the end of the evidence log instead of next to their +group's other events. Each `edit_ablation` and `edit_ablation_failed` event +therefore carries `skill` (the fan-out group's skill name, empty for the +night's aggregate run), `night`, and `trial_index` (the position of its trial +among that run's gate trials). + Rule-based heading checks are intentionally additive. `section_present` keeps its legacy strict behavior for existing task sets: an ATX heading cannot append text after the requested name. Use `section_contains` when a numbered, diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index e85928af..f594180d 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -12,13 +12,15 @@ from dataclasses import dataclass, field from typing import Callable, List, Optional, Tuple -from skillopt_sleep.backend import Backend +from skillopt_sleep.backend import Backend, CursorBackendError # Self-contained validation gate (vendored from SkillOpt; zero dependency on the # research package, so this open-source tool stays decoupled from the paper code). from skillopt_sleep.gate import evaluate_gate, select_gate_score +from skillopt_sleep.handoff_backend import PendingCalls from skillopt_sleep.memory import apply_edits_detailed from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.staging import StagingError from skillopt_sleep.types import ( EditRecord, ReplayResult, @@ -280,6 +282,7 @@ def consolidate( night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, edit_ablation_queue: Optional[List[Callable[[], bool]]] = None, + skill_name: str = "", ) -> ConsolidationResult: """Run one consolidation epoch: reflect -> bounded edit -> gate. @@ -313,6 +316,20 @@ def consolidate( ``edit_ablation_queue``: this epoch's ablation is appended to it instead of being run here, and the caller runs it with :func:`run_edit_ablations` once every consolidation has decided. + + An ablation that raises does not cost the epoch its result, on either + path: the decision is already made. The row being measured is + ``unmeasured`` with a note, a ``gate`` / ``edit_ablation_failed`` event is + logged, and no further ablation replay of this epoch starts. Only + ``PendingCalls``, ``CursorBackendError`` and ``StagingError`` propagate, + as they do from a group in ``consolidate_groups``. + + ``skill_name`` labels this epoch's ablation events. They are logged after + every decision, so in fan-out they sit at the end of the evidence log, + away from their group's other events. Each one carries ``skill``, + ``night`` and ``trial_index`` (the trial's position in ``gate_trials``), + so it can be matched to its trial without comparing edit content. Leave + ``skill_name`` empty for a run that is not a per-skill group. """ from skillopt_sleep import evidence as evlog ev = evlog.get(backend) @@ -387,7 +404,9 @@ def _edits_payload(edits: List[EditRecord]) -> List[dict]: # memory step, the final validation replay or a later epoch still checks. pending_ablations: List[dict] = [] - def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: + def _ablate_edits( + plan: dict, budget_spent: bool, failure: str + ) -> Tuple[List[dict], bool, str]: """Leave-one-out replay of one accepted trial. For each applied edit, rebuild the candidate from the pre-trial @@ -399,10 +418,24 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: The budget is checked before every replay. Returns the rows and whether the budget is spent, so a later trial does not start a replay the budget no longer covers. + + The epoch has already decided, so an error here must not cost it its + result. A row whose replay or scoring raises is ``unmeasured``, the + error is logged, and no further replay starts: ``failure`` carries the + error to the rows and trials that follow and is returned with the + rows. ``PendingCalls``, ``CursorBackendError`` and ``StagingError`` + still propagate, as the cycle-level control flow they are. """ which = plan["which"] doc, new_doc, applied = plan["doc"], plan["new_doc"], plan["applied"] with_pairs, with_score = plan["with_pairs"], plan["with_score"] + # Where this trial sits in the epoch's gate_trials. Ablation events are + # logged after every decision (in fan-out, after every group), so they + # name their trial instead of relying on their place in the log. + trial_index = next( + (i for i, trial in enumerate(gate_trials) if trial is plan["trial"]), + None, + ) rows: List[dict] = [] for index, edit in enumerate(applied): remaining = [e for j, e in enumerate(applied) if j != index] @@ -419,43 +452,60 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: ] replayed = False note = "" - without_pairs: Optional[List[Tuple[TaskRecord, ReplayResult]]] - if not still_applied: - without_pairs = plan["prior_pairs"] - note = "scored by the replay that preceded this trial" - elif without_doc == new_doc: - without_pairs = with_pairs - note = "document is the same without this edit" - else: - if not budget_spent and budget_exhausted is not None: - budget_spent = bool(budget_exhausted()) - if budget_spent and ev is not None: - ev.log("budget", "skip", step="edit ablation", - night=night) - if budget_spent: + without_score: Optional[float] = None + task_rows: List[dict] = [] + try: + without_pairs: Optional[List[Tuple[TaskRecord, ReplayResult]]] + if not still_applied: + without_pairs = plan["prior_pairs"] + note = "scored by the replay that preceded this trial" + elif without_doc == new_doc: + without_pairs = with_pairs + note = "document is the same without this edit" + elif failure: without_pairs = None - note = "not replayed: token budget spent" + note = "not replayed: ablation stopped after a failure" else: - evlog.set_phase(backend, f"gate_ablation:{which}") - without_pairs = replay_batch( - backend, - val_tasks, - without_doc if which == "skill" else plan["skill"], - without_doc if which == "memory" else plan["memory"], + if not budget_spent and budget_exhausted is not None: + budget_spent = bool(budget_exhausted()) + if budget_spent and ev is not None: + ev.log("budget", "skip", step="edit ablation", + night=night) + if budget_spent: + without_pairs = None + note = "not replayed: token budget spent" + else: + evlog.set_phase(backend, f"gate_ablation:{which}") + without_pairs = replay_batch( + backend, + val_tasks, + without_doc if which == "skill" else plan["skill"], + without_doc if which == "memory" else plan["memory"], + ) + replayed = True + if without_pairs is not None: + task_rows = _task_deltas( + val_tasks, without_pairs, with_pairs, + gate_metric, gate_mixed_weight, ) - replayed = True - if without_pairs is None: - without_score = None - task_rows: List[dict] = [] - else: - h, s = aggregate_scores(without_pairs) - without_score = select_gate_score( - h, s, gate_metric, gate_mixed_weight - ) - task_rows = _task_deltas( - val_tasks, without_pairs, with_pairs, - gate_metric, gate_mixed_weight, - ) + # Assigned last: a row whose scoring raised has no score, + # which is what reports it as unmeasured below. + h, s = aggregate_scores(without_pairs) + without_score = select_gate_score( + h, s, gate_metric, gate_mixed_weight + ) + except (PendingCalls, CursorBackendError, StagingError): + # Cycle-level control flow (a paused handoff night, Cursor's + # fail-closed path), not evidence about one edit. + raise + except Exception as exc: + failure = f"{type(exc).__name__}: {exc}"[:300] + note = f"ablation failed: {failure}" + if ev is not None: + ev.log("gate", "edit_ablation_failed", target=which, + skill=skill_name, night=night, + trial_index=trial_index, edit_index=index, + error=failure) observed = _ablation_observed_change( with_score, without_score, task_rows ) @@ -488,19 +538,25 @@ def _ablate_edits(plan: dict, budget_spent: bool) -> Tuple[List[dict], bool]: }) if ev is not None: ev.log("gate", "edit_ablation", target=which, + skill=skill_name, night=night, trial_index=trial_index, n_edits=len(applied), n_replays=sum(1 for row in rows if row["replayed"]), rows=rows) - return rows, budget_spent + return rows, budget_spent, failure def _run_pending_ablations() -> bool: """Ablate every accepted trial of this epoch, in trial order. Returns True when a replay was skipped because the budget was spent. + A failed ablation is not a budget skip and is not raised: it is + reported on its rows and in the evidence log. """ budget_spent = False + failure = "" for plan in pending_ablations: - rows, budget_spent = _ablate_edits(plan, budget_spent) + rows, budget_spent, failure = _ablate_edits( + plan, budget_spent, failure + ) plan["trial"]["edit_ablation"] = rows return budget_spent diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index 8446038a..800ca6f3 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -1202,6 +1202,9 @@ def budget_exhausted() -> bool: # Every decision of the night is final here. The replays below are # ordinary model calls: their tokens are part of "tokens used" and count # toward max_tokens_per_night, which is checked before each of them. + # A replay that fails is contained by the queued run itself: its rows are + # unmeasured and the night is staged as decided. Only cycle-level errors + # reach this handler. if ablation_queue: _progress(cfg, "edit ablation start") try: diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 84f48caa..6d9d1c04 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -148,6 +148,7 @@ def dream_consolidate( night: int = 1, budget_exhausted: Optional[Callable[[], bool]] = None, edit_ablation_queue: Optional[List[Callable[[], bool]]] = None, + skill_name: str = "", ) -> ConsolidationResult: """Recall similar past experience + dream synthetic variants, then run one gated consolidation epoch over the enlarged training pool. @@ -156,8 +157,8 @@ def dream_consolidate( augmentation only enlarge the TRAIN split, so the val slice the gate scores on is never polluted. With ``recall_k=0`` and ``dream_rollouts=1`` (the defaults) this is exactly the previous single-shot ``consolidate``. - ``budget_exhausted`` and ``edit_ablation_queue`` are passed through to - ``consolidate``. + ``budget_exhausted``, ``edit_ablation_queue`` and ``skill_name`` are passed + through to ``consolidate``. """ train = [t for t in tasks if t.split == "train"] enlarged = list(tasks) @@ -189,4 +190,5 @@ def dream_consolidate( evolve_memory=evolve_memory, night=night, budget_exhausted=budget_exhausted, edit_ablation_queue=edit_ablation_queue, + skill_name=skill_name, ) diff --git a/skillopt_sleep/multi_skill.py b/skillopt_sleep/multi_skill.py index 5645592d..41cba37a 100644 --- a/skillopt_sleep/multi_skill.py +++ b/skillopt_sleep/multi_skill.py @@ -91,8 +91,10 @@ def consolidate_groups( last group has decided, so its replays cannot use budget a later group still needs. A caller with further budget-dependent work passes its own ``edit_ablation_queue`` and runs it afterwards; otherwise the ablations run - here, after the loop. Either way a skipped ablation never changes a - group's status or reason. + here, after the loop. Either way a skipped or failed ablation never changes + a group's status or reason. Each group's name is passed to + ``consolidate_fn`` as ``skill_name``, so its ablation events can be told + apart from the other groups'. """ # This wrapper's contract is stricter than consolidate(): shared memory is # always read-only. Override a caller-supplied value instead of passing a @@ -134,6 +136,11 @@ def consolidate_groups( continue try: group_kwargs = dict(consolidate_kwargs) + if consolidate_kwargs.get("gate_edit_ablation"): + # Ablation events are logged once every group has decided, at + # the end of the evidence log; the name ties each one back to + # its group. + group_kwargs["skill_name"] = name if group_kwargs_fn is not None: group_kwargs.update(group_kwargs_fn(group)) # A per-group factory cannot weaken the shared-memory invariant. diff --git a/tests/test_gate_edit_ablation.py b/tests/test_gate_edit_ablation.py index d1b2f31d..d381f45d 100644 --- a/tests/test_gate_edit_ablation.py +++ b/tests/test_gate_edit_ablation.py @@ -9,15 +9,18 @@ import pytest +from skillopt_sleep import evidence as evidence_log from skillopt_sleep.__main__ import _report_payload -from skillopt_sleep.backend import Backend, MockBackend +from skillopt_sleep.backend import Backend, CursorBackendError, MockBackend from skillopt_sleep.config import DEFAULTS, load_config from skillopt_sleep.consolidate import run_edit_ablations from skillopt_sleep.cycle import _render_report_md, run_sleep_cycle from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona +from skillopt_sleep.handoff_backend import PendingCalls from skillopt_sleep.memory import set_learned from skillopt_sleep.mine import assign_splits -from skillopt_sleep.multi_skill import SkillGroup, consolidate_groups +from skillopt_sleep.multi_skill import CONSOLIDATED, SkillGroup, consolidate_groups +from skillopt_sleep.staging import StagingError from skillopt_sleep.types import EditRecord, ReplayResult, SleepReport, TaskRecord RULE_A = "Rule A: answer the question that was asked." @@ -1199,3 +1202,533 @@ def test_held_out_test_replay_keeps_its_budget(tmp_path) -> None: ("gate", "edit_ablation") ) assert free.report.tokens_used > spend + + +# ── a failed ablation never costs a decided night its result ───────────────── + +FAILED_NOTE = "ablation failed: RuntimeError: backend down" +STOPPED_NOTE = "not replayed: ablation stopped after a failure" + + +def _ablation_replays_raise(error: BaseException, scorer=_free_rider_scores): + """Score like ``scorer``, but raise ``error`` in every ablation replay. + + An ablation replay is the only validation replay that scores a skill with + exactly one of Rule A and Rule B: the baseline has neither, and the trial + and final replays have both. + """ + + def scores(task_id: str, skill: str, memory: str) -> float: + if task_id != "train" and (RULE_A in skill) != (RULE_B in skill): + raise error + return scorer(task_id, skill, memory) + + return scores + + +def test_failed_ablation_replay_leaves_the_decision_and_marks_the_rows( + monkeypatch, +) -> None: + _patch_replay(monkeypatch, _free_rider_scores) + off = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=False) + calls = _patch_replay( + monkeypatch, _ablation_replays_raise(RuntimeError("backend down")) + ) + + # No queue: the epoch runs its own ablation, after its decision. + on = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + assert on.accepted is True + _assert_same_decisions(on, off) + failed, stopped = _skill_trial(on)["edit_ablation"] + assert failed["note"] == FAILED_NOTE + assert stopped["note"] == STOPPED_NOTE + for row in (failed, stopped): + assert row["observed_change"] == "unmeasured" + assert row["replayed"] is False + assert row["score_without"] is None + assert row["task_changes"] == [] + # Baseline, trial, final, and the one ablation replay that raised: the + # second ablation replay never started. + assert len(_val_replays(calls)) == 4 + # A failure is not a budget skip. + assert on.budget_skipped == [] + json.dumps(on.gate_trials, allow_nan=False) + + +def test_queued_ablation_failure_is_not_raised_to_the_caller(monkeypatch) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("backend down"))) + queue: list = [] + result = _consolidate( + [_add(RULE_A), _add(RULE_B)], + gate_edit_ablation=True, + edit_ablation_queue=queue, + ) + + # The caller's queue run neither raises nor reports a budget skip. + assert run_edit_ablations(queue) is False + + assert queue == [] + assert result.accepted is True + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + + +def test_rows_that_need_no_replay_are_still_measured_after_a_failure( + monkeypatch, +) -> None: + _patch_replay( + monkeypatch, + _ablation_replays_raise( + RuntimeError("backend down"), scorer=_skill_and_memory_scores + ), + ) + + result = _consolidate( + [_add(RULE_A), _add(RULE_B), _add(MEMORY_RULE, target="memory")], + evolve_memory=True, + gate_edit_ablation=True, + ) + + assert result.accepted is True and MEMORY_RULE in result.new_memory + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + memory_trial = next(t for t in result.gate_trials if t["target"] == "memory") + (memory_row,) = memory_trial["edit_ablation"] + # The failure stops further replays, not the rows that never needed one. + assert memory_row["replayed"] is False + assert memory_row["observed_change"] == "higher_with_edit" + + +def test_failure_in_one_trial_stops_the_replays_of_the_next(monkeypatch) -> None: + second_memory_rule = "Memory rule: note which sources were checked." + + def scores(task_id: str, skill: str, memory: str) -> float: + if task_id == "essay": + return 1.0 if RULE_A in skill else 0.0 + if task_id == "router": + return 0.9 if MEMORY_RULE in memory else 0.8 + return 0.0 + + calls = _patch_replay( + monkeypatch, + _ablation_replays_raise(RuntimeError("backend down"), scorer=scores), + ) + + result = _consolidate( + [ + _add(RULE_A), + _add(RULE_B), + _add(MEMORY_RULE, target="memory"), + _add(second_memory_rule, target="memory"), + ], + evolve_memory=True, + gate_edit_ablation=True, + ) + + assert result.accepted is True + assert [row["note"] for row in _skill_trial(result)["edit_ablation"]] == [ + FAILED_NOTE, + STOPPED_NOTE, + ] + memory_trial = next(t for t in result.gate_trials if t["target"] == "memory") + memory_rows = memory_trial["edit_ablation"] + # The memory trial is ablated after the skill trial, whose replay failed: + # neither of its two replays starts. + assert [row["note"] for row in memory_rows] == [STOPPED_NOTE, STOPPED_NOTE] + assert all(row["replayed"] is False for row in memory_rows) + assert not [ + call for call in calls + if (MEMORY_RULE in call["memory"]) != (second_memory_rule in call["memory"]) + ] + + +def test_failed_ablation_note_is_bounded(monkeypatch) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("x" * 5000))) + + result = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + failed = _skill_trial(result)["edit_ablation"][0] + assert failed["note"].startswith("ablation failed: RuntimeError: xxx") + assert len(failed["note"]) < 400 + + +def test_failure_stops_the_ablation_replays_that_follow(monkeypatch) -> None: + ablation_replays = {"count": 0} + + def scores(task_id: str, skill: str, memory: str) -> float: + rules = [rule in skill for rule in (RULE_A, RULE_B, RULE_C, RULE_D)] + if task_id == "essay" and any(rules) and not all(rules): + ablation_replays["count"] += 1 + if ablation_replays["count"] == 2: + raise RuntimeError("backend down") + return _free_rider_scores(task_id, skill, memory) + + calls = _patch_replay(monkeypatch, scores) + + result = _consolidate( + [_add(RULE_A), _add(RULE_B), _add(RULE_C), _add(RULE_D)], + gate_edit_ablation=True, + ) + + assert result.accepted is True + rows = _skill_trial(result)["edit_ablation"] + # The first replay finished and keeps its row; the second raised; the + # third and fourth never started. + assert [row["replayed"] for row in rows] == [True, False, False, False] + assert [row["observed_change"] for row in rows] == [ + "higher_with_edit", + "unmeasured", + "unmeasured", + "unmeasured", + ] + assert [row["note"] for row in rows] == ["", FAILED_NOTE, STOPPED_NOTE, STOPPED_NOTE] + assert len(_val_replays(calls)) == 5 + + +@pytest.mark.parametrize("missing", ["hard", "soft"]) +def test_ablation_result_without_a_numeric_score_is_unmeasured( + monkeypatch, missing +) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + + def fake_replay_batch(backend, tasks, skill, memory): + ablation = (RULE_A in skill) != (RULE_B in skill) + pairs = [] + for task in tasks: + score = _free_rider_scores(task.id, skill, memory) + result = ReplayResult(id=task.id, hard=score, soft=score, response="r") + if ablation and task.id == "router": + # Before a decision, an absent numeric score aborts the + # evaluation. After it, the row is all it can cost. A missing + # hard score fails the per-task comparison, a missing soft + # score the aggregate that follows it. + setattr(result, missing, None) + pairs.append((task, result)) + return pairs + + monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch) + + result = _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + assert result.accepted is True + failed, stopped = _skill_trial(result)["edit_ablation"] + # The replay itself finished; scoring its result is what raised. + assert failed["replayed"] is True + assert failed["observed_change"] == "unmeasured" + assert failed["score_without"] is None + assert failed["task_changes"] == [] + assert failed["note"].startswith("ablation failed: TypeError") + assert stopped["note"] == STOPPED_NOTE + json.dumps(result.gate_trials, allow_nan=False) + + +@pytest.mark.parametrize( + "error", + [ + PendingCalls({"call-1": {"prompt": "replay"}}), + CursorBackendError("cursor refused"), + StagingError("staging refused"), + KeyboardInterrupt(), + ], + ids=lambda error: type(error).__name__, +) +def test_control_flow_errors_still_leave_the_ablation(monkeypatch, error) -> None: + _patch_replay(monkeypatch, _ablation_replays_raise(error)) + + with pytest.raises(type(error)): + _consolidate([_add(RULE_A), _add(RULE_B)], gate_edit_ablation=True) + + +def test_one_group_failing_its_ablation_leaves_every_group_decided( + monkeypatch, +) -> None: + off, _off_calls = _consolidate_two_groups(monkeypatch, ablation=False) + calls = _patch_replay( + monkeypatch, _ablation_replays_raise(RuntimeError("backend down")) + ) + + on = consolidate_groups( + ScriptedBackend([_add(RULE_A), _add(RULE_B)]), + _two_groups(), + "", + gate_metric="hard", + gate_edit_ablation=True, + ) + + assert list(on) == list(off) + for name, expected in off.items(): + outcome = on[name] + assert outcome.status == expected.status == CONSOLIDATED + assert outcome.reason == expected.reason == "" + _assert_same_decisions(outcome.result, expected.result) + assert [ + row["note"] for row in _skill_trial(outcome.result)["edit_ablation"] + ] == [FAILED_NOTE, STOPPED_NOTE] + # Each group stops at its own first failure: one attempt per group. + assert sum(_is_ablation_replay(call) for call in calls) == 2 + + +class AblationFailingMock(MockBackend): + """MockBackend whose model calls raise during edit-ablation replays only.""" + + def attempt(self, *args, **kwargs): + if evidence_log.phase(self).startswith("gate_ablation:"): + raise RuntimeError("backend down") + return super().attempt(*args, **kwargs) + + +def _failing_night(tmp_path, name: str, *, ablation: bool, fanout: bool = False): + root = tmp_path / name + project = root / "project" + project.mkdir(parents=True) + claude_home = root / ".claude" + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + fanout_config = {} + if fanout: + for skill_name in ("research-skill", "programming-skill"): + skill_path = claude_home / "skills" / skill_name / "SKILL.md" + skill_path.parent.mkdir(parents=True) + skill_path.write_text(f"# {skill_name}\n", encoding="utf-8") + programming = assign_splits( + programmer_persona(), holdout_fraction=0.34, seed=1 + ) + tasks = [replace(task, skill_hint="research-skill") for task in tasks] + tasks += [ + replace(task, id=f"prog-{task.id}", skill_hint="programming-skill") + for task in programming + ] + fanout_config = { + "managed_skill_name": "skillopt-sleep-learned", + "multi_skill_fanout": True, + } + config = load_config( + invoked_project=str(project), + projects="invoked", + backend="mock", + state_dir=str(root / "state"), + claude_home=str(claude_home), + gate_edit_ablation=ablation, + auto_adopt=False, + **fanout_config, + ) + return run_sleep_cycle(config, seed_tasks=tasks, backend=AblationFailingMock()) + + +def _staged_proposals(outcome) -> dict: + proposals = {} + for file_name in sorted(os.listdir(outcome.staging_dir)): + if file_name.startswith("proposed_"): + with open( + os.path.join(outcome.staging_dir, file_name), encoding="utf-8" + ) as handle: + proposals[file_name] = handle.read() + return proposals + + +def _events(outcome, name: str) -> list[dict]: + return [ + event + for event in evidence_log.read_events( + os.path.join(outcome.staging_dir, "evidence.jsonl") + ) + if event["event"] == name + ] + + +def test_night_is_staged_when_an_ablation_replay_fails(tmp_path) -> None: + off = _failing_night(tmp_path, "off", ablation=False) + on = _failing_night(tmp_path, "on", ablation=True) + + # The night used to raise here and stage nothing but its evidence log. + assert off.report.accepted is True + assert _night_decisions(on) == _night_decisions(off) + assert _staged_proposals(off) + assert _staged_proposals(on) == _staged_proposals(off) + assert ("stage", "staged") in _evidence_events(on) + accepted_trial = next( + trial for trial in on.report.gate_trials + if trial["target"] != "final" and trial["accepted"] + ) + notes = [row["note"] for row in accepted_trial["edit_ablation"]] + assert notes[0] == FAILED_NOTE + assert notes[1:] and set(notes[1:]) == {STOPPED_NOTE} + (failure,) = _events(on, "edit_ablation_failed") + assert failure["stage"] == "gate" + assert failure["error"] == "RuntimeError: backend down" + assert failure["edit_index"] == 0 + # A failure is reported on its rows and in the evidence log, never as a + # step the token budget skipped. + assert _budget_notes(on) == [] + with open(os.path.join(on.staging_dir, "report.md"), encoding="utf-8") as handle: + assert FAILED_NOTE in handle.read() + + +def test_fanout_night_is_staged_when_ablation_replays_fail(tmp_path) -> None: + off = _failing_night(tmp_path, "off", ablation=False, fanout=True) + on = _failing_night(tmp_path, "on", ablation=True, fanout=True) + + assert [group.accepted for group in off.report.skill_groups] == [True, True] + assert _night_decisions(on) == _night_decisions(off) + assert len(_staged_proposals(off)) == 3 + assert _staged_proposals(on) == _staged_proposals(off) + # One failure per consolidation that had a replay to run, each named. + failures = _events(on, "edit_ablation_failed") + assert failures + assert failures[0]["skill"] == "" + group_names = {group.skill_name for group in on.report.skill_groups} + assert {failure["skill"] for failure in failures[1:]} <= group_names + assert all( + failure["error"] == "RuntimeError: backend down" for failure in failures + ) + + +# ── ablation events name the trial they describe ───────────────────────────── + + +def test_ablation_event_names_its_skill_night_and_trial(monkeypatch, tmp_path) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _skill_and_memory_scores) + backend = ScriptedBackend( + [_add(RULE_A), _add(RULE_B), _add(MEMORY_RULE, target="memory")] + ) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + result = consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + set_learned("# Memory\n", []), + gate_metric="hard", + gate_edit_ablation=True, + night=7, + skill_name="alpha", + ) + + events = [ + event + for event in evidence_log.read_events(str(log_path)) + if event["event"] == "edit_ablation" + ] + assert [event["target"] for event in events] == ["skill", "memory"] + for event in events: + assert event["skill"] == "alpha" + assert event["night"] == 7 + # trial_index points at the trial that holds these rows. + trial = result.gate_trials[event["trial_index"]] + assert trial["target"] == event["target"] + assert [row["content"] for row in trial["edit_ablation"]] == [ + row["content"] for row in event["rows"] + ] + + +def test_ablation_event_of_an_unnamed_run_has_an_empty_skill( + monkeypatch, tmp_path +) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _free_rider_scores) + backend = ScriptedBackend([_add(RULE_A), _add(RULE_B)]) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + "", + gate_metric="hard", + evolve_memory=False, + gate_edit_ablation=True, + ) + + (event,) = [ + event + for event in evidence_log.read_events(str(log_path)) + if event["event"] == "edit_ablation" + ] + assert event["skill"] == "" + assert event["night"] == 1 + assert event["trial_index"] == 0 + + +def test_failed_ablation_event_names_its_trial_and_edit(monkeypatch, tmp_path) -> None: + from skillopt_sleep.evidence import EvidenceLog + + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + _patch_replay(monkeypatch, _ablation_replays_raise(RuntimeError("backend down"))) + backend = ScriptedBackend([_add(RULE_A), _add(RULE_B)]) + log_path = tmp_path / "evidence.jsonl" + evidence_log.attach(backend, EvidenceLog(str(log_path))) + + consolidate_module.consolidate( + backend, + _tasks(), + set_learned("# Skill\n", []), + "", + gate_metric="hard", + evolve_memory=False, + gate_edit_ablation=True, + night=3, + skill_name="alpha", + ) + + events = evidence_log.read_events(str(log_path)) + (failure,) = [e for e in events if e["event"] == "edit_ablation_failed"] + assert failure["stage"] == "gate" + assert failure["target"] == "skill" + assert failure["skill"] == "alpha" + assert failure["night"] == 3 + assert failure["trial_index"] == 0 + assert failure["edit_index"] == 0 + assert failure["error"] == "RuntimeError: backend down" + # The trial still gets its one edit_ablation event, with every row. + (rows_event,) = [e for e in events if e["event"] == "edit_ablation"] + assert rows_event["n_replays"] == 0 + assert [row["note"] for row in rows_event["rows"]] == [FAILED_NOTE, STOPPED_NOTE] + + +def test_groups_pass_their_name_only_with_ablation_on() -> None: + seen: list[dict] = [] + + def record(backend, tasks, skill, memory, **kwargs): + seen.append(kwargs) + raise RuntimeError("stop after recording") + + for ablation in (False, True): + consolidate_groups( + ScriptedBackend([]), + _two_groups(), + "", + consolidate_fn=record, + gate_edit_ablation=ablation, + ) + + off_alpha, off_beta, on_alpha, on_beta = seen + # Off by default: a custom consolidate_fn sees the arguments it always saw. + assert "skill_name" not in off_alpha and "skill_name" not in off_beta + assert on_alpha["skill_name"] == "alpha" + assert on_beta["skill_name"] == "beta" + + +def test_fanout_ablation_events_name_their_group(tmp_path) -> None: + outcome = _fanout_night(tmp_path, "night", ablation=True, max_tokens=0) + + accepted_groups = [ + group.skill_name for group in outcome.report.skill_groups if group.accepted + ] + assert len(accepted_groups) == 2 + events = _events(outcome, "edit_ablation") + # The aggregate run is not a per-skill group; each group's event carries + # the group's name, so it no longer has to be matched by edit content. + assert [event["skill"] for event in events] == ["", *accepted_groups] + assert all(event["night"] == outcome.report.night for event in events) + assert all(isinstance(event["trial_index"], int) for event in events)