From a6b828dc21ca0a4e9cf87c309d6444f7f3fb4391 Mon Sep 17 00:00:00 2001
From: uipreliga <tomasz.religa@uipath.com>
Date: Thu, 20 Aug 2026 19:38:24 -0700
Subject: [PATCH] feat(plugin): add optimize-skill and the --split row filter
 it measures with
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds the `/coder-eval:optimize-skill` skill — prose only, no Python — plus the
one product change it needs to measure anything: `Dataset.split_field` and a
`--split` row filter in `task_loader.expand_dataset`. Both are inert when
unused: a dataset with no `split_field` and a run with no `--split` behave
exactly as before.

Stage B here is a manual reading of two runs. There is no gate, no statistics
and no `coder_eval.optimize` package in this PR.

One published skill changes behaviour: `analyze`'s frontmatter `description` is
replaced by the measured winner of the round that tutorial 08 documents.

Also fixes a dependency defect this PR's CI surfaced, unrelated to the split:
`evaluation/judge_bedrock.py` imports `httpx`, which `pyproject.toml` never
declared. It arrived transitively via `anthropic` until `anthropic` 1.0.0 (released
2026-08-20) moved to `httpx2`. A FRESH resolve — what `uv tool install` and
`pip install coder-eval` do, since a wheel carries no lockfile — then stopped
installing `httpx`, so `criteria/llm_judge.py` raised on import, the discovery loop
swallowed it, and `llm_judge` vanished from the registry. `uv.lock` hid this from
every `uv sync --frozen` job.

`httpx` is now declared, and `tests/test_declared_dependencies.py` asserts the
invariant on the DECLARATION rather than the installed set — the only form that
fails in the locked jobs, where this bug was invisible. Imports guarded by
`try/except ImportError` are derived as optional and exempt, so the soft
dependencies (`google.antigravity`, `openai`) need no allowlist.

Squashed from feat/plugin-optimize-skill:

  7376062 feat(dataset): 1/3 — add Dataset.split_field and the --split row filter
  240d66c feat(plugin): 2/3 — add the optimize-skill skill and split-label the activation template
  8920410 docs: 3/3 — add the skill-optimization tutorial, and fix the reachability guidance it disproved
  2c30397 style: apply ruff format to the reachability lint assertion
  ae3c39b docs(harness): record the all-skipped-run-exits-0 gap found while adding --split
  b53c7d4 fix: code review fixes for the split-field / optimize-skill plan
  d7d56f1 feat(plugin): promote a measured `analyze` description, and close the two open findings
  844348d docs(tutorial): Stage C completed — the analyze promotion is confirmed on holdout
  e340b58 docs: record the bare-name collision hazard, and mark the plan complete

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .claude/harness-candidates.md                 |  15 +
 .claude/shared/run-layout.md                  |  47 ++
 CLAUDE.md                                     |   4 +-
 README.md                                     |   6 +-
 docs/AB_EXPERIMENTS.md                        |   1 +
 docs/DATASETS.md                              |  59 +-
 docs/PLUGIN.md                                |  17 +-
 docs/TASK_DEFINITION_GUIDE.md                 |   2 +
 docs/USER_GUIDE.md                            |   3 +-
 docs/llms.txt                                 |   1 +
 docs/tutorials/07-plugin-in-claude-code.md    |   8 +-
 docs/tutorials/08-optimizing-a-skill.md       | 515 ++++++++++++++++++
 docs/tutorials/README.md                      |   1 +
 mkdocs.yml                                    |   1 +
 plugins/coder-eval/README.md                  |   8 +-
 plugins/coder-eval/reference/run-layout.md    |  47 ++
 .../reference/templates/activation-rows.jsonl |  12 +-
 .../reference/templates/activation.yaml       |  35 +-
 plugins/coder-eval/skills/analyze/SKILL.md    |   2 +-
 .../coder-eval/skills/check-skill/SKILL.md    |  34 +-
 plugins/coder-eval/skills/ci/SKILL.md         |  10 +-
 plugins/coder-eval/skills/init/SKILL.md       |   2 +-
 plugins/coder-eval/skills/lint-tasks/SKILL.md |   2 +-
 .../coder-eval/skills/optimize-skill/SKILL.md | 349 ++++++++++++
 plugins/coder-eval/skills/task/SKILL.md       |   2 +-
 pyproject.toml                                |   6 +
 src/coder_eval/cli/run_command.py             |  17 +
 src/coder_eval/models/tasks.py                |  12 +
 src/coder_eval/orchestration/config.py        |   8 +
 src/coder_eval/orchestration/experiment.py    |   1 +
 src/coder_eval/orchestration/task_loader.py   |  68 ++-
 tasks/skills/lint-tasks-activation-rows.jsonl |  28 +
 tasks/skills/lint-tasks-activation.yaml       |  89 +++
 tests/test_custom_lint.py                     | 162 +++++-
 tests/test_dataset_expansion.py               | 246 +++++++++
 tests/test_declared_dependencies.py           | 128 +++++
 uv.lock                                       |   2 +
 37 files changed, 1894 insertions(+), 56 deletions(-)
 create mode 100644 docs/tutorials/08-optimizing-a-skill.md
 create mode 100644 plugins/coder-eval/skills/optimize-skill/SKILL.md
 create mode 100644 tasks/skills/lint-tasks-activation-rows.jsonl
 create mode 100644 tasks/skills/lint-tasks-activation.yaml
 create mode 100644 tests/test_declared_dependencies.py

diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md
index e067e9ad..4d9e58e4 100644
--- a/.claude/harness-candidates.md
+++ b/.claude/harness-candidates.md
@@ -455,3 +455,18 @@ with the two `action.yml` items above — one considered change to the action's
   `final_status`, which does not exist in `run.json` and would have made a new
   assertion dead on arrival. Guard: assert the key set that non-Python consumers
   depend on, mirroring how CE030 pins doc/schema parity.
+## From the split-field / optimize-skill plan (2026-08-12)
+
+- [ ] **A run whose every task is skipped exits 0 — a green run of zero tasks.** When
+  `resolve_all_tasks` demotes every task to `skipped_tasks` (a load failure, `skip: true`,
+  or now a `--split` selector matching no labelled row), the run reports success: nothing
+  failed, so the exit gate in `cli/run_command.py` — which keys only on failed/errored tasks
+  and suite gates — passes. Verified directly: `coder-eval run <suite> --split holdou`
+  prints one yellow "1 task file(s) skipped" line and exits 0. This is pre-existing, but
+  `--split` makes it reachable by a one-character CLI typo rather than a broken file, and
+  the whole point of a holdout confirmation is that you trust its verdict. Not guarded, and
+  not a five-minute fix: making an all-skipped run non-green changes exit semantics for
+  every skipped-task path (including deliberate `skip: true` suites and tag filters that
+  match nothing), so it needs a decision about which of those should be fatal, plus tests
+  per case. A narrower option is to fail only when a CLI *selector* (`--split`, `--tags`)
+  eliminated everything, since that is unambiguously a user error rather than repo state.
diff --git a/.claude/shared/run-layout.md b/.claude/shared/run-layout.md
index 57edc4a6..7e62e707 100644
--- a/.claude/shared/run-layout.md
+++ b/.claude/shared/run-layout.md
@@ -14,6 +14,53 @@ runs/<run_id>/<variant_id>/<task_id>/<NN>/{task.json, task.log, artifacts/}
 - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it.
 - `task.log` — the human-readable task log; `artifacts/` — files the agent produced.
 
+## Suite rollups (dataset-backed tasks only)
+
+A task carrying `dataset:` fans out into one row-task per row and additionally writes a
+per-suite rollup:
+
+```
+runs/<run_id>/<variant_id>/<suite_id>/{suite.json, suite.md}
+```
+
+`<suite_id>` is the original (pre-fan-out) `task_id`. Nothing is written for a task
+without `dataset:`.
+
+`suite.json` carries the suite's pass counts plus `criterion_aggregates[]` — one entry per
+criterion that opted into across-row aggregation, each with:
+
+- `criterion_type`, and `description` (set when a task stacks several criteria of the same
+  type, e.g. one `skill_triggered` per skill — that is what distinguishes them);
+- `rows_total` and `rows_excluded` — the denominator and what was dropped from it. A row
+  that errored before criteria ran (a timeout, say) is **excluded** rather than scored, so
+  metrics are computed over `rows_total - rows_excluded`;
+- `metrics` — a **flat** name → float map. Classification-style criteria emit
+  `accuracy`, `macro_f1`, and per-label `precision.<label>` / `recall.<label>` /
+  `f1.<label>` (for `skill_triggered` the labels are `yes` / `no`). It also carries
+  `completion_rate` — the surviving fraction of the denominator above — which is an
+  ordinary metric, so `suite_thresholds: {completion_rate: 1.0}` gates a run whose sample
+  eroded;
+- `details` — `labels`, `per_label`, `confusion`, `total_pairs`;
+- `threshold_checks` + `passed`, from the criterion's `suite_thresholds`.
+
+Alongside the aggregates, `failed_samples[]` lists failed/errored rows **by `row_id`** with
+their failure reasons, capped at a fixed limit. It is the only place in `suite.json` that
+carries row identity — `metrics` and `details` are counts only — so a consumer that needs
+to know *which* rows failed reads `failed_samples`, then falls back to the per-row
+`<variant_id>/<suite_id>/<row_id>/<NN>/task.json` for anything past the cap.
+
+**Read `metrics`; never recompute a metric from `details.confusion`.** The criterion layer
+owns that arithmetic, including its division-by-zero convention, and a consumer that
+re-derives F1 will disagree with the gate the run already applied.
+
+**Rollups pool replicates.** The grouping key is `(variant_id, suite_id)` — the replicate
+index is *not* part of it. So `--repeats N` over an M-row suite writes **one** `suite.json`
+per variant, with `rows_total: N × M` and a single pooled confusion matrix. There is no
+per-replicate metric in that file. A consumer that needs replicate-to-replicate spread must
+invoke the run N times and read N run directories; `--repeats` cannot serve that purpose.
+(Contrast `experiment.md`'s `## Paired Comparison` block, which averages replicates per row
+before pairing — there `--repeats` is exactly the right tool.)
+
 **Scope-marker files** (used to detect what a given path represents):
 
 - `run.json` at the run root → **run scope**. If `experiment.json` (+ `experiment.md`) is also present → multi-variant experiment.
diff --git a/CLAUDE.md b/CLAUDE.md
index f4320f48..9fdbbcad 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -124,7 +124,7 @@ tests/                             # Test suite
 docs/                              # Documentation
 templates/                         # Sandbox template directories
 .claude-plugin/marketplace.json    # Makes this repo a Claude Code plugin marketplace (`/plugin marketplace add UiPath/coder_eval`); lists the one plugin below.
-plugins/coder-eval/                # The published Claude Code plugin: `.claude-plugin/plugin.json` (its `version` is a derived pin of pyproject's, bumped by release.yml, guarded by tests/test_action_version_pin.py), `skills/<name>/SKILL.md` × 6 (`/coder-eval:init`, `/coder-eval:check-skill`, `/coder-eval:task`, `/coder-eval:lint-tasks`, `/coder-eval:analyze`, `/coder-eval:ci`), and `reference/` — everything a skill reads must live here, since an installed plugin is copied to ~/.claude/plugins/cache/ WITHOUT its parent dirs (address it via `${CLAUDE_PLUGIN_ROOT}`). `reference/criteria.md` is generated (`make plugin-reference`, CE033); `reference/run-layout.md` is a verbatim mirror of `.claude/shared/run-layout.md`; `reference/task-rubric.md` is the shared task-quality rubric that `task` and `lint-tasks` both read (plugin-only — no repo-side twin); `reference/repo-layout.md` is the eval-tree DISCOVERY policy every skill reads (`SKILL_NEEDS_EVAL_ROOT_DISCOVERY`, which a new skill must declare a stance in) — glob for `task_id:` files and `run.json`, never assume `tasks/`/`runs/latest` — as distinct from `run-layout.md`, which describes what is inside a run directory. Every skill must appear in all four surfaces in `SKILL_DOC_SURFACES` (derived test), and their combined frontmatter `description` length is capped (`SKILL_LISTING_BUDGET_CHARS`) because the skill listing's budget is shared with every skill the user has installed. **Skill naming is verb-first imperative** — a skill is a command you issue (`/coder-eval:<name>`) and every one of them takes an action, so name it for the action: a bare verb where that is unambiguous (`init`, `analyze` — the object comes from the argument), otherwise `<verb>-<object>` (`lint-tasks`, `check-skill`). Never `<object>-<verb>`: `skill-check` was renamed to `check-skill` precisely because it read backwards next to `lint-tasks`. `task` and `ci` predate the rule and stay — renaming a published skill breaks every user's muscle memory for no functional gain, since activation keys on the `description`, never the name. Distinct from `.claude/commands/`, which stays repo-local contributor tooling.
+plugins/coder-eval/                # The published Claude Code plugin: `.claude-plugin/plugin.json` (its `version` is a derived pin of pyproject's, bumped by release.yml, guarded by tests/test_action_version_pin.py), `skills/<name>/SKILL.md` × 7 (`/coder-eval:init`, `/coder-eval:check-skill`, `/coder-eval:optimize-skill`, `/coder-eval:task`, `/coder-eval:lint-tasks`, `/coder-eval:analyze`, `/coder-eval:ci`), and `reference/` — everything a skill reads must live here, since an installed plugin is copied to ~/.claude/plugins/cache/ WITHOUT its parent dirs (address it via `${CLAUDE_PLUGIN_ROOT}`). `reference/criteria.md` is generated (`make plugin-reference`, CE033); `reference/run-layout.md` is a verbatim mirror of `.claude/shared/run-layout.md`; `reference/task-rubric.md` is the shared task-quality rubric that `task` and `lint-tasks` both read (plugin-only — no repo-side twin); `reference/repo-layout.md` is the eval-tree DISCOVERY policy every skill reads (`SKILL_NEEDS_EVAL_ROOT_DISCOVERY`, which a new skill must declare a stance in) — glob for `task_id:` files and `run.json`, never assume `tasks/`/`runs/latest` — as distinct from `run-layout.md`, which describes what is inside a run directory. Every skill must appear in all four surfaces in `SKILL_DOC_SURFACES` (derived test), and their combined frontmatter `description` length is capped (`SKILL_LISTING_BUDGET_CHARS`) because the skill listing's budget is shared with every skill the user has installed. **Skill naming is verb-first imperative** — a skill is a command you issue (`/coder-eval:<name>`) and every one of them takes an action, so name it for the action: a bare verb where that is unambiguous (`init`, `analyze` — the object comes from the argument), otherwise `<verb>-<object>` (`lint-tasks`, `check-skill`, `optimize-skill`). Never `<object>-<verb>`: `skill-check` was renamed to `check-skill` precisely because it read backwards next to `lint-tasks`, and `optimize-skill` was named that way rather than `skill-optimize` for the same reason — the rule is applied at authoring time now, not repaired later. `task` and `ci` predate the rule and stay — renaming a published skill breaks every user's muscle memory for no functional gain, since activation keys on the `description`, never the name. Distinct from `.claude/commands/`, which stays repo-local contributor tooling.
 action.yml                         # Published composite GitHub Action (coder-eval as a CI gate). release.yml's `release` job maintains its `version:` default; its `promote` job (gated on publish-pypi) moves the `v<major>` tag + cuts the Release, so nothing consumer-visible moves before the wheel is on PyPI. verify-published-action.yml then verifies the published composite (tag/pin/PyPI/Marketplace parity, plus a real consumer run) after each Release and nightly. Runbook: CONTRIBUTING.md § Releasing.
 ```
 
@@ -139,7 +139,7 @@ action.yml                         # Published composite GitHub Action (coder-ev
 - **Single declarative merge resolver**: All five config layers merge through ONE engine (`orchestration/config_merge.py::resolve_root`) for the three `-D`-reachable roots (`agent`/`run_limits`/`sandbox`). Each field declares *how it merges* once, on the model, via `MergeField(strategy="deep"|"append"|"replace")` (or a type-aware default: nested `BaseModel`/free-form `dict` → `deep`; `list`/scalar → `replace`). `resolve_task_for_variant` (layers 1–4) and `apply_overrides` (layer 5) build `Layer` lists and call the same `resolve_root`, so a field merges identically regardless of which layer supplied it (the unification invariant, enforced by `tests/test_merge_unification.py`). Lint rule CE014 forces every list field to declare its strategy explicitly.
 - **Generic CLI overrides (`-D`/`--set`)**: Layer 5 is a thin wrapper (`orchestration/overrides.py`) over the resolver above. `coder-eval run -D agent.model=opus -D run_limits.max_turns=30` overrides any field on the resolved `TaskDefinition` (`agent`/`run_limits`/`sandbox` roots), schema-validated with did-you-mean. Only `--model` (→ `agent.model`) and `--driver` (→ `sandbox.driver`) survive as active thin aliases that emit the equivalent `-D` entry; an alias and `-D` targeting the same path is a hard error. `--type` (→ `agent.type`) is a separate, lighter alias that does NOT route through that collision check — `--type` and `-D agent.type=…` last-win rather than hard-error (the `-D` value wins). Tools, plugins, and SDK options are `-D`-only.
 - **All core models importable from `coder_eval.models`** regardless of submodule
-- **Dataset fan-out**: `TaskDefinition.dataset` (inline rows or JSONL path) expands a single task into N row-tasks with `${row.<field>}` substitution in `initial_prompt` and `success_criteria` string fields. Expansion runs in `task_loader.expand_dataset` **before** variant resolution, so variants cannot override the dataset. Row sampling: CLI `--sample N` (fixed-seed uniform-random N over the whole dataset) overrides `--sample-per-stratum N` / `dataset.sample_per_stratum` (stratified random N-per-stratum, keyed on `stratify_field`, default `expected_skill` — for classification suites like activation). Stratified sampling (whether the N-per-stratum count comes from the **CLI** `--sample-per-stratum` flag or **YAML** `dataset.sample_per_stratum`) is **nondeterministic** by default — it re-draws each run (so the nightly activation suite broadens coverage over time). Set `dataset.sample_seed` to pin a reproducible sample; an explicit seed always wins. (Only `--sample N` uses a fixed seed, since a smoke test wants the same N rows each run.)
+- **Dataset fan-out**: `TaskDefinition.dataset` (inline rows or JSONL path) expands a single task into N row-tasks with `${row.<field>}` substitution in `initial_prompt` and `success_criteria` string fields. Expansion runs in `task_loader.expand_dataset` **before** variant resolution, so variants cannot override the dataset. Row selection is filter-then-sample: CLI `--split <name>` (keep only rows whose `dataset.split_field` value matches, default field `split`) runs **first** and is orthogonal to the sampler win-order — a row is unlabelled when the field is absent/`null`/`""` and a task whose rows are all unlabelled passes through unfiltered; partial labelling drops the unlabelled rows. A *labelled* task with no matching row raises a `ValueError` listing the splits that exist — which `resolve_all_tasks` catches into `skipped_tasks` like any load failure, so a mistyped selector yields a zero-task run that still exits 0. Filtering before sampling is a correctness requirement: sampling first would leave an unpredictable (possibly zero) number of rows per split, destroying the tune/holdout comparison. Then sampling: CLI `--sample N` (fixed-seed uniform-random N over the whole dataset) overrides `--sample-per-stratum N` / `dataset.sample_per_stratum` (stratified random N-per-stratum, keyed on `stratify_field`, default `expected_skill` — for classification suites like activation). Stratified sampling (whether the N-per-stratum count comes from the **CLI** `--sample-per-stratum` flag or **YAML** `dataset.sample_per_stratum`) is **nondeterministic** by default — it re-draws each run (so the nightly activation suite broadens coverage over time). Set `dataset.sample_seed` to pin a reproducible sample; an explicit seed always wins. (Only `--sample N` uses a fixed seed, since a smoke test wants the same N rows each run.)
 - **Per-criterion aggregation**: Each `BaseCriterion` subclass exposes `aggregate(criterion, per_row_results) -> CriterionAggregate | None`. Default emits `count / mean / median / std / min / max` so every criterion is suite-thresholdable for free. Classification-style criteria return `ClassificationCriterionResult` (subclass of `CriterionResult`) and layer accuracy / P/R/F1 / confusion via the shared `overlay_classification_metrics` utility. `BaseSuccessCriterion.suite_thresholds` gates the suite on those metrics; CLI exits non-zero on any gate failure.
 - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection.
 - **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports.
diff --git a/README.md b/README.md
index 104e3f7d..78813f17 100644
--- a/README.md
+++ b/README.md
@@ -107,9 +107,9 @@ results — runs inside the agent:
 /plugin install coder-eval@coder-eval
 ```
 
-That adds six slash commands: `/coder-eval:init`, `/coder-eval:check-skill`,
-`/coder-eval:task`, `/coder-eval:lint-tasks`, `/coder-eval:analyze` and
-`/coder-eval:ci`. They drive the `coder-eval` CLI, so install it too
+That adds seven slash commands: `/coder-eval:init`, `/coder-eval:check-skill`,
+`/coder-eval:optimize-skill`, `/coder-eval:task`, `/coder-eval:lint-tasks`,
+`/coder-eval:analyze` and `/coder-eval:ci`. They drive the `coder-eval` CLI, so install it too
 (`uv tool install coder-eval`). See [Claude Code Plugin](docs/PLUGIN.md).
 
 ## Use as a GitHub Action
diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md
index 4392d7f6..8ac47b6d 100644
--- a/docs/AB_EXPERIMENTS.md
+++ b/docs/AB_EXPERIMENTS.md
@@ -355,6 +355,7 @@ if any listed metric is below its minimum.
 | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ |
 | `-e, --experiment <path\|name>`                                                                                                                                   | Experiment YAML. Bare name → `experiments/<name>.yaml`.                                                |
 | `--sample N`                                                                                                                                                      | For dataset-backed tasks, use a fixed-seed random N-row sample (reproducible, unbiased across paths; cheap smoke test). |
+| `--split NAME`                                                                                                                                                    | For dataset-backed tasks, keep only rows whose `dataset.split_field` value matches (e.g. `tune` / `holdout`). Applied before `--sample`. Unlabelled tasks unaffected. |
 | `--repeats N`                                                                                                                                                     | Run each `(task, variant)` N times; overrides YAML `repeats`.                                          |
 | `--driver tempdir\|docker`                                                                                                                                        | Override sandbox driver for all tasks.                                                                 |
 | `-j, --max-parallel N`                                                                                                                                            | Run up to N tasks concurrently.                                                                        |
diff --git a/docs/DATASETS.md b/docs/DATASETS.md
index 498addab..5a8e1b98 100644
--- a/docs/DATASETS.md
+++ b/docs/DATASETS.md
@@ -124,11 +124,38 @@ Load-time errors, with their message shapes:
 | Two rows share an id | `Duplicate dataset row id for task '<task_id>': '<x>'` |
 | Both/neither `rows` and `paths` | `Dataset must specify either 'paths' or 'rows'` / `... only one of ...` |
 | `paths: []` | `Dataset.paths must be a non-empty list` |
+| `--split X` on a labelled dataset with no `X` rows | `Dataset for task '<task_id>' has no rows in split 'X' (split_field='split'); labelled splits present: ['holdout', 'tune']` |
 
-## Sampling a subset
+## Selecting a subset
 
-A full dataset is expensive. Two independent mechanisms cut it down, and **`--sample` wins whenever
-both apply**:
+A full dataset is expensive. Three mechanisms cut it down, in two stages: **`--split` filters
+first**, then **one** of the two samplers runs over what survives, with **`--sample` winning
+whenever both samplers apply**.
+
+**Stage 1 — the filter.** `--split` is not in that win-order; it is orthogonal, and always applies
+first.
+
+**`--split <name>` (CLI only)** — keep only rows whose `dataset.split_field` value (default field:
+`split`) equals `<name>`. Exact string match, no case normalization; non-string values compare via
+`str()`.
+
+Three behaviours are worth knowing before you rely on it:
+
+- **A task whose rows carry no split label at all passes through unfiltered.** `--split` is global to
+  the invocation, so an unlabelled dataset sitting beside a labelled one in the same run must not
+  fail. A row counts as unlabelled when the field is absent, `null`, or `""`.
+- **Partial labelling drops the unlabelled rows.** If only some rows carry a label, `--split tune`
+  keeps just the `tune` rows — the unlabelled ones are excluded rather than folded in. That is the
+  safe direction (an unlabelled row never leaks into a named split), but during an incremental
+  migration it silently *shrinks* the suite, which moves the aggregate metrics `suite_thresholds`
+  gates on. Finish labelling before you compare two runs.
+- **A labelled task with no row in the requested split is skipped, not fatal.** Expansion raises,
+  naming the splits that do exist, and the run records it in `run.json`'s `skipped_tasks` and carries
+  on with whatever else resolved. So a mistyped selector (`--split holdou`) produces a run of **zero
+  tasks that still exits 0** — check the skipped-task count, not just the exit code, when a split run
+  comes back suspiciously clean.
+
+**Stage 2 — the samplers**, over whatever survived the filter:
 
 1. **`--sample N` (CLI only)** — a flat uniform-random N rows over the whole dataset. Fixed seed, so
    the same N rows come back every run: a reproducible, cheap smoke flavor of a big suite. Unlike a
@@ -155,6 +182,32 @@ both apply**:
     coverage over time matters more than run-to-run comparability. Note the contrast with
     `--sample N`, which is fixed-seed and reproducible by default.
 
+### Tune and holdout splits
+
+Label each row with a split and you can develop against one half and confirm on the other, which is
+what keeps a measured improvement from being an artifact of the rows you tuned on:
+
+```jsonl
+{"id": "pos-1", "prompt": "review my task files", "expected_skill": "lint-tasks", "split": "tune"}
+{"id": "pos-2", "prompt": "are my evals any good?", "expected_skill": "lint-tasks", "split": "holdout"}
+```
+
+```bash
+coder-eval run tasks/skills/activation.yaml --split tune      # iterate here
+coder-eval run tasks/skills/activation.yaml --split holdout   # confirm here, once
+```
+
+**The filter runs before either sampler, and that ordering is load-bearing.** Sampling first would
+leave an unpredictable — possibly zero — number of rows per split, so the two arms of the comparison
+would no longer be the same size or the same rows. Filter-then-sample means `--split tune --sample 8`
+is always drawn from the tune rows alone — at most eight of them, and all of them if `tune` holds
+fewer than eight.
+
+Two consequences worth planning for. A split **halves each side of the suite**, so a dataset sized
+for a single one-shot measurement is undersized once split — budget roughly double the rows you
+would otherwise want. And a holdout is only worth what its independence buys: consult it to confirm
+a decision already made on `tune`, not to choose between candidates, or it becomes a second tune set.
+
 ## Suite-level scoring
 
 Per-row pass/fail is rarely the number you care about on a dataset — the suite metric is. Every
diff --git a/docs/PLUGIN.md b/docs/PLUGIN.md
index 9b118482..9bd27be7 100644
--- a/docs/PLUGIN.md
+++ b/docs/PLUGIN.md
@@ -1,6 +1,6 @@
 ---
 description: >-
-  Install the Coder Eval plugin for Claude Code — six slash commands to scaffold,
+  Install the Coder Eval plugin for Claude Code — seven slash commands to scaffold,
   author, review, run and analyze evaluation suites, including an activation suite
   that measures whether your own Claude Code skills actually trigger.
 ---
@@ -43,19 +43,21 @@ yourself.
 Running a suite additionally needs credentials for whichever agent the tasks use —
 `ANTHROPIC_API_KEY` for the default `claude-code` agent.
 
-## The six skills
+## The seven skills
 
 | Command | What it does |
 | --- | --- |
 | `/coder-eval:init` | Scans the repository for what is worth evaluating (Claude Code skills, an MCP server, a CLI), reports the findings, then scaffolds a task directory with one real task. |
 | `/coder-eval:check-skill` | Builds and runs an activation suite for one of your skills — does the agent engage it when it should, and leave it alone when it shouldn't? |
+| `/coder-eval:optimize-skill` | Takes an activation suite's confusion matrix and proposes description rewrites, A/B tests them as experiment variants, and promotes only what beats run-to-run noise and survives a held-out split. |
 | `/coder-eval:task` | Turns a natural-language description into task YAML with criteria that check output *content*, validated through `coder-eval plan`. |
 | `/coder-eval:lint-tasks` | Reviews task YAML that already exists and reports, per task, criteria that cannot fail, prompts that leak the answer, fixtures with no cleanup and near-duplicates — each with a severity and a fix. Read-only. |
 | `/coder-eval:analyze` | Reads a finished run directory and writes `analysis.md`: systemic failure patterns, per-task findings, and concrete fixes. |
 | `/coder-eval:ci` | Emits a GitHub Actions workflow that runs the suite as a gate, or on a schedule to catch skill drift. |
 
-`init` and `ci` are explicit-invocation only — scaffolding a directory or writing
-a workflow is never something to do unprompted. The other four can also be
+`init`, `ci` and `optimize-skill` are explicit-invocation only — scaffolding a
+directory, writing a workflow, or starting rounds of paid A/B runs is never
+something to do unprompted. The other four can also be
 reached by the agent on its own when a request clearly calls for them.
 
 `task` and `lint-tasks` are two halves of the same concern and share one bundled
@@ -106,10 +108,13 @@ It then:
 One prerequisite the suite cannot infer: the evaluated agent runs in a fresh
 sandbox holding none of your files, so it is offered no skills unless the task
 says where they live. The template reads that location from an environment
-variable — point it at the directory *containing* the skill's own directory:
+variable, and it must point at a **plugin root** — a directory holding a `skills/`
+subdirectory, so the skill sits at `<path>/skills/<name>/SKILL.md`. For
+`.claude/skills/pdf-forms/SKILL.md` that root is `.claude`, **not** `.claude/skills`;
+pointing one level too deep loads nothing:
 
 ```bash
-export SKILL_SOURCE_PATH="$(pwd)/.claude/skills"
+export SKILL_SOURCE_PATH="$(pwd)/.claude"
 ```
 
 Leave it unset and the skill is simply absent, every positive row scores 0, and
diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md
index a16edb43..735094d6 100644
--- a/docs/TASK_DEFINITION_GUIDE.md
+++ b/docs/TASK_DEFINITION_GUIDE.md
@@ -83,6 +83,7 @@ dataset:
   sample_per_stratum: 5                # optional: keep up to N rows per stratum
   stratify_field: "expected_skill"     # which row field defines the stratum
   sample_seed: 1234                    # optional: pin the stratified draw
+  split_field: "split"                 # which row field names the row's split (CLI --split)
 ```
 
 | Field | Default | Description |
@@ -93,6 +94,7 @@ dataset:
 | `sample_per_stratum` | `null` | Stratified random sample: keep up to N rows per stratum. Overridden by CLI `--sample`. |
 | `stratify_field` | `"expected_skill"` | Row field whose value defines the stratum for `sample_per_stratum`. |
 | `sample_seed` | `null` | Seed for the stratified draw. Unset means the sample is **re-drawn every run**; set an integer to pin it. CLI `--sample` is separately fixed-seed and always reproducible. |
+| `split_field` | `"split"` | Row field naming the row's split (e.g. `tune` / `holdout`). CLI `--split <name>` keeps only rows whose value here matches, **before** any sampling. A task whose rows never set this field is unaffected by `--split`. Splits are open strings. |
 
 Full guide — row sources, substitution rules, sampling precedence, suite-level scoring, and worked
 examples: **[Bring Your Own Dataset](DATASETS.md)**.
diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
index f7daba37..c749e246 100644
--- a/docs/USER_GUIDE.md
+++ b/docs/USER_GUIDE.md
@@ -44,6 +44,7 @@ coder-eval run tasks/hello_date.yaml --stream full  # live LLM output
 | `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. |
 | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). |
 | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). |
+| `--split NAME` | For dataset-backed tasks, keep only rows whose `dataset.split_field` value (default field: `split`) matches — e.g. `--split tune` / `--split holdout`. Applied **before** `--sample` / `--sample-per-stratum`. Tasks whose rows carry no split label are unaffected. See [Bring Your Own Dataset](DATASETS.md). |
 | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). |
 | `--exclude-tags` | Skip tasks matching any of these tags (comma-separated) |
 | `--tags, -t` | Only run tasks matching any of these tags (comma-separated) |
@@ -139,7 +140,7 @@ in **any** repository rather than only this one:
 | `/coder-eval:task` | Create evaluation task YAML files from a natural language description. |
 | `/coder-eval:analyze <path>` | Analyze evaluation runs and suggest improvements to tasks, config, and prompts. Works at task, variant, or run scope. |
 
-See [Claude Code Plugin](PLUGIN.md) for the full set of six skills. The
+See [Claude Code Plugin](PLUGIN.md) for the full set of seven skills. The
 `.claude/commands/` directory in this repository holds contributor-only tooling
 (code review, planning) that is deliberately not shipped.
 
diff --git a/docs/llms.txt b/docs/llms.txt
index db96ce1f..1b8186a0 100644
--- a/docs/llms.txt
+++ b/docs/llms.txt
@@ -50,6 +50,7 @@ and A/B plumbing.
 - [05 · Comparing two models](https://coder-eval.com/docs/tutorials/05-comparing-models)
 - [06 · Docker isolation](https://coder-eval.com/docs/tutorials/06-use-docker-isolation)
 - [07 · Driving Coder Eval from Claude Code](https://coder-eval.com/docs/tutorials/07-plugin-in-claude-code)
+- [08 · Optimizing a skill description](https://coder-eval.com/docs/tutorials/08-optimizing-a-skill)
 <!-- docs-index:end:tutorials -->
 
 ## Source
diff --git a/docs/tutorials/07-plugin-in-claude-code.md b/docs/tutorials/07-plugin-in-claude-code.md
index a5d7c527..532304bd 100644
--- a/docs/tutorials/07-plugin-in-claude-code.md
+++ b/docs/tutorials/07-plugin-in-claude-code.md
@@ -131,10 +131,12 @@ reaches for one at the right moment.
 
 **Export the skill location first** — the evaluated agent runs in a fresh sandbox
 holding none of your files, so it is offered no skills unless the task says where
-they live. Point at the directory *containing* the skill's own directory:
+they live. Point at a **plugin root** — a directory holding a `skills/` subdirectory, so the
+skill sits at `<path>/skills/<name>/SKILL.md`. For `.claude/skills/pdf-forms/SKILL.md` that
+root is `.claude`, **not** `.claude/skills`:
 
 ```bash
-export SKILL_SOURCE_PATH="$(pwd)/.claude/skills"
+export SKILL_SOURCE_PATH="$(pwd)/.claude"
 ```
 
 ```
@@ -158,7 +160,7 @@ covers telling them apart with `/doctor` and `/context`.
 | No `/coder-eval:` commands after installing | Check `/plugin`; re-run the install |
 | A skill offers to install the CLI, or Bash reports `command not found` | The CLI isn't installed or isn't on `PATH` — accept the offer, or install it yourself |
 | `coder-eval run` matches nothing | Wrong directory — use the path `init` reported in step 2 |
-| Every positive row in step 5 scores 0 | `SKILL_SOURCE_PATH` is unset, so the skill was never offered |
+| Every positive row in step 5 scores 0 | `SKILL_SOURCE_PATH` is unset **or one level too deep** — it must name a plugin root holding `skills/`, e.g. `.claude`, not `.claude/skills`. Either way the skill was never offered |
 
 To update after the marketplace moves, `/plugin marketplace update coder-eval`; to
 remove it, `/plugin uninstall`.
diff --git a/docs/tutorials/08-optimizing-a-skill.md b/docs/tutorials/08-optimizing-a-skill.md
new file mode 100644
index 00000000..e09f972c
--- /dev/null
+++ b/docs/tutorials/08-optimizing-a-skill.md
@@ -0,0 +1,515 @@
+# Optimizing a skill description
+
+`check-skill` tells you whether a skill triggers. This tutorial covers the next question:
+**can its description be made to trigger better, and how would you know?**
+
+The honest answer is usually "measure first, and often the answer is don't change it." This
+walkthrough is a real run against a real skill in this repository, reported with the numbers
+it actually produced — including the part where the first attempt measured nothing at all.
+
+**What you will do:** build an activation suite for `lint-tasks`, split it into tune and
+holdout rows, baseline it, read the confusion matrix, and decide whether to spend anything.
+For `lint-tasks` the answer turns out to be no — about 20 agent runs settle it. The suite
+then points at a sibling that *does* have headroom, `analyze`, and the full three-stage A/B
+runs against that: roughly 350 further runs, ending in a promotion that survives the
+holdout.
+
+All of it on Sonnet. Never Opus for a suite this size.
+
+**Prerequisites:** the `coder-eval` CLI, the plugin installed, and credentials for the
+`claude-code` agent. Start with [Your first evaluation](01-first-evaluation.md) if you have
+not run anything yet.
+
+## Why this skill
+
+`lint-tasks` was picked for four reasons, and they are worth stealing as selection criteria:
+
+1. **It is model-invokable.** Its description actually enters the activation decision. A
+   skill with `disable-model-invocation: true` — here `init`, `ci`, and `optimize-skill`
+   itself — can never be optimized this way, because its description never competes for
+   anything. `optimize-skill` hard-stops on exactly that.
+2. **It has a real boundary dispute with a sibling.** `lint-tasks` reviews existing tasks;
+   `task` authors new ones. *"Are my evals any good?"* and *"add a task for X"* genuinely
+   misroute between them.
+3. **Its description had just been trimmed** by 66 characters to fit the plugin's shared
+   listing budget. That trim was an unmeasured change to activation behaviour. This turns it
+   into a measured one.
+4. **It supplies its own distractors.** The description mentions auditing and gaps, so
+   *"audit my dependencies"* is a natural false-positive probe.
+
+## Step 1 — Build the suite
+
+Do not hand-author it. Run the sibling skill:
+
+```
+/coder-eval:check-skill lint-tasks
+```
+
+That produces a task YAML plus a JSONL row file. The suite used here has **21 rows**,
+counted by `expected_skill` — the field that actually decides a row's polarity:
+
+| Kind | `expected_skill` | Total | tune | holdout |
+| --- | --- | --- | --- | --- |
+| Positive | `lint-tasks` | 8 | 5 | 3 |
+| Distractor | `""` | 9 | 6 | 3 |
+| Sibling-owned | `task` | 3 | 2 | 1 |
+| Sibling-owned | `analyze` | 1 | 1 | 0 |
+
+**Label sibling rows by what should *fire*, not by whose territory it is.** Two rows here
+ask for project setup — work that belongs to `init`, which sets
+`disable-model-invocation: true` and therefore *cannot* be engaged by the model. Labelling
+them `expected_skill: "init"` would assert something unsatisfiable by construction: those
+rows could never pass, the suite could never go green, and `init`'s recall would be pinned
+at 0.0 while telling you nothing. They are labelled `""` instead, which asks the question
+that actually has an answer — *does some other skill wrongly claim this?*
+
+**Sizing, and why this suite is at the low end.** `check-skill` asks for 8–12 rows of each
+polarity. A split **halves each side**, so a suite you intend to optimize wants roughly
+16–24 of each. This one is smaller than that, deliberately, so the tutorial stays affordable
+— and you will see the cost of that in the result.
+
+## Step 2 — Label the splits
+
+Add a `split` to every row and point the dataset at the field:
+
+```yaml
+dataset:
+  paths:
+    - "lint-tasks-activation-rows.jsonl"
+  split_field: "split"
+```
+
+```jsonl
+{"id": "pos-1", "expected_skill": "lint-tasks", "split": "tune",    "prompt": "..."}
+{"id": "pos-5", "expected_skill": "lint-tasks", "split": "holdout", "prompt": "Are my evals any good?"}
+```
+
+Then select one at run time with `--split tune` or `--split holdout`. The filter runs
+**before** any sampling, so `--split tune --sample 8` is always drawn from tune rows alone.
+
+Keep both polarities on both sides. A holdout of only positives measures recall and calls it
+a result.
+
+## Step 3 — Make the skill reachable, and check that it worked
+
+This is the step that decides whether anything downstream means anything, so it gets a
+section of its own.
+
+```yaml
+agent:
+  type: "claude-code"
+  plugins:
+    - type: "local"
+      path: "$SKILL_SOURCE_PATH"
+  setting_sources: []
+```
+
+**`path` must be a plugin root** — a directory holding a `skills/` subdirectory, so the
+skill sits at `<path>/skills/<name>/SKILL.md`. A `.claude-plugin/plugin.json` is optional;
+without one the namespace defaults to the directory's own name.
+
+The intuitive path is the wrong one. For `.claude/skills/my-skill/SKILL.md` the root is
+**`.claude`**, not `.claude/skills`.
+
+```bash
+export SKILL_SOURCE_PATH="$(pwd)/plugins/coder-eval"   # the plugin root
+```
+
+!!! danger "The first baseline for this tutorial scored recall 0.0"
+
+    It pointed one level too deep, at the bare skills directory. Nothing loaded, no `Skill`
+    tool was ever offered, and every positive row failed. On an earlier, smaller draft of
+    this suite — 10 tune rows rather than the 14 below — that produced:
+
+    ```
+    lint-tasks   recall.yes=0.000 precision.yes=0.000 f1.yes=0.000
+                 confusion: [('no','no',4), ('yes','no',3)]
+    ```
+
+    Note the confusion matrix accounts for only 7 of the 10 rows; the missing 3 are the
+    next section.
+
+    A wrong path is only a **warning**, never an error. The run completes, the report
+    renders, and the number it produces is indistinguishable from a skill whose description
+    is hopeless. In a one-shot check that misleads you once; in an optimization loop it
+    poisons everything — every candidate scores zero, nothing separates from the incumbent,
+    and you conclude your rewrites are all bad when your wiring is broken.
+
+    **So the first thing to check is never the description. It is whether recall is
+    non-zero at all.**
+
+Point it at the plugin root and the same suite goes to `recall.yes=1.000`. Nothing about the
+descriptions changed.
+
+## Step 4 — Cap the run, or pay for exploration
+
+Activation is decided in the first assistant turn. Without caps the agent explores a sandbox
+that deliberately contains no eval files, and rows time out after five minutes each:
+
+```
+pos-1  ERROR  Agent turn timed out after 300s (iteration 1)
+```
+
+A timed-out row is **excluded from the confusion matrix**, not scored — so the metrics are
+computed over fewer rows than the dataset holds. In that first baseline 3 of 10 rows
+vanished this way, which is why its matrix summed to 7.
+
+The rollup does tell you, if you look. `suite.json` carries `rows_total`, `rows_excluded`
+and a `completion_rate` metric, and `suite.md` spells it out:
+
+```
+**Rows**: 14 total — 13 passed, 1 failed, 0 errored
+_Denominator: 14/14 rows (0 excluded)_
+```
+
+Because `completion_rate` is an ordinary metric, you can gate on it and stop trusting
+eroded runs by hand:
+
+```yaml
+suite_thresholds:
+  completion_rate: 1.0
+```
+
+```yaml
+run_limits:
+  max_turns: 2
+  turn_timeout: 120
+  task_timeout: 300
+```
+
+That buys signal, not just cost.
+
+## Step 5 — Baseline on the tune split
+
+```bash
+coder-eval run tasks/skills/lint-tasks-activation.yaml \
+  --split tune -D run_limits.stop_early=false
+```
+
+`stop_early: false` is insurance: if a suite arms early stop, a pass-stop can end a run
+before a later sibling misfire is observable, and authoritative precision needs the full
+trajectory. A stock `check-skill` suite arms nothing, so on that suite this flag changes
+nothing.
+
+**Check the resolved row count, not just the exit code.** A mistyped split (`--split holdou`)
+is reported as a skipped task and the run still **exits 0** — a green run over zero rows.
+
+Result, 14 tune rows:
+
+| Skill | recall.yes | precision.yes | f1.yes |
+| --- | --- | --- | --- |
+| `lint-tasks` | 1.000 | 1.000 | **1.000** |
+| `task` | 1.000 | 1.000 | 1.000 |
+| `analyze` | 0.000 | 0.000 | 0.000 |
+
+## Step 6 — Read it, and decide whether to spend anything
+
+**The target skill is at ceiling.** `lint-tasks` caught all five positives, including
+*"My evals are a mess. Can you go through them and tell me what to fix?"*, and stayed off all
+nine negatives, including *"Audit my dependencies for known vulnerabilities"* and *"Lint my
+Python code"* — the two probes designed to catch it over-claiming.
+
+**So the optimization loop stops here, and that is the correct outcome.** With F1 already at
+1.0 there are no false negatives or false positives to build a hypothesis from, and the
+promotion gate — `min(candidate F1) > max(incumbent F1)` — cannot be satisfied by any
+candidate. Running the three A/B stages anyway, on this suite with three candidates and two
+survivors, would have cost `(3+1)×14 + 3×(2+1)×14 + 6×7` = **224 further agent runs** to
+chase a number that is not reachable.
+
+The first finding is therefore about the change that prompted this: **the 66-character trim
+was safe.** That is now measured rather than assumed.
+
+The second finding is that **a ceiling result is as much a statement about the suite as about
+the description.** Fourteen well-separated rows could not distinguish a good description from
+a better one. If you need to tell those apart, the fix is more rows and harder ones — not a
+looser gate.
+
+### Where the headroom actually was
+
+The sibling matrix is the part worth reading, and in a multi-skill plugin it usually is:
+
+- **`analyze` under-claims.** *"Look at my evaluation results from last night's run and tell
+  me what regressed"* engaged **nothing at all**, in every run. Its description names "why a
+  run failed" but never regression, and nothing else claimed the request either.
+
+This is what sibling-owned rows buy you. A plain distractor would have said only that
+something misfired; these say **where the request went**, which is the difference between
+"this description is vague" and "these two descriptions are fighting."
+
+### The finding that evaporated
+
+An earlier pair of runs showed something else: on both *"Set up coder-eval for this
+repository"* and *"Get evaluations going in this project from scratch"*, the `task` skill
+fired — apparently annexing setup work — dropping its precision to 0.667 and 0.500. It
+reproduced on **both splits**, which is normally the signal that a finding is real.
+
+Running the same tune split three more times settled it. The row fires **two times in
+three**, on byte-identical prompts (`expected_skill` is a dataset label the criterion reads
+and the agent never sees, so nothing about the rows changed):
+
+```
+hard-3   expected=(distractor)   ['task', 'task', '-']    <-- UNSTABLE
+```
+
+So the effect is real but intermittent, and the two runs that agreed were luck, not
+evidence. **That is the whole reason the promotion gate demands non-overlapping replicate
+ranges.** Had this been a candidate description rather than an incumbent's quirk, those two
+agreeing runs would have "proved" an improvement worth shipping. Report what replicates;
+treat anything else as a hypothesis.
+
+Note what the same three runs said about `analyze`:
+
+```
+an-1     expected=analyze   ['analyze', 'analyze', 'analyze']
+hard-2   expected=analyze   ['-', '-', '-']
+```
+
+Recall **0.500 in all three runs**, precision 1.000 in all three. Perfectly stable, one row
+consistently missed, no over-claiming. That is a hypothesis worth spending on — which is
+where the rest of this tutorial goes.
+
+## Optimizing the skill that actually had headroom
+
+`lint-tasks` had nothing to fix. `analyze` did, so the loop ran for real against it. The
+hypothesis, phrased as a claim about specific rows: **the description never names
+regression or comparison between runs**, so requests about things getting *worse* find
+nothing to match.
+
+Three candidates, one per hypothesis, each differing from the incumbent only in `analyze`'s
+frontmatter `description`, each a full seven-skill snapshot:
+
+| Candidate | Hypothesis |
+| --- | --- |
+| `a-regression` | Name regression directly: *"what regressed or got worse since a previous run"* |
+| `b-results` | Users say "results" and "scores"; the description only says "run" |
+| `c-symptom` | Lead the trigger clause with symptoms rather than the operation |
+
+### Stage A — triage (68 runs)
+
+| Arm | analyze recall | precision | F1 | completion |
+| --- | --- | --- | --- | --- |
+| incumbent | 0.500 | 1.000 | 0.667 | 1.000 |
+| `a-regression` | 0.750 | 1.000 | **0.857** | 1.000 |
+| `c-symptom` | 0.750 | 1.000 | **0.857** | 1.000 |
+| `b-results` | 0.667 | 1.000 | 0.800 | **0.941** |
+
+`b-results` looks competitive and **is not comparable**: it lost a row to an error, so its
+recall was computed over 3 analyze rows instead of 4. Check `completion_rate` before ranking
+anything. The two clean leaders went through to the gate.
+
+### Stage B — the gate (153 runs, three separate invocations)
+
+| Arm | run 1 | run 2 | run 3 | Range |
+| --- | --- | --- | --- | --- |
+| incumbent | 0.667 | *(row excluded)* | 0.667 | [0.667, 0.667] |
+| `c-symptom` | 0.857 | 0.857 | 0.857 | [0.857, 0.857] |
+| **`a-regression`** | **1.000** | **1.000** | **1.000** | **[1.000, 1.000]** |
+
+Both candidates clear `min(candidate) > max(incumbent)` with no overlap, and every arm held
+`lint-tasks` and `task` at recall 1.000 — no sibling regression. `a-regression` wins
+outright: it turns the one consistently-missed row into a consistent hit without touching
+precision.
+
+One incumbent invocation dropped a row and was excluded from the gate rather than averaged
+in. A gate computed over a shifting denominator is not a gate.
+
+## Step 7 — Confirm on the holdout
+
+Even with nothing to promote, the holdout is what turns "the trim was safe" from a claim
+about the rows you looked at into a claim about rows you did not.
+
+```bash
+coder-eval run tasks/skills/lint-tasks-activation.yaml \
+  --split holdout -D run_limits.stop_early=false
+```
+
+| Skill | recall.yes | precision.yes | f1.yes |
+| --- | --- | --- | --- |
+| `lint-tasks` | 1.000 | 1.000 | **1.000** |
+| `task` | 1.000 | 1.000 | 1.000 |
+
+`lint-tasks` reproduces at ceiling on rows it was never checked against, including the
+oblique *"Are my evals any good?"*. Across every run in this tutorial — two wiring states,
+two splits, three sittings — it never missed a positive and never took a distractor. That is
+about as much as a suite this size can say, and it is enough to conclude the trim did no
+harm.
+
+### Stage C, and a holdout that could not answer the question
+
+The winner went to holdout as a **two-variant** experiment at `--repeats 3`, which is the one
+place `--repeats` is correct: `paired_comparison` fires only for exactly two variants and
+averages replicates per row before pairing.
+
+Both arms scored `analyze` F1 **1.000**, and the paired block was a flat tie:
+
+```
+**Paired mean diff (incumbent - a-regression)**: +0.000 [95% CI +0.000, +0.000], p = 1.000
+```
+
+That is not a confirmation. It is an **uninformative holdout**: its `analyze` rows happened
+to be ones the incumbent already caught, because every regression-phrased row had been put
+in the tune half. A holdout can only confirm a fix for a failure mode it actually contains.
+
+The remedy is the one `optimize-skill` prescribes — author **fresh holdout rows** that
+exercise the failure mode, at promotion time, when you know what needs confirming:
+
+```jsonl
+{"id": "an-6", "expected_skill": "analyze", "split": "holdout", "prompt": "Which of my tasks got worse after I switched the model?"}
+{"id": "an-7", "expected_skill": "analyze", "split": "holdout", "prompt": "Something in the suite degraded this week. Find out what."}
+```
+
+Write them as requests a real user would send, and commit to whatever they say — rows
+authored to flatter a candidate confirm nothing.
+
+### And then the infrastructure lied to us
+
+The re-run returned a result that looked publishable and was worthless:
+
+```
+**Paired mean diff (incumbent - a-regression)**: +0.162 [95% CI +0.011, +0.312], d = 0.72, p = 0.038
+```
+
+Read the sign: that says the **incumbent** was better, significantly. It is an artifact.
+`completion_rate` gives it away — `a-regression` lost 11 of 33 rows, the incumbent 6:
+
+```
+a-regression   rows_total=33 rows_excluded=11 completion=0.667
+incumbent      rows_total=33 rows_excluded= 6 completion=0.818
+```
+
+The excluded rows were not timeouts. They carried:
+
+```
+Communication with agent failed: Claude Code returned an error result
+Details: You've hit your org's monthly spend limit
+```
+
+A billing limit reached mid-run removed rows from one arm more than the other, and the
+paired test faithfully reported the resulting difference at *p* = 0.038. **A p-value
+computed over an eroded, asymmetric sample is not evidence of anything.** This is why
+`completion_rate` is a gateable metric, and why the first thing to read in a rollup is the
+denominator, not the effect. The run was discarded, not interpreted.
+
+### Stage C, run properly
+
+With budget restored, the same experiment ran again over the 11-row holdout. Erosion this
+time was one row against `a-regression` and none against the incumbent — near-symmetric, and
+pointing *against* the candidate, so any positive result is conservative:
+
+```
+a-regression   rows_total=33 rows_excluded=1 completion=0.970
+incumbent      rows_total=33 rows_excluded=0 completion=1.000
+```
+
+| Arm | analyze recall | precision | **F1** | siblings |
+| --- | --- | --- | --- | --- |
+| incumbent | 0.833 | 1.000 | **0.909** | `task` precision 0.750 |
+| **`a-regression`** | **1.000** | **1.000** | **1.000** | all 1.000 |
+
+**The direction reproduces on rows the candidate was never tuned against**, which is what
+Stage C is required to show. One row separates them — and it is one of the *fresh* rows
+authored at promotion time:
+
+```
+an-6  "Which of my tasks got worse after I switched the model?"
+        incumbent      ['analyze', '-', '-']              1 of 3
+        a-regression   ['analyze', 'analyze', 'analyze']  3 of 3
+```
+
+Every other holdout row is identical across the arms. The incumbent also shows the
+intermittent `task` misfire on the setup row once more (precision 0.750), consistent with
+the 2-in-3 rate measured earlier.
+
+**And the paired comparison says nothing:**
+
+```
+**Paired mean diff (incumbent - a-regression)**: +0.000 [95% CI -0.088, +0.088], d = 0.00, p = 1.000
+```
+
+That is not a contradiction — it is the documented limit of that block. It pairs per-row
+`weighted_score`, which averages **all three** criteria on every row, so a gain confined to
+one criterion on one row out of eleven is diluted below what 11 pairs can resolve. F1 is the
+promotion metric; the paired block corroborates direction when it can and, at this suite
+size, it cannot. Report both, and do not let a null paired result overturn a metric it was
+never measuring.
+
+**Verdict: promote.** Gated on tune (1.000 vs 0.667, non-overlapping, three invocations),
+confirmed in direction on holdout (1.000 vs 0.909), no sibling regression anywhere, and
+precision never off 1.000 in any run of either stage.
+
+## The three stages, and when each applies
+
+They did not run for `lint-tasks`, because that incumbent was already perfect. They did run
+for `analyze`, above. In summary:
+
+| Stage | What it does | Replicates |
+| --- | --- | --- |
+| **A — triage** | All candidates + incumbent, one invocation, `--split tune`. Rank by F1, discard anything at or below the incumbent. Decides nothing. | one run |
+| **B — gate** | Survivors + incumbent, `--split tune`, **three separate `coder-eval run` invocations**. Promote only on `min(candidate) > max(incumbent)`. | three runs |
+| **C — confirm** | Best survivor vs. incumbent only, `--split holdout --repeats 3`. Read the rendered `## Paired Comparison` block. | `--repeats 3` |
+
+**Stage B must be three invocations, not `--repeats 3`.** Suite rollups are keyed on
+`(variant, suite)` alone, so `--repeats` pools every replicate into a single `suite.json`
+with one confusion matrix — the per-replicate F1 the gate compares would not exist. The cost
+is identical either way.
+
+**Stage C is the one place `--repeats` is correct.** `paired_comparison` fires only for
+exactly two variants and averages replicates per row *before* pairing, which is what makes
+the paired *t*-test valid. It pairs per-row `weighted_score` — row correctness, not F1 — so
+it corroborates the direction; it does not re-test the promotion metric.
+
+And be precise about what Stage B proves. The replicate spread measures agent stochasticity
+over a **fixed** set of rows. It does not correct for the survivors having been chosen on
+those same rows in Stage A. Stage B bounds run noise; **the holdout is what bounds the fit.**
+
+## Two caveats about what else is in the sandbox
+
+**The machine's other skills are part of the experiment.** One distractor row engaged a
+skill named `review` — not from this plugin at all, but from the operator's own install.
+Activation is a competition for a shared listing budget, so whatever else is installed
+competes too. If you need a clean-room measurement, isolate the environment; if you want a
+realistic one, record what was installed alongside your results.
+
+**And `skill_name` matching is by bare name, which can collide.** The criterion strips any
+`plugin:` prefix before comparing, so `coder-eval:init` and a differently-owned `init` are
+indistinguishable to it. That is not hypothetical here: Claude Code ships its own unscoped
+`init` skill, and the setup rows in this suite engage *that* one. A criterion written as
+
+```yaml
+- type: "skill_triggered"
+  skill_name: "init"
+```
+
+would have quietly scored a different skill's activation as though it were the plugin's.
+
+Before trusting a `skill_triggered` result, check that the skill's bare name is unique among
+everything installed — `/context` and `/doctor` both list the active set. A colliding name
+does not error; it just measures the wrong thing. Where a collision exists, the honest
+options are to rename the skill, or to measure it somewhere the collision is absent.
+
+## What to take away
+
+- **Check that recall is non-zero before you believe any low score.** A misconfigured path
+  looks exactly like a bad description and costs the same to produce.
+- **Read the denominator before the effect.** `completion_rate` invalidated two separate
+  comparisons here — one candidate ranked on 3 rows instead of 4, and a *p* = 0.038 result
+  that was a billing limit eroding one arm harder than the other. An effect measured over a
+  sample that moved is not an effect.
+- **A ceiling baseline means stop.** `lint-tasks` was perfect, so the loop declined to spend
+  224 runs chasing a number the gate makes unreachable. Then it found the real headroom next
+  door.
+- **Two agreeing runs are not evidence.** The `task` misfire reproduced on both splits and
+  still turned out to be 2-in-3 variance. Only replicates told the difference.
+- **A holdout only confirms failure modes it contains.** The first one here was a flat tie
+  because every regression-phrased row sat in the tune half. Fresh rows, authored to test the
+  hypothesis rather than to flatter the candidate, are the fix.
+- **A holdout confirms direction, not significance.** `a-regression` shipped on F1 1.000 vs
+  0.909 on unseen rows while the paired *t*-test read exactly zero. At eleven pairs, across
+  three criteria, that block cannot resolve a one-criterion gain — which is a fact about the
+  test, not about the change. Report both and say which one you promoted on.
+
+## Next
+
+- [Comparing models](05-comparing-models.md) — the same experiment-variant machinery, applied to models instead of descriptions.
+- [Bring Your Own Dataset](../DATASETS.md) — split labels, sampling precedence, and suite-level thresholds in full.
+- [Claude Code plugin](../PLUGIN.md) — the other six skills.
diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md
index 0c5c974f..b74dafb2 100644
--- a/docs/tutorials/README.md
+++ b/docs/tutorials/README.md
@@ -20,6 +20,7 @@ the task-file schema see the [Task Definition Guide](../TASK_DEFINITION_GUIDE.md
 | 05 | [Comparing two models](05-comparing-models.md) | Use the experiment layer to A/B two configurations |
 | 06 | [Running tasks in Docker isolation](06-use-docker-isolation.md) | Run each task in a fresh container; add task-specific dependencies |
 | 07 | [Driving Coder Eval from Claude Code](07-plugin-in-claude-code.md) | Install the plugin and drive the whole loop — scaffold, author, review, run, analyze — from slash commands |
+| 08 | [Optimizing a skill description](08-optimizing-a-skill.md) | Measure whether a skill's description can be improved — tune/holdout splits, the reachability check that decides everything, and when to stop |
 
 > Contributions welcome — add a numbered `NN-title.md` file and link it in the
 > table above. Keep tutorials short, copy-pasteable, and outcome-focused.
diff --git a/mkdocs.yml b/mkdocs.yml
index 5b42c6cb..d12dc7f2 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -105,6 +105,7 @@ nav:
       - 05 · Comparing two models: tutorials/05-comparing-models.md
       - 06 · Docker isolation: tutorials/06-use-docker-isolation.md
       - 07 · Driving Coder Eval from Claude Code: tutorials/07-plugin-in-claude-code.md
+      - 08 · Optimizing a skill description: tutorials/08-optimizing-a-skill.md
   - Guides:
       - User Guide: USER_GUIDE.md
       - Task Definition Guide: TASK_DEFINITION_GUIDE.md
diff --git a/plugins/coder-eval/README.md b/plugins/coder-eval/README.md
index 316ce8be..9164a4c5 100644
--- a/plugins/coder-eval/README.md
+++ b/plugins/coder-eval/README.md
@@ -5,7 +5,7 @@ Code — including whether your own Claude Code skills actually trigger.
 
 Coder Eval runs a real coding agent in a sandbox against declarative YAML tasks and scores
 the files and commands it actually produced. This plugin puts the authoring, running, and
-analysis loop behind six slash commands.
+analysis loop behind seven slash commands.
 
 ## Install
 
@@ -34,18 +34,20 @@ Running a suite also needs credentials for whichever agent the tasks use (e.g.
 `ANTHROPIC_API_KEY` for the default `claude-code` agent). `lint-tasks` needs neither the CLI
 nor credentials — it only reads files.
 
-## The six skills
+## The seven skills
 
 | Command | What it does |
 | --- | --- |
 | `/coder-eval:init` | Scans the repo for what is worth evaluating (skills, an MCP server, a CLI), then scaffolds a task directory with one real task. |
 | `/coder-eval:check-skill` | Generates and runs an activation suite for one of your skills — does the agent engage it when it should, and leave it alone when it shouldn't? |
+| `/coder-eval:optimize-skill` | Turns that suite's confusion matrix into candidate description rewrites, A/B tests them as experiment variants, and promotes only what survives a held-out split. |
 | `/coder-eval:task` | Turns a natural-language description into a task YAML with the right success criteria. |
 | `/coder-eval:lint-tasks` | Reviews task YAML you already have and reports criteria that cannot fail, prompts that give away the answer, and fixtures with no cleanup. Read-only. |
 | `/coder-eval:analyze` | Reads a finished run directory and reports systemic failure patterns, per-task findings, and concrete fixes. |
 | `/coder-eval:ci` | Emits a GitHub Actions workflow that runs your suite as a CI gate (or on a schedule, to catch skill drift). |
 
-`init` and `ci` are explicit-invocation only. `check-skill`, `task`, `lint-tasks`, and
+`init`, `ci` and `optimize-skill` are explicit-invocation only — the last because it spends
+real money across several rounds of runs. `check-skill`, `task`, `lint-tasks`, and
 `analyze` can also be reached by the agent on its own when a request clearly calls for them.
 
 ## Bundled reference
diff --git a/plugins/coder-eval/reference/run-layout.md b/plugins/coder-eval/reference/run-layout.md
index a080dedb..f009b51c 100644
--- a/plugins/coder-eval/reference/run-layout.md
+++ b/plugins/coder-eval/reference/run-layout.md
@@ -13,6 +13,53 @@ runs/<run_id>/<variant_id>/<task_id>/<NN>/{task.json, task.log, artifacts/}
 - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it.
 - `task.log` — the human-readable task log; `artifacts/` — files the agent produced.
 
+## Suite rollups (dataset-backed tasks only)
+
+A task carrying `dataset:` fans out into one row-task per row and additionally writes a
+per-suite rollup:
+
+```
+runs/<run_id>/<variant_id>/<suite_id>/{suite.json, suite.md}
+```
+
+`<suite_id>` is the original (pre-fan-out) `task_id`. Nothing is written for a task
+without `dataset:`.
+
+`suite.json` carries the suite's pass counts plus `criterion_aggregates[]` — one entry per
+criterion that opted into across-row aggregation, each with:
+
+- `criterion_type`, and `description` (set when a task stacks several criteria of the same
+  type, e.g. one `skill_triggered` per skill — that is what distinguishes them);
+- `rows_total` and `rows_excluded` — the denominator and what was dropped from it. A row
+  that errored before criteria ran (a timeout, say) is **excluded** rather than scored, so
+  metrics are computed over `rows_total - rows_excluded`;
+- `metrics` — a **flat** name → float map. Classification-style criteria emit
+  `accuracy`, `macro_f1`, and per-label `precision.<label>` / `recall.<label>` /
+  `f1.<label>` (for `skill_triggered` the labels are `yes` / `no`). It also carries
+  `completion_rate` — the surviving fraction of the denominator above — which is an
+  ordinary metric, so `suite_thresholds: {completion_rate: 1.0}` gates a run whose sample
+  eroded;
+- `details` — `labels`, `per_label`, `confusion`, `total_pairs`;
+- `threshold_checks` + `passed`, from the criterion's `suite_thresholds`.
+
+Alongside the aggregates, `failed_samples[]` lists failed/errored rows **by `row_id`** with
+their failure reasons, capped at a fixed limit. It is the only place in `suite.json` that
+carries row identity — `metrics` and `details` are counts only — so a consumer that needs
+to know *which* rows failed reads `failed_samples`, then falls back to the per-row
+`<variant_id>/<suite_id>/<row_id>/<NN>/task.json` for anything past the cap.
+
+**Read `metrics`; never recompute a metric from `details.confusion`.** The criterion layer
+owns that arithmetic, including its division-by-zero convention, and a consumer that
+re-derives F1 will disagree with the gate the run already applied.
+
+**Rollups pool replicates.** The grouping key is `(variant_id, suite_id)` — the replicate
+index is *not* part of it. So `--repeats N` over an M-row suite writes **one** `suite.json`
+per variant, with `rows_total: N × M` and a single pooled confusion matrix. There is no
+per-replicate metric in that file. A consumer that needs replicate-to-replicate spread must
+invoke the run N times and read N run directories; `--repeats` cannot serve that purpose.
+(Contrast `experiment.md`'s `## Paired Comparison` block, which averages replicates per row
+before pairing — there `--repeats` is exactly the right tool.)
+
 **Scope-marker files** (used to detect what a given path represents):
 
 - `run.json` at the run root → **run scope**. If `experiment.json` (+ `experiment.md`) is also present → multi-variant experiment.
diff --git a/plugins/coder-eval/reference/templates/activation-rows.jsonl b/plugins/coder-eval/reference/templates/activation-rows.jsonl
index f6ed667a..6e5a3ccf 100644
--- a/plugins/coder-eval/reference/templates/activation-rows.jsonl
+++ b/plugins/coder-eval/reference/templates/activation-rows.jsonl
@@ -1,6 +1,6 @@
-{"id": "pos-1", "expected_skill": "my-skill", "prompt": "REPLACE: the most obvious request my-skill exists to handle, in a user's own words. Paraphrase the skill's description — never quote it."}
-{"id": "pos-2", "expected_skill": "my-skill", "prompt": "REPLACE: the same need phrased with different vocabulary from the description, so a pass proves activation rather than string overlap."}
-{"id": "pos-3", "expected_skill": "my-skill", "prompt": "REPLACE: an oblique in-scope request — the user describes their problem, not the operation the skill performs."}
-{"id": "neg-1", "expected_skill": "", "prompt": "REPLACE: an adjacent request that shares vocabulary with the description but is out of scope for my-skill."}
-{"id": "neg-2", "expected_skill": "", "prompt": "REPLACE: a request in the same domain that a different tool or skill should handle."}
-{"id": "neg-3", "expected_skill": "", "prompt": "REPLACE: an everyday request with no relationship to my-skill at all, so a misfire here means the description over-claims badly."}
+{"id": "pos-1", "expected_skill": "my-skill", "split": "tune", "prompt": "REPLACE: the most obvious request my-skill exists to handle, in a user's own words. Paraphrase the skill's description — never quote it."}
+{"id": "pos-2", "expected_skill": "my-skill", "split": "tune", "prompt": "REPLACE: the same need phrased with different vocabulary from the description, so a pass proves activation rather than string overlap."}
+{"id": "pos-3", "expected_skill": "my-skill", "split": "holdout", "prompt": "REPLACE: an oblique in-scope request — the user describes their problem, not the operation the skill performs."}
+{"id": "neg-1", "expected_skill": "", "split": "tune", "prompt": "REPLACE: an adjacent request that shares vocabulary with the description but is out of scope for my-skill."}
+{"id": "neg-2", "expected_skill": "", "split": "tune", "prompt": "REPLACE: a request in the same domain that a different tool or skill should handle."}
+{"id": "neg-3", "expected_skill": "", "split": "holdout", "prompt": "REPLACE: an everyday request with no relationship to my-skill at all, so a misfire here means the description over-claims badly."}
diff --git a/plugins/coder-eval/reference/templates/activation.yaml b/plugins/coder-eval/reference/templates/activation.yaml
index a54c083d..582c2a7e 100644
--- a/plugins/coder-eval/reference/templates/activation.yaml
+++ b/plugins/coder-eval/reference/templates/activation.yaml
@@ -4,20 +4,40 @@
 #
 # One row per prompt; each row is scored on whether the agent engaged the skill.
 # Positive rows set expected_skill to the skill's name, distractor rows set "".
+#
+# Rows also carry a `split` — `tune` or `holdout`. `/coder-eval:optimize-skill`
+# proposes description rewrites against `tune` and confirms the winner on `holdout`,
+# so an improvement has to survive rows it was not fitted to. Select one with
+# `coder-eval run <this file> --split tune`; with no --split every row runs.
+#
+# A split HALVES each side, so it multiplies the sizing rule: a suite you intend to
+# optimize wants 16-24 of each polarity, not the 8-12 a one-shot check aims for.
+# The six rows below are the illustrative minimum, not a target — split 4/2 they are
+# too small to gate an optimization round on, and optimize-skill will say so.
 task_id: "my-skill-activation"
 description: "Does the agent engage `my-skill` when it should, and leave it alone when it shouldn't?"
 tags: [activation]
 
 # REPLACE: the skill under test must be REACHABLE by the sandboxed agent, or every
 # positive row scores 0 and the suite reports recall 0.0 — which reads exactly like a
-# broken skill. `path` is the directory CONTAINING the skill's directory (for
-# `.claude/skills/my-skill/SKILL.md` that is `.claude/skills`), supplied through an
-# environment variable so the committed suite stays portable across machines and CI:
+# broken skill.
+#
+# `path` must be a PLUGIN ROOT: a directory holding a `skills/` subdirectory, i.e.
+# `<path>/skills/<skill-name>/SKILL.md`. A `.claude-plugin/plugin.json` is optional —
+# without one the namespace defaults to the directory's own name.
+#
+# So for `.claude/skills/my-skill/SKILL.md` the path is `.claude` — NOT `.claude/skills`.
+# Pointing at a bare directory of skill directories loads NOTHING, silently, and produces
+# exactly the recall 0.0 this comment warns about.
+#
+#   export SKILL_SOURCE_PATH=/abs/path/to/.claude
 #
-#   export SKILL_SOURCE_PATH=/abs/path/to/.claude/skills
+# Supplied through an environment variable so the committed suite stays portable across
+# machines and CI. An unset or wrong path is only a warning, so confirm the first run's
+# recall is non-zero before trusting a low score.
 #
-# An unset variable is logged as a warning and leaves the skill unreachable, so check
-# the first run's recall before trusting a low score.
+# `skill_name` below stays the BARE name even when a plugin namespaces the skill as
+# `plugin:skill` — the checker strips the namespace before comparing.
 agent:
   plugins:
     - type: "local"
@@ -26,6 +46,9 @@ agent:
 dataset:
   paths:
     - "activation-rows.jsonl"
+  # Row field naming each row's split. `coder-eval run --split tune` keeps only
+  # matching rows, and filters before any sampling so the selection is predictable.
+  split_field: "split"
 
 initial_prompt: |
   ${row.prompt}
diff --git a/plugins/coder-eval/skills/analyze/SKILL.md b/plugins/coder-eval/skills/analyze/SKILL.md
index c9d84242..39e548ac 100644
--- a/plugins/coder-eval/skills/analyze/SKILL.md
+++ b/plugins/coder-eval/skills/analyze/SKILL.md
@@ -1,5 +1,5 @@
 ---
-description: Analyze a finished coder-eval run and write analysis.md — cluster failures into systemic patterns, diagnose prompts, criteria, config, environment and cost, and recommend concrete fixes. Use when the user wants to know why a run failed, what to fix, or what a run says about their tasks.
+description: Analyze a finished coder-eval run and write analysis.md — cluster failures into systemic patterns and recommend fixes. Use when the user wants to know why a run failed, what regressed or got worse since a previous run, what to fix, or what a run says about their tasks.
 allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"]
 ---
 
diff --git a/plugins/coder-eval/skills/check-skill/SKILL.md b/plugins/coder-eval/skills/check-skill/SKILL.md
index e451a7af..e77f80fe 100644
--- a/plugins/coder-eval/skills/check-skill/SKILL.md
+++ b/plugins/coder-eval/skills/check-skill/SKILL.md
@@ -1,5 +1,5 @@
 ---
-description: Generate and run a coder-eval activation suite for a Claude Code skill — does the agent actually engage it when it should, and leave it alone when it shouldn't? Use when the user asks whether a skill triggers, wants to test skill activation, or worries a skill has silently stopped firing.
+description: Generate and run a coder-eval activation suite for a Claude Code skill. Use when the user asks whether a skill triggers, wants to test skill activation, or worries a skill has silently stopped firing.
 allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"]
 ---
 
@@ -156,6 +156,17 @@ relative to the task file):
 - each positive row's `expected_skill` → the same bare name; distractor rows keep `""`;
   a sibling-owned row's `expected_skill` → that **sibling's** bare name
 - every row's `prompt` → the requests designed in step 4, one JSON object per line
+- **every row's `split`** → `tune` or `holdout`, keeping both polarities on both sides
+
+**Label every row, including any you add.** The template ships `split` on all six rows and
+`dataset.split_field` on the task, which lets `coder-eval run --split tune` select a subset
+and `/coder-eval:optimize-skill` develop against one half while confirming on the other.
+
+A **partly** labelled dataset is the one state to avoid: `--split` keeps the matching rows
+and silently **drops the unlabelled ones**, so the suite shrinks and the aggregate metrics
+your `suite_thresholds` gate on are computed over fewer rows than you think. Either label
+every row or label none — never some. (With no labels anywhere, `--split` leaves the task
+untouched, which is safe.)
 
 **Then make the skill reachable, which is the step that decides whether the suite measures
 anything.** The task runs in a fresh sandbox that contains none of the user's files, so the
@@ -164,12 +175,17 @@ block in the template — and it is the template's job only **when nothing alrea
 skill**: if step 3 found an experiment supplying that block, inherit it and delete the
 template's copy rather than writing a second declaration.
 
-Otherwise, fill it in: `path` is the directory **containing** the skill's own directory —
-for `.claude/skills/pdf-forms/SKILL.md` that is `.claude/skills`. Tell the user to export it
-before running, and to use the same variable in CI:
+Otherwise, fill it in. **`path` must be a plugin root: a directory holding a `skills/`
+subdirectory**, so that the skill sits at `<path>/skills/<skill-name>/SKILL.md`. A
+`.claude-plugin/plugin.json` is optional — without one the namespace defaults to the
+directory's own name.
+
+For `.claude/skills/pdf-forms/SKILL.md` that root is **`.claude`**, not `.claude/skills`.
+Pointing at a bare directory of skill directories loads nothing at all. Tell the user to
+export it before running, and to use the same variable in CI:
 
 ```bash
-export SKILL_SOURCE_PATH="$(pwd)/.claude/skills"
+export SKILL_SOURCE_PATH="$(pwd)/.claude"
 ```
 
 Keep it an environment variable rather than baking an absolute path into the YAML — the
@@ -182,6 +198,14 @@ invoked as `plugin:skill` — the checker strips the namespace before comparing.
 namespaced value here silently scores zero recall on every row, which reads exactly
 like a broken skill.
 
+**Because matching is by bare name, it cannot survive a name collision.** Two skills called
+`init` — one from a plugin, one built in — are the same string to the criterion, so it will
+credit whichever fires as though it were the one under test. Check the name is unique across
+everything installed before trusting a result (`/context` and `/doctor` list the active set).
+A collision does not error; it measures the wrong skill. If one exists, say so rather than
+reporting the number — renaming the skill, or measuring where the collision is absent, are
+the only honest fixes.
+
 For criterion fields beyond this template, read
 `${CLAUDE_PLUGIN_ROOT}/reference/criteria.md`.
 
diff --git a/plugins/coder-eval/skills/ci/SKILL.md b/plugins/coder-eval/skills/ci/SKILL.md
index 9b367b7e..c5bd203a 100644
--- a/plugins/coder-eval/skills/ci/SKILL.md
+++ b/plugins/coder-eval/skills/ci/SKILL.md
@@ -153,11 +153,17 @@ find**, resolved against the checkout:
 ```yaml
 env: |
   ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
-  SKILL_SOURCE_PATH=${{ github.workspace }}/.claude/skills
+  SKILL_SOURCE_PATH=${{ github.workspace }}/.claude
 ```
 
 Use the directory the repository actually keeps skills in, from step 1, not the path
-above. This is the one omission the scheduled trigger cannot survive: unset, the skill is
+above — and note **what level that variable points at**. A local plugin path must be a
+**plugin root**: a directory holding a `skills/` subdirectory, so the skill sits at
+`<path>/skills/<name>/SKILL.md`. For `.claude/skills/my-skill/SKILL.md` that is `.claude`,
+**not** `.claude/skills`. Pointing one level too deep loads nothing at all and produces the
+same permanent red as leaving it unset.
+
+This is the one omission the scheduled trigger cannot survive: unset, the skill is
 never offered to the sandboxed agent, every positive row scores 0, and the job fails its
 `recall` threshold every week — a permanent red that looks exactly like the drift the
 schedule exists to detect, so the real thing goes unnoticed when it arrives.
diff --git a/plugins/coder-eval/skills/init/SKILL.md b/plugins/coder-eval/skills/init/SKILL.md
index 1ea3819f..ce1b5d26 100644
--- a/plugins/coder-eval/skills/init/SKILL.md
+++ b/plugins/coder-eval/skills/init/SKILL.md
@@ -1,5 +1,5 @@
 ---
-description: Set up coder-eval in this repository — scan for what is worth evaluating (Claude Code skills, an MCP server, a CLI), then scaffold a task directory with one real, passing-or-failing task and the exact command to run it.
+description: Set up coder-eval in this repository — scan for what is worth evaluating (Claude Code skills, an MCP server, a CLI), then scaffold a task directory with one real task and the command to run it.
 disable-model-invocation: true
 allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"]
 ---
diff --git a/plugins/coder-eval/skills/lint-tasks/SKILL.md b/plugins/coder-eval/skills/lint-tasks/SKILL.md
index 03b8efc1..93988af8 100644
--- a/plugins/coder-eval/skills/lint-tasks/SKILL.md
+++ b/plugins/coder-eval/skills/lint-tasks/SKILL.md
@@ -1,5 +1,5 @@
 ---
-description: Review coder-eval task YAML that already exists — find criteria that cannot fail, prompts that give away the answer, fixtures with no cleanup, and near-duplicate tasks, each with a severity and a concrete fix. Read-only. Use when the user wants existing tasks reviewed, linted, audited, or checked for gaps.
+description: Review existing coder-eval task YAML — find criteria that cannot fail, prompts that leak the answer, and near-duplicate tasks, each with a fix. Read-only. Use when the user wants existing tasks reviewed, linted, audited, or checked for gaps.
 allowed-tools: ["Read", "Glob", "Grep"]
 disallowed-tools: ["Write", "Edit", "NotebookEdit"]
 ---
diff --git a/plugins/coder-eval/skills/optimize-skill/SKILL.md b/plugins/coder-eval/skills/optimize-skill/SKILL.md
new file mode 100644
index 00000000..2b695e23
--- /dev/null
+++ b/plugins/coder-eval/skills/optimize-skill/SKILL.md
@@ -0,0 +1,349 @@
+---
+description: Improve a skill description from activation-suite results — propose rewrites, A/B them as experiment variants, promote only what wins on a held-out split. Use when a skill triggers too rarely or fires on the wrong requests.
+disable-model-invocation: true
+allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"]
+---
+
+# Optimize a skill description
+
+`/coder-eval:check-skill` answers *does this skill trigger?* This answers the next
+question: **can the description be made to trigger better, and how would you know?**
+
+The method is an A/B test, not a rewrite. Candidate descriptions become experiment
+variants, every arm runs the same labelled suite, and a candidate is promoted only when it
+beats the incumbent by more than the run-to-run noise — then survives rows it was never
+tuned on. Most rounds promote nothing. That is a real result, not a failure.
+
+The user's request is: `$ARGUMENTS`
+
+**Every row is a full agent run.** This skill spends real money across three stages. State
+the projected run count and ask before each one.
+
+## Step 1 — Confirm coder-eval is installed
+
+Run `coder-eval --version`. Every later step shells out to it.
+
+If it is missing, follow `${CLAUDE_PLUGIN_ROOT}/reference/cli-setup.md`: offer the install,
+**ask before running it**, and confirm afterwards. Never install unprompted. That reference
+also covers whether this project pins a version and what to do when the installed one
+disagrees.
+
+## Step 2 — Locate the skill, and check it can be optimized at all
+
+`$ARGUMENTS` may be a path to a `SKILL.md`, a path to the skill's directory, a skill name,
+or empty. Empty: glob `.claude/skills/*/SKILL.md` and `**/skills/*/SKILL.md`; one match →
+use it, several → ask, none → say so and stop.
+
+The **bare skill name is the directory name** containing `SKILL.md`. Locate the eval tree
+by following `${CLAUDE_PLUGIN_ROOT}/reference/repo-layout.md` — discover it, never assume a
+path.
+
+Two hard stops, both before anything is spent:
+
+- **No frontmatter `description`.** That is the finding. A skill with no description can
+  never be model-invoked, so there is nothing to optimize and a suite would score zero
+  recall by construction.
+- **`disable-model-invocation: true`.** The description never enters the activation
+  decision for such a skill, so an activation suite measures nothing and a rewrite round is
+  pure spend. Say so and stop. What *does* drive discovery for an explicit-invocation skill
+  is its name, its body, and the documentation telling users the command exists — offer
+  those instead.
+
+Read the incumbent `description` and keep it in front of you. It is the only thing this
+skill changes.
+
+## Step 3 — Find its activation suite
+
+Glob the eval tree for a task carrying a `skill_triggered` criterion naming this skill.
+
+**No suite → stop and point at `/coder-eval:check-skill`.** Do not author one here; that
+skill owns row design, and a suite written as a side effect of optimizing is a suite fitted
+to the thing it is meant to judge.
+
+**Suite adequacy is `check-skill`'s to define** — it owns the sizing rule, and this skill
+does not restate its numbers. But state the part that belongs here: **a split halves each
+side.** A suite sized for a single one-shot check is undersized for an optimization loop,
+because you are now measuring a *difference* between arms rather than one level, and the
+tune half is all you get to develop against. Roughly double what a one-shot check would
+want. If the suite is too small to gate on, say so and hand back rather than producing a
+confident number from four rows.
+
+## Step 4 — Require split labels
+
+The suite's rows must carry split labels, and the run selects one with `--split`.
+
+Without a holdout, every reported improvement is fitted to the sample you tuned on: with
+3–5 candidates scored on the same rows, the winner is partly whichever one happened to suit
+those rows. The holdout is what separates a real gain from that.
+
+- **Rows already labelled** → carry on.
+- **Rows *partly* labelled** → stop and finish the labelling first. This is the dangerous
+  state, because it does not look like one: `--split` keeps the matching rows and silently
+  **drops the unlabelled ones**, so the run succeeds, the report renders, and every metric
+  is computed over a smaller suite than the file suggests. Count the rows the run actually
+  resolved against the rows in the dataset before believing any number.
+- **Rows unlabelled and the suite is big enough to halve** → explain the discipline and
+  offer to add `split_field` plus a `tune`/`holdout` label per row, keeping both polarities
+  on both sides.
+- **Rows unlabelled and the suite is too small to halve** → do not carve a token two-row
+  holdout out of an already scarce sample; it confirms nothing and reads as if it did. The
+  better trade is to keep every row in `tune` and author **fresh holdout rows at promotion
+  time**, when you know which single candidate needs confirming. Offer that.
+
+  If you take that route, note what follows: with every row labelled `tune`, a later
+  `--split holdout` matches nothing, which is reported as a skipped task and a green run of
+  zero rows. Write the fresh rows first, then run Stage C.
+
+**Refuse to run a proposal round with neither.** Say why rather than proceeding quietly.
+
+## Step 5 — Baseline
+
+```bash
+coder-eval run <suite> --split tune -D run_limits.stop_early=false
+```
+
+**Check the resolved row count, not just the exit code.** A mistyped split name (`--split
+holdou`) is reported as a skipped task and the run still exits 0 — a green run of zero rows.
+If the count is zero or lower than the tune rows you expect, that is a wiring problem, not a
+result.
+
+On `stop_early`: a suite that arms it can pass-stop a run before a later sibling misfire is
+observable, so authoritative precision/recall needs a full run. Say plainly that a stock
+`check-skill` suite arms nothing, so on that suite the flag changes nothing — it is
+insurance against an armed suite, not a fix for anything already there.
+
+## Step 6 — Diagnose
+
+Read `<run>/<variant>/<suite>/suite.json`; its shape is documented in
+`${CLAUDE_PLUGIN_ROOT}/reference/run-layout.md`.
+
+Take the aggregate whose criterion names **this** skill. `details.confusion` and
+`details.per_label` give you the *counts* — how many false negatives (rows that should have
+engaged it and did not) and how many false positives (rows that engaged it and should not
+have).
+
+**Counts are not enough here; you need the rows.** Neither field carries a row identity.
+For that, read `failed_samples[]` in the same `suite.json` (each entry names its `row_id`,
+though the list is capped) and, when a row you need is not in it, the per-row
+`<run>/<variant>/<suite_id>/<row_id>/<NN>/task.json`. Pair each failing row with the prompt
+that produced it before drawing any conclusion — a hypothesis about wording that cannot
+name the requests it explains is not testable.
+
+**Group failures into named hypotheses, each pointing at specific rows.** *"It misses
+oblique requests because the description names the operation and never the symptom."*
+*"It fires on 'audit my dependencies' because the description claims audit vocabulary
+generally."* Not "improve the wording" — a hypothesis you cannot phrase as a claim about
+specific rows is not one you can test.
+
+**Sibling-owned rows.** A suite may carry rows whose `expected_skill` names a *different*
+skill in the same repository, with one `skill_triggered` criterion stacked per skill —
+which yields a per-skill confusion matrix from the same traces. That turns "this candidate
+misfires" into "this candidate is stealing the other skill's requests", which is the
+finding that actually tells you what to write. If the repository has sibling skills and the
+suite has no sibling rows, say the sibling half of the gate cannot be evaluated, and offer
+to hand back to `/coder-eval:check-skill` to add them.
+
+## Step 7 — Propose 3–5 candidates
+
+One candidate per hypothesis. Each candidate is a snapshot of **the whole skills
+directory**, not of one skill:
+
+```
+.optimize-skill/<skill>/<round>-<slug>/        <- this path is what a variant mounts
+    skills/
+        <skill-name>/SKILL.md                  <- the candidate: description rewritten
+        <sibling-a>/SKILL.md                   <- every sibling, copied unchanged
+        <sibling-b>/SKILL.md
+```
+
+**The `skills/` level is required, not decorative.** A local plugin path must be a **plugin
+root** — a directory holding a `skills/` subdirectory. Mount a bare directory of skill
+directories and nothing loads at all, which is the recall-0.0 failure below.
+
+The round-slug directory is the arm's replacement for the user's whole skill source, so it
+must contain everything that source contained. **Copy the siblings in unchanged.** Two
+things break if you snapshot only the target skill, and both fail silently at full cost:
+
+- Sibling `skill_triggered` criteria would observe `no` on every row in every arm, because
+  the sibling is not in the sandbox at all. The sibling half of the gate would
+  then "pass" by measuring nothing.
+- The listing the model chooses from would contain one skill instead of the real set.
+  Activation is a *competition* between descriptions; a description tested alone is tested
+  against a rival it will never face.
+
+Each candidate differs from the incumbent **only in the target skill's frontmatter
+`description`**. Body edits are out of scope here — say so if asked, and note the body does
+not drive activation anyway, so changing it would not be measured by this suite.
+
+Snapshot the incumbent the same way (`<round>-incumbent/`), siblings included, so every arm
+is mounted by the identical mechanism and the comparison has no confound.
+
+## Step 8 — Materialize as an experiment
+
+One variant per candidate, plus `incumbent`. **Reachability uses `agent.plugins`, the same
+mechanism the activation template itself uses** — `path` is the round-slug directory, which
+is a plugin root holding `skills/`, written **absolute**:
+
+```yaml
+experiment_id: "optimize-my-skill-round-1"
+defaults:
+  run_limits:
+    stop_early: false
+variants:
+  - variant_id: incumbent
+    agent:
+      plugins:
+        - type: "local"
+          path: "/abs/path/to/.optimize-skill/<skill>/<round>-incumbent"
+  - variant_id: cand-a-widen-vocabulary
+    agent:
+      plugins:
+        - type: "local"
+          path: "/abs/path/to/.optimize-skill/<skill>/<round>-a-widen-vocabulary"
+```
+
+`experiment_id` is required and must be kebab-case — an experiment without one fails to
+load. Name it for the skill and the round.
+
+Five facts that decide whether this measures anything:
+
+- A variant's `plugins` block **replaces** the task's rather than stacking with it. That is
+  what makes one-variant-per-candidate work — an arm never mounts two copies of the skill.
+- **Because it replaces, the task's own block is gone in every arm.** Whatever that block
+  exposed — typically the user's whole skills directory via `$SKILL_SOURCE_PATH` — is not
+  mounted. That is exactly why step 7's snapshot has to carry the siblings: the variant
+  path is the *only* skill source the arm gets.
+- Plugin paths resolve against the **process working directory**, not the task file's
+  directory. A relative path silently points elsewhere the moment the user runs from a
+  subdirectory, so write absolute paths.
+- Each `path` is a **plugin root**, so it must hold `skills/<skill-name>/SKILL.md`. This is
+  the single easiest thing to get wrong, and it is silent.
+- A wrong `path` leaves the skill unreachable and **every arm scores recall 0.0**. In a
+  one-shot check that reads as "broken skill"; in an optimization loop it reads as "all my
+  candidates are bad", which is far more convincing and completely wrong. An unset
+  environment variable in a path is only a **warning**, so a silent zero is the expected
+  symptom of this mistake.
+
+**The wiring check: confirm the `incumbent` arm reproduces the baseline's F1 before
+trusting any comparison.** If it does not, stop — the mounting is wrong and nothing
+downstream means anything.
+
+## Step 9 — Three stages, and the gate
+
+State the projected run count before each stage and ask. With N candidates, S survivors,
+M<sub>tune</sub> tune rows and M<sub>holdout</sub> holdout rows: Stage A is
+`(N+1) × M_tune` runs, Stage B is `3 × (S+1) × M_tune`, Stage C is `6 × M_holdout`.
+
+**Every stage runs the suite through the experiment.** The suite is the positional
+argument; the experiment carrying the arms is passed with `-e`. Passing the experiment file
+positionally instead would treat it as a task, which resolves to a skipped task and a green
+run of zero rows.
+
+### Stage A — triage (cheap)
+
+All candidates plus the incumbent, **one** invocation, `--split tune`:
+
+```bash
+coder-eval run <suite> -e <experiment> --split tune
+```
+
+Rank by the target label's F1 from each variant's `suite.json`; discard anything at or
+below the incumbent.
+
+This decides nothing — it only narrows. Replicate pooling is irrelevant here because
+nothing is being gated on.
+
+### Stage B — the gate (replicates)
+
+Incumbent plus survivors, `--split tune`, **invoked three separate times** — three
+`coder-eval run` commands, **not** `--repeats 3`:
+
+```bash
+coder-eval run <suite> -e <experiment> --split tune --run-dir <runs>/round<N>-gate-1
+coder-eval run <suite> -e <experiment> --split tune --run-dir <runs>/round<N>-gate-2
+coder-eval run <suite> -e <experiment> --split tune --run-dir <runs>/round<N>-gate-3
+```
+
+**This is the instruction most likely to be "simplified" into a bug.** Suite rollups are
+keyed on `(variant, suite)` only, so `--repeats 3` pools all three replicates into a single
+`suite.json` with one confusion matrix — the per-replicate F1 this gate reads would not
+exist in that file. Three invocations produce three run directories and three `suite.json`
+files. The cost is identical.
+
+Read `metrics["f1.yes"]` per invocation per arm. **Never recompute F1 from the confusion
+counts** — the criterion layer owns that arithmetic including its division-by-zero
+convention, and a re-derivation will disagree with the gate the run already applied.
+
+Promote only when all of these hold:
+
+- **`min(candidate F1) > max(incumbent F1)`** across the three invocations — the ranges do
+  not overlap. A difference smaller than the run-to-run spread cannot clear it.
+- **F1 improves, not just recall.** Widening recall while shedding precision is not an
+  improvement, it is a different trade.
+- **No sibling regression.** For every other `skill_triggered` criterion in the suite, its
+  **`recall.yes`** must not drop. A candidate that wins by annexing a sibling's requests has
+  moved the failure, not fixed it.
+
+  Read `recall.yes`, not `precision.yes`, and the reason is worth keeping: annexation makes
+  the sibling's criterion `expected=yes, observed=no` on that row — a false negative. That
+  lowers recall. Precision is `tp/(tp+fp)`, and annexation lowers `tp` while leaving `fp`
+  untouched, so on a suite where the sibling never misfires precision reads 1.0 however many
+  requests are stolen — right up until the last one, where `tp` and `fp` are both 0 and the
+  div-by-zero convention drops it to 0.0. Either way it is blind to the thing you are
+  watching for: gating on precision here would be gating on a constant that finally moves
+  only when the sibling has already lost everything.
+- **Print per-invocation F1 and confusion counts for every arm.** The verdict is never
+  reported without the numbers behind it.
+
+Why replicates rather than a fixed threshold: each arm's spread measures the noise floor
+for *this* suite at *this* size. A hardcoded "≥ 0.05 F1" would be far too lax on a six-row
+suite and needlessly strict on a forty-row one. **Do not introduce one.**
+
+**Be precise about what this bounds, because it is easy to claim more.** The replicate
+spread measures agent stochasticity over a *fixed* set of rows. It does not measure
+row-sampling variance, and it does not correct for the fact that the survivors were already
+chosen on these same tune rows in Stage A — so with S survivors each tested independently,
+some separation by luck is expected. Stage B bounds run noise. **The holdout is what bounds
+the fit**, and it is why Stage C is not optional. Report the gate as "separated beyond
+run-to-run noise on the tune rows", never as "proven better".
+
+### Stage C — confirm on holdout
+
+Only the best candidate that already passed Stage B, as a **two-variant** experiment
+(incumbent + that candidate) at `--split holdout --repeats 3`.
+
+Here `--repeats` is correct and required. The experiment reporter renders a
+`## Paired Comparison` block in `experiment.md` — mean difference, 95% confidence interval,
+Cohen's *d*, and a paired-*t* p-value — but it fires **only** for exactly two variants, and
+it averages replicates per row *before* pairing, which is precisely what pooling breaks in
+`suite.json` and exactly what is wanted here.
+
+Report that block verbatim alongside the holdout F1s, and state its limit honestly: it
+pairs per-row `weighted_score` — row-level correctness, an accuracy-flavoured quantity —
+**not** F1. F1 remains the promotion metric; the paired block corroborates the direction on
+the paired rows, it does not re-test the promotion metric.
+
+Require the F1 direction to reproduce on holdout. Do **not** require replicate separation
+there — a holdout is usually smaller and the separation usually weaker — and say that,
+rather than implying a stronger result than was obtained.
+
+## Step 10 — Ledger
+
+Append to `.optimize-skill/<skill>/history.json`: round, candidate slug, hypothesis,
+per-invocation tune F1, holdout F1, the paired-comparison line, sibling recalls, verdict,
+and the run ids. Append-only — never rewrite an earlier round. An existing directory means
+read `history.json` first and continue the numbering.
+
+## Step 11 — Present a diff; do not apply it
+
+Show the incumbent description against the promoted one and let the user apply it. This
+skill writes candidate snapshots, never the live `SKILL.md`.
+
+**Report a negative result plainly when nothing promotes.** That is the common outcome. The
+honest version — "three candidates, none separated from the incumbent beyond run-to-run
+noise, here are the numbers" — is worth more than a promotion that will not reproduce.
+
+## Step 12 — Stop rule
+
+Stop after two consecutive rounds that promote nothing. Continuing past that is fitting to
+the tune set, and the holdout will eventually stop catching it.
diff --git a/plugins/coder-eval/skills/task/SKILL.md b/plugins/coder-eval/skills/task/SKILL.md
index 037cfe31..7226cdea 100644
--- a/plugins/coder-eval/skills/task/SKILL.md
+++ b/plugins/coder-eval/skills/task/SKILL.md
@@ -1,5 +1,5 @@
 ---
-description: Turn a natural-language description into one or more coder-eval task YAML files — minimal prompts, weighted success criteria that check output content, validated with `coder-eval plan`. Use when the user wants to write, add, or generate an evaluation task.
+description: Turn a natural-language description into coder-eval task YAML — minimal prompts, weighted criteria that check output content, validated with `coder-eval plan`. Use when the user wants to write, add, or generate an evaluation task.
 allowed-tools: ["Read", "Glob", "Grep", "Write", "Bash"]
 ---
 
diff --git a/pyproject.toml b/pyproject.toml
index ae1bcdb9..3606d9b4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,6 +35,12 @@ dependencies = [
     "rich>=14.3.3",
     "python-dotenv>=1.2.2",
     "anthropic>=0.86.0",
+    # Imported directly by evaluation/judge_bedrock.py. Declared explicitly rather than
+    # leaned on transitively: it arrived via `anthropic` until 1.0.0 moved to `httpx2`,
+    # at which point a FRESH resolve (what `uv tool install` / `pip install coder-eval`
+    # does — a wheel carries no lockfile) dropped it, `criteria/llm_judge.py` failed to
+    # import, and `llm_judge` silently vanished from the registry. `uv.lock` hid it.
+    "httpx>=0.28.1",
     "claude-agent-sdk>=0.2.124",
     "anyio>=4.13.0",
     "radon>=6.0.1",
diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py
index 9b22929c..19346451 100644
--- a/src/coder_eval/cli/run_command.py
+++ b/src/coder_eval/cli/run_command.py
@@ -297,6 +297,17 @@ def run_command(
         ),
         min=1,
     ),
+    split: str | None = typer.Option(
+        None,
+        "--split",
+        help=(
+            "For dataset-backed tasks, keep only rows whose dataset.split_field value "
+            "(default field: split) matches this name — e.g. --split tune / --split holdout. "
+            "Applied BEFORE --sample / --sample-per-stratum, so a sampled split keeps a "
+            "predictable size. Tasks whose rows are all unlabelled are unaffected; a "
+            "labelled task with no row in this split is reported in skipped_tasks."
+        ),
+    ),
     repeats: int | None = typer.Option(
         None,
         "--repeats",
@@ -409,6 +420,7 @@ def run_command(
                 experiment_path=resolved_experiment,
                 max_rows=sample,
                 sample_per_stratum=sample_per_stratum,
+                split=split,
                 repeats=repeats,
                 verbose=verbose,
                 resume=resume,
@@ -434,6 +446,7 @@ async def _run_all_tasks(
     experiment_path: Path | None = None,
     max_rows: int | None = None,
     sample_per_stratum: int | None = None,
+    split: str | None = None,
     repeats: int | None = None,
     verbose: bool = False,
     resume: bool = False,
@@ -457,6 +470,9 @@ async def _run_all_tasks(
             from -D/--set and the bespoke flag aliases
         stream_mode: Optional stream mode ('full' or 'minimal') for real-time output
         experiment_path: Optional path to experiment YAML (default: experiments/default.yaml)
+        split: Optional dataset row filter (--split): keep only rows whose
+            dataset.split_field value matches. Applied before max_rows /
+            sample_per_stratum; tasks whose rows carry no split label are unaffected.
         junit_xml: Optional path to write a JUnit XML report to, after the run
             summary is persisted and before the failure exit-code gate.
     """
@@ -481,6 +497,7 @@ async def _run_all_tasks(
         overrides=overrides or {},
         max_rows=max_rows,
         sample_per_stratum=sample_per_stratum,
+        split=split,
         repeats=repeats,
         verbose=verbose,
         include_skipped=include_skipped,
diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py
index b3034727..d3831f6c 100644
--- a/src/coder_eval/models/tasks.py
+++ b/src/coder_eval/models/tasks.py
@@ -308,6 +308,18 @@ class Dataset(BaseModel):
         default="expected_skill",
         description="Row field whose value defines the stratum for 'sample_per_stratum' (default: 'expected_skill').",
     )
+    split_field: str = Field(
+        default="split",
+        description=(
+            "Row field naming the row's split (e.g. 'tune' / 'holdout' / 'holdback'). "
+            "CLI --split <name> keeps only rows whose value for this field matches, and "
+            "is applied BEFORE any sampling so a sampled split still has a predictable "
+            "size. A row is unlabelled when this field is absent, null, or ''; a task "
+            "whose rows are all unlabelled is unaffected by --split, while partial "
+            "labelling keeps only the matching rows and drops the unlabelled ones. "
+            "Splits are open strings; nothing here constrains the set of names."
+        ),
+    )
     sample_seed: int | None = Field(
         default=None,
         description=(
diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py
index 695b5594..f871bc88 100644
--- a/src/coder_eval/orchestration/config.py
+++ b/src/coder_eval/orchestration/config.py
@@ -86,6 +86,14 @@ class BatchRunConfig(BaseModel):
             "stratified dataset without editing the task YAML. Ignored when max_rows is set."
         ),
     )
+    split: str | None = Field(
+        default=None,
+        description=(
+            "CLI --split: keep only dataset rows whose dataset.split_field value matches. "
+            "Applied before max_rows / sample_per_stratum. Tasks whose rows carry no split "
+            "label are unaffected."
+        ),
+    )
 
     # Replicate count override
     repeats: int | None = Field(
diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py
index d103436b..afd7de41 100644
--- a/src/coder_eval/orchestration/experiment.py
+++ b/src/coder_eval/orchestration/experiment.py
@@ -628,6 +628,7 @@ def resolve_all_tasks(
                 task_file.parent,
                 max_rows=config.max_rows,
                 sample_per_stratum=config.sample_per_stratum,
+                split=config.split,
             )
         # Narrow set: real load failures only. We deliberately don't catch
         # AttributeError / TypeError / ImportError — those signal a regression
diff --git a/src/coder_eval/orchestration/task_loader.py b/src/coder_eval/orchestration/task_loader.py
index 9cf62c58..b255b93a 100644
--- a/src/coder_eval/orchestration/task_loader.py
+++ b/src/coder_eval/orchestration/task_loader.py
@@ -341,6 +341,26 @@ def _stratified_sample(
     return out
 
 
+def _reject_duplicate_row_ids(rows: list[dict[str, Any]], task: TaskDefinition) -> None:
+    """Raise if two rows share an id, considering the dataset as a whole.
+
+    Called before ``--split`` filtering and sampling narrow the row set: a duplicate id is
+    malformed data regardless of which rows a given invocation happens to select. Rows
+    missing the id field are skipped here — that is reported per-row during expansion,
+    where the row index is known.
+    """
+    assert task.dataset is not None
+    id_field = task.dataset.id_field
+    seen: set[str] = set()
+    for row in rows:
+        if id_field not in row:
+            continue
+        row_id = str(row[id_field])
+        if row_id in seen:
+            raise ValueError(f"Duplicate dataset row id for task '{task.task_id}': {row_id!r}")
+        seen.add(row_id)
+
+
 def _substitute_row_in_str(s: str, row: dict[str, Any]) -> str:
     """Replace ${row.<field>} occurrences in s with scalar values from row."""
 
@@ -374,6 +394,7 @@ def expand_dataset(
     task_file_dir: Path,
     max_rows: int | None = None,
     sample_per_stratum: int | None = None,
+    split: str | None = None,
 ) -> list[TaskDefinition]:
     """Fan out a task with ``dataset:`` into one TaskDefinition per row.
 
@@ -402,13 +423,24 @@ def expand_dataset(
             Lets a runner cap a stratified dataset without editing the task YAML
             (the nightly activation suite uses this). Ignored when ``max_rows``
             is set. When None, falls back to ``dataset.sample_per_stratum``.
+        split: Optional CLI row filter (``--split``) — keep only rows whose
+            ``dataset.split_field`` value equals this. Applied BEFORE either
+            sampler, so a sampled split still has a predictable size. A row is
+            unlabelled when the field is absent, ``None``, or ``""``. A task
+            whose rows are all unlabelled passes through unfiltered (``--split``
+            is global to the invocation, so an unlabelled task in a multi-task
+            run must not fail); partial labelling keeps the matching rows and
+            drops the unlabelled ones; a *labelled* task with no matching row
+            raises. Note the raise is caught by ``resolve_all_tasks`` into
+            ``skipped_tasks``, so at the run level a mistyped split name is a
+            skipped suite rather than an aborted run.
 
     Returns:
         Expanded list of TaskDefinitions. Length is 1 when dataset is None.
 
     Raises:
-        ValueError: Empty dataset, duplicate row ids, missing id_field, or
-            malformed row id.
+        ValueError: Empty dataset, duplicate row ids, missing id_field,
+            malformed row id, or a labelled dataset with no row in ``split``.
         FileNotFoundError: Dataset path does not exist.
     """
     if task.dataset is None:
@@ -418,6 +450,32 @@ def expand_dataset(
     if not rows:
         raise ValueError(f"Dataset for task '{task.task_id}' is empty")
 
+    # --split filters BEFORE either sampler below: sampling first would leave an
+    # unpredictable (possibly zero) number of rows per split, destroying the
+    # tune/holdout comparison the split exists to protect.
+    # Duplicate ids are a property of the DATASET, so check the whole row set BEFORE any
+    # filtering or sampling narrows it. Checking only what survives would let a duplicate
+    # sitting in an unselected split validate under every --split and surface only on a
+    # full run — and the split workflow always passes one.
+    _reject_duplicate_row_ids(rows, task)
+
+    if split is not None:
+        field = task.dataset.split_field
+        # Unlabelled means the key is absent, null, or "" — the same "no value here"
+        # convention _stratified_sample applies to a missing field, extended to the
+        # explicit null a half-labelled JSONL carries. Any other value, including a
+        # falsy 0, is a real label and compares via str() (also as _stratified_sample does).
+        labelled = [r for r in rows if r.get(field) not in (None, "")]
+        if labelled:
+            rows = [r for r in labelled if str(r[field]) == split]
+            if not rows:
+                raise ValueError(
+                    f"Dataset for task '{task.task_id}' has no rows in split {split!r} "
+                    + f"(split_field={field!r}); labelled splits present: "
+                    + f"{sorted({str(r[field]) for r in labelled})}"
+                )
+        # else: no row in this task carries a split label -> --split does not apply here.
+
     # Row selection precedence:
     #   1. CLI --sample (max_rows): flat uniform-random N over the whole dataset.
     #      Fixed seed => reproducible across runs, but (unlike a first-N slice)
@@ -438,7 +496,6 @@ def expand_dataset(
         rows = _stratified_sample(rows, ds.stratify_field, n_per_stratum, stratum_seed)
 
     id_field = task.dataset.id_field
-    seen_ids: set[str] = set()
     expanded: list[TaskDefinition] = []
 
     for i, row in enumerate(rows):
@@ -450,9 +507,8 @@ def expand_dataset(
                 f"Dataset row id {row_id!r} must match {_ROW_ID_PATTERN.pattern}"
                 + " (letters, digits, underscore, hyphen, dot)"
             )
-        if row_id in seen_ids:
-            raise ValueError(f"Duplicate dataset row id for task '{task.task_id}': {row_id!r}")
-        seen_ids.add(row_id)
+        # Uniqueness is already enforced across the whole dataset by
+        # _reject_duplicate_row_ids, before filtering narrowed `rows`.
 
         data = task.model_dump(exclude_unset=True)
         if isinstance(data.get("initial_prompt"), str):
diff --git a/tasks/skills/lint-tasks-activation-rows.jsonl b/tasks/skills/lint-tasks-activation-rows.jsonl
new file mode 100644
index 00000000..4bf73055
--- /dev/null
+++ b/tasks/skills/lint-tasks-activation-rows.jsonl
@@ -0,0 +1,28 @@
+{"id": "pos-1", "expected_skill": "lint-tasks", "split": "tune", "prompt": "Some of my eval definitions in this repo seem to pass no matter what the agent actually does. Go through them and tell me which ones are broken."}
+{"id": "pos-2", "expected_skill": "lint-tasks", "split": "tune", "prompt": "I inherited a folder of YAML evaluation definitions from a teammate who left and I don't trust them. What's wrong with them?"}
+{"id": "pos-3", "expected_skill": "lint-tasks", "split": "tune", "prompt": "Go over the evaluation suite in this repo and tell me where it is weak."}
+{"id": "pos-4", "expected_skill": "lint-tasks", "split": "tune", "prompt": "A couple of my test cases look almost the same as each other. Can you check the whole set for redundancy?"}
+{"id": "pos-5", "expected_skill": "lint-tasks", "split": "holdout", "prompt": "Are my evals any good?"}
+{"id": "pos-6", "expected_skill": "lint-tasks", "split": "holdout", "prompt": "Before I add more cases, sanity-check the ones I already have."}
+{"id": "neg-1", "expected_skill": "", "split": "tune", "prompt": "Audit my dependencies for known vulnerabilities."}
+{"id": "neg-2", "expected_skill": "", "split": "tune", "prompt": "Review this pull request for bugs before I merge it."}
+{"id": "neg-3", "expected_skill": "", "split": "tune", "prompt": "Check my GitHub Actions workflow for gaps in the test matrix."}
+{"id": "neg-4", "expected_skill": "", "split": "tune", "prompt": "Lint my Python code and fix the style issues."}
+{"id": "neg-5", "expected_skill": "", "split": "holdout", "prompt": "Summarize what this repository does, based on its README."}
+{"id": "neg-6", "expected_skill": "", "split": "holdout", "prompt": "Rename the variable `foo` to `bar` in utils.py."}
+{"id": "sib-1", "expected_skill": "task", "split": "tune", "prompt": "Add an evaluation that checks whether the agent writes a valid Dockerfile."}
+{"id": "sib-2", "expected_skill": "task", "split": "tune", "prompt": "I want a new test case covering the CSV export feature. Create one for me."}
+{"id": "sib-3", "expected_skill": "task", "split": "holdout", "prompt": "Write me an eval for the login flow."}
+{"id": "hard-1", "expected_skill": "lint-tasks", "split": "tune", "prompt": "My evals are a mess. Can you go through them and tell me what to fix?"}
+{"id": "hard-2", "expected_skill": "analyze", "split": "tune", "prompt": "Look at my evaluation results from last night's run and tell me what regressed."}
+{"id": "hard-3", "expected_skill": "", "split": "tune", "prompt": "Set up coder-eval for this repository."}
+{"id": "hard-4", "expected_skill": "", "split": "tune", "prompt": "Review my CI configuration for problems."}
+{"id": "hard-5", "expected_skill": "lint-tasks", "split": "holdout", "prompt": "I have a bunch of eval files and I am not sure they are testing the right things."}
+{"id": "hard-6", "expected_skill": "", "split": "holdout", "prompt": "Get evaluations going in this project from scratch."}
+{"id": "an-1", "expected_skill": "analyze", "split": "tune", "prompt": "Dig into the last run and tell me why so many of the tasks failed."}
+{"id": "an-2", "expected_skill": "analyze", "split": "holdout", "prompt": "My pass rate dropped from 80% to 60% between the last two runs. What happened?"}
+{"id": "an-3", "expected_skill": "analyze", "split": "holdout", "prompt": "I have a finished run directory. Tell me what it says about my tasks."}
+{"id": "an-4", "expected_skill": "analyze", "split": "tune", "prompt": "Compare last week's eval results with this week's and tell me what got worse."}
+{"id": "an-5", "expected_skill": "analyze", "split": "tune", "prompt": "Overnight my suite's scores fell off a cliff. Work out what changed from the run output."}
+{"id": "an-6", "expected_skill": "analyze", "split": "holdout", "prompt": "Which of my tasks got worse after I switched the model?"}
+{"id": "an-7", "expected_skill": "analyze", "split": "holdout", "prompt": "Something in the suite degraded this week. Find out what."}
diff --git a/tasks/skills/lint-tasks-activation.yaml b/tasks/skills/lint-tasks-activation.yaml
new file mode 100644
index 00000000..ce832053
--- /dev/null
+++ b/tasks/skills/lint-tasks-activation.yaml
@@ -0,0 +1,89 @@
+# Activation suite for the plugin's own `lint-tasks` skill, with `task` as the named
+# sibling. Written by following /coder-eval:check-skill, then split-labelled so
+# /coder-eval:optimize-skill can propose against `tune` and confirm on `holdout`.
+#
+# Reachability: `path` must be a PLUGIN ROOT — a directory holding a `skills/` subdirectory
+# (a `.claude-plugin/plugin.json` is optional; without one the namespace defaults to the
+# directory name). Pointing a local plugin at a bare directory of skill directories loads
+# NOTHING. Here that root is the plugin itself, which mounts all seven skills at once —
+# the point, since activation is a competition between descriptions:
+#
+#   export SKILL_SOURCE_PATH="$(pwd)/plugins/coder-eval"
+#
+# An unset or wrong path is only a warning and leaves every skill unreachable, so confirm
+# the first run's recall is non-zero before trusting any low score.
+#
+# `skill_name` below stays the BARE name even though the plugin namespaces these as
+# `coder-eval:lint-tasks` — the checker strips the namespace before comparing.
+task_id: "lint-tasks-activation"
+description: "Does the agent engage `lint-tasks` when it should, leave it alone when it shouldn't, and not steal `task`'s requests?"
+tags: [activation, skills]
+
+agent:
+  # The discriminator is required for `setting_sources` to be in schema — it is a
+  # claude-code field, not a field on the shared agent base.
+  type: "claude-code"
+  plugins:
+    - type: "local"
+      path: "$SKILL_SOURCE_PATH"
+  # The agent must not need the host project's CLAUDE.md or settings — this measures the
+  # skill listing, and injecting a 20 KB project guide into every call is both expensive
+  # and a confound.
+  setting_sources: []
+
+# Activation is decided in the first assistant turn: either the skill is engaged or it is
+# not. Without caps the agent spends turns exploring a sandbox that deliberately holds no
+# eval files, and a row that times out is EXCLUDED from the confusion matrix rather than
+# scored — shrinking the denominator the metrics are computed over (watch `completion_rate`
+# in the rollup). Cap tightly: it buys signal, not just cost.
+run_limits:
+  max_turns: 2
+  turn_timeout: 120
+  task_timeout: 300
+
+dataset:
+  paths:
+    - "lint-tasks-activation-rows.jsonl"
+  split_field: "split"
+
+initial_prompt: |
+  ${row.prompt}
+
+success_criteria:
+  # The skill under test. Positive rows expect engagement; distractor and sibling-owned
+  # rows expect none.
+  - type: "skill_triggered"
+    description: "lint-tasks activation for row ${row.id}"
+    skill_name: "lint-tasks"
+    expected_skill: "${row.expected_skill}"
+    suite_thresholds:
+      recall.yes: 0.7
+      precision.yes: 0.7
+
+  # The sibling. Stacking one criterion per skill yields a per-skill confusion matrix from
+  # the same traces, which turns "this misfired" into "this stole `task`'s requests".
+  # No suite_thresholds: this criterion is the sibling-regression read-out, not a gate.
+  - type: "skill_triggered"
+    description: "task (sibling) activation for row ${row.id}"
+    skill_name: "task"
+    expected_skill: "${row.expected_skill}"
+
+  - type: "skill_triggered"
+    description: "analyze (sibling) activation for row ${row.id}"
+    skill_name: "analyze"
+    expected_skill: "${row.expected_skill}"
+
+  # NOTE on the two setup rows (`hard-3`, `hard-6`) and the absent `init` criterion.
+  # Two independent reasons not to write one:
+  #
+  # 1. coder-eval's `init` sets `disable-model-invocation: true`, so the model will not
+  #    reach for it on its own. `expected_skill: "init"` would assert something the run
+  #    cannot satisfy — a criterion that can never pass, exactly what `lint-tasks` flags.
+  # 2. `skill_name` is matched on the BARE name (the checker strips any `plugin:` prefix),
+  #    and Claude Code ships its own unscoped `init` skill. A bare-name criterion cannot
+  #    tell the two apart, so it would silently score a different skill's activation.
+  #    Observed live: these rows engage that other `init`.
+  #
+  # Labelled `""` instead, which asks the question that has an answer: does a
+  # model-invokable sibling wrongly claim setup requests? It scores against every criterion
+  # above, and `task` does misfire here intermittently.
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index be312a91..e7d095a1 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -1183,7 +1183,9 @@ def test_drift_is_detected(self, tmp_path: Path):
 # `analyze` reads a finished run directory, `ci` only emits a workflow, and `lint-tasks`
 # has no `Bash` at all (the `coder-eval plan` in its report is a suggestion to the user,
 # not a command it runs) — so none of those three needs the CLI.
-SKILLS_REQUIRING_THE_CLI = {"init", "check-skill", "task"}
+# `optimize-skill` drives the CLI harder than any other skill here — a baseline, a triage
+# stage, three gate invocations and a holdout confirmation, all `coder-eval run`.
+SKILLS_REQUIRING_THE_CLI = {"init", "check-skill", "task", "optimize-skill"}
 
 # The skills that read the shared task-quality rubric. A rubric no skill reads is
 # dead weight; a reader that stops reading it has silently forked the rubric.
@@ -1209,6 +1211,9 @@ def test_drift_is_detected(self, tmp_path: Path):
     "lint-tasks": True,
     "check-skill": True,
     "task": True,
+    # Reads both trees: globs the task tree for an activation suite, then reads that
+    # run's suite.json rollups.
+    "optimize-skill": True,
 }
 
 # Ways of naming one repository's layout as the DEFAULT. The bare paths `runs/latest`
@@ -1231,6 +1236,9 @@ def test_drift_is_detected(self, tmp_path: Path):
     "lint-tasks": False,
     "check-skill": False,
     "task": False,
+    # Multi-round and expensive — a baseline plus three A/B stages of full agent runs.
+    # Never something to start because a message mentioned a skill's wording.
+    "optimize-skill": True,
 }
 
 # The surfaces that must name every shipped skill, so a new one cannot ship
@@ -1334,6 +1342,66 @@ def test_activation_template_thresholds_use_real_metric_keys(self):
                 f"emit (available: {sorted(aggregate.metrics)})"
             )
 
+    def test_reachability_guidance_names_the_plugin_root_layout(self):
+        # A local plugin path must be a PLUGIN ROOT — a directory holding `skills/`, so the
+        # skill sits at `<path>/skills/<name>/SKILL.md`. A manifest is optional (the
+        # namespace then defaults to the directory name). Pointing at a bare directory of
+        # skill directories loads NOTHING, silently, and every positive row scores 0 —
+        # indistinguishable from a skill that never triggers.
+        #
+        # The shipped guidance said the opposite (".claude/skills" for
+        # ".claude/skills/my-skill/SKILL.md"), so every generated suite reported recall 0.0.
+        # Caught only by running one for real. This pins the corrected form in all three
+        # places that state it.
+        surfaces = {
+            "activation template": self.TEMPLATES / "activation.yaml",
+            "check-skill": PLUGIN_ROOT / "skills" / "check-skill" / "SKILL.md",
+            "optimize-skill": PLUGIN_ROOT / "skills" / "optimize-skill" / "SKILL.md",
+            "ci": PLUGIN_ROOT / "skills" / "ci" / "SKILL.md",
+            "docs/PLUGIN.md": self.REPO_ROOT / "docs" / "PLUGIN.md",
+            "tutorial 07": self.REPO_ROOT / "docs" / "tutorials" / "07-plugin-in-claude-code.md",
+            "tutorial 08": self.REPO_ROOT / "docs" / "tutorials" / "08-optimizing-a-skill.md",
+        }
+        for name, path in surfaces.items():
+            text = path.read_text(encoding="utf-8")
+            assert "plugin root" in text.lower(), (
+                f"{name} no longer says the plugin `path` must be a PLUGIN ROOT — a bare "
+                "directory of skill directories loads nothing and scores recall 0.0"
+            )
+            # The required layout, spelled out. `skills/` alone would NOT do: the pre-fix
+            # (wrong) text contained it too, via `.claude/skills/my-skill/SKILL.md`, so an
+            # assertion on the bare token passes on exactly the guidance it must exclude.
+            assert "skills/<name>/SKILL.md" in text or "skills/<skill-name>/SKILL.md" in text, (
+                f"{name} no longer shows the required `<path>/skills/<name>/SKILL.md` layout"
+            )
+
+        # No surface may hand out a SKILL_SOURCE_PATH ending in /skills, nor revive the
+        # "directory containing the skill's directory" rule. Both are the pre-fix form, and
+        # the ci skill's copy of it shipped straight into users' CI workflows.
+        for name, path in surfaces.items():
+            text = path.read_text(encoding="utf-8")
+            for line_no, line in enumerate(text.splitlines(), start=1):
+                if "SKILL_SOURCE_PATH=" in line or "SKILL_SOURCE_PATH=" in line.replace('"', ""):
+                    value = line.split("SKILL_SOURCE_PATH=", 1)[1].strip().strip('"').rstrip("`")
+                    assert not value.endswith("/skills"), (
+                        f"{name}:{line_no} assigns SKILL_SOURCE_PATH a path ending in /skills "
+                        f"({value!r}) — that is one level too deep and loads NOTHING; the "
+                        "plugin root is its parent"
+                    )
+            assert "directory *containing* the skill" not in text, (
+                f"{name} revived the pre-fix rule 'the directory containing the skill's "
+                "directory' — a bare directory of skill directories loads nothing"
+            )
+
+        # The user-facing surfaces must name the trap by example, since the wrong form is
+        # the intuitive one.
+        for name in ("activation template", "check-skill", "ci", "docs/PLUGIN.md", "tutorial 07", "tutorial 08"):
+            text = surfaces[name].read_text(encoding="utf-8")
+            assert "`.claude/skills`" in text and ".claude" in text, (
+                f"{name} no longer contrasts `.claude` against `.claude/skills` — the wrong "
+                "path is the intuitive one, so naming it is the whole point"
+            )
+
     def test_activation_template_makes_the_skill_reachable(self):
         # The suite runs in a fresh sandbox holding none of the user's files, so without a
         # plugin source the agent is never OFFERED the skill: every positive row scores 0,
@@ -1368,6 +1436,35 @@ def test_activation_rows_have_both_polarities(self):
         assert any(label for label in labels), "no positive rows — recall would be undefined"
         assert "" in labels, "no distractor rows — precision is 1.0 by definition and meaningless"
 
+    def test_activation_rows_split_both_polarities_both_sides(self):
+        # `optimize-skill` and the template both require both polarities on BOTH sides of
+        # the split: a holdout of only positives measures recall and calls it a result, and
+        # a tune half with no distractors cannot see a candidate over-claiming. The shipped
+        # template is the worked example everyone copies, so the balance it demonstrates has
+        # to hold — an edit that moved one row could break it silently.
+        import json
+
+        rows = [
+            json.loads(line)
+            for line in (self.TEMPLATES / "activation-rows.jsonl").read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        ]
+        assert all(r.get("split") for r in rows), (
+            "every template row must carry a `split` — a PARTLY labelled dataset is the one "
+            "bad state: --split keeps the matching rows and drops the unlabelled ones, "
+            "shrinking the suite the metrics are computed over"
+        )
+        by_split: dict[str, set[bool]] = {}
+        for r in rows:
+            by_split.setdefault(str(r["split"]), set()).add(bool(r["expected_skill"]))
+        assert len(by_split) >= 2, f"template rows collapsed to a single split: {sorted(by_split)}"
+        for split, polarities in sorted(by_split.items()):
+            assert polarities == {True, False}, (
+                f"split {split!r} carries only {'positive' if True in polarities else 'distractor'} "
+                "rows — both splits need positives AND distractors, or one half of the "
+                "tune/holdout comparison measures nothing"
+            )
+
     @pytest.mark.parametrize("skill", PLUGIN_SKILLS, ids=[p.parent.name for p in PLUGIN_SKILLS])
     def test_skill_md_frontmatter_is_valid(self, skill: Path):
         # `claude plugin validate --strict` does NOT check skill frontmatter, so this
@@ -1537,6 +1634,55 @@ def test_lint_tasks_does_not_flag_the_shipped_activation_template(self):
             "condition no longer holds, so lint-tasks would flag the suite check-skill writes"
         )
 
+    def test_optimize_skill_keeps_its_load_bearing_instructions(self):
+        # optimize-skill's correctness lives entirely in its prose: it drives paid runs and
+        # every one of these instructions is something a well-meaning edit would "simplify"
+        # away, leaving a skill that still reads plausibly and measures nothing. Same
+        # deletion-sensor shape as the lint-tasks read-only guard above.
+        text = " ".join((PLUGIN_ROOT / "skills" / "optimize-skill" / "SKILL.md").read_text(encoding="utf-8").split())
+
+        # The single most important invariant. Suite rollups pool replicates, so --repeats
+        # writes ONE pooled suite.json and the per-replicate F1 the Stage B gate reads would
+        # not exist. Collapsing three invocations into --repeats 3 is the likeliest edit,
+        # and it fails silently — the gate would compare a number against itself.
+        assert "not** `--repeats 3`" in text or "**not** `--repeats`" in text, (
+            "optimize-skill no longer says Stage B must be THREE SEPARATE INVOCATIONS rather "
+            "than --repeats 3 — suite rollups pool replicates, so --repeats yields one pooled "
+            "suite.json with no per-replicate F1 for the gate to read"
+        )
+        assert "keyed on `(variant, suite)`" in text, (
+            "optimize-skill states the not---repeats rule but no longer says WHY (the rollup "
+            "grouping key) — a rule with no reason attached is the first thing an editor drops"
+        )
+
+        for token, why in (
+            ("--split tune", "the tune split drives proposals and the gate"),
+            ("--split holdout", "the holdout split is what makes a promotion more than a fit"),
+            ("--split holdout --repeats 3", "Stage C's paired comparison needs replicates averaged per row"),
+            ("stop_early", "an armed suite can pass-stop before a sibling misfire is observable"),
+            ("agent.plugins", "reachability wiring — the mechanism, not an invented one"),
+            ("recall 0.0", "the silent failure mode of a wrong plugin path"),
+            ("disable-model-invocation", "the hard stop on a skill whose description never triggers"),
+            ('metrics["f1.yes"]', "F1 is read from the rollup, never recomputed"),
+            ("process working directory", "plugin paths resolve against the process cwd, not the task file's dir"),
+            # Annexation shows up as a sibling FALSE NEGATIVE, so it moves recall, not
+            # precision — on a suite where the sibling never misfires, precision.yes is
+            # pinned at 1.0 and a precision gate would be gating on a constant.
+            ("**`recall.yes`**", "the sibling-regression gate must read recall, not precision"),
+            # Step 7's snapshot must carry the siblings: a variant's plugins block REPLACES
+            # the task's, so the round-slug dir is the arm's only skill source. Snapshot one
+            # skill and every sibling criterion silently observes `no` in every arm.
+            ("copied unchanged", "each arm's snapshot must include the sibling skills, not just the target"),
+        ):
+            assert token in text, f"optimize-skill lost {token!r} — {why}"
+
+        # Its sibling's real name. `/coder-eval:skill-check` is a dangling command, and three
+        # steps plus several edge cases hand control back to check-skill.
+        assert "/coder-eval:check-skill" in text, "optimize-skill lost its pointer to /coder-eval:check-skill"
+        assert "skill-check" not in text.replace("check-skill", ""), (
+            "optimize-skill names `skill-check` — that command does not exist; the sibling is `check-skill`"
+        )
+
     def test_skill_listing_budget_is_bounded(self):
         # See SKILL_LISTING_BUDGET_CHARS for why a plugin should self-limit here.
         # Filesystem-derived: no hardcoded skill names, no per-skill numbers.
@@ -1854,13 +2000,17 @@ def test_cli_setup_conditions_the_upgrade_suggestion_on_a_pin(self):
 
     def test_repo_layout_is_bundled_and_read_by_its_readers(self):
         # Sibling of the cli-setup guard above, for the third shared reference. "Where does
-        # this repository keep its tasks and runs?" is a question five skills ask and a sixth
-        # writes into a CI workflow; inlining the answer six times is how the hardcoded
-        # `tasks/` / `runs/latest` assumptions got there in the first place. Both halves are
-        # asserted: the reference ships, and every skill that needs it points at it.
+        # this repository keep its tasks and runs?" is a question every shipped skill asks —
+        # most to read the trees, `ci` to write the resolved glob into a workflow; inlining
+        # the answer once per skill is how the hardcoded `tasks/` / `runs/latest` assumptions
+        # got there in the first place. Both halves are asserted: the reference ships, and
+        # every skill that needs it points at it. (Counts stay out of this comment on
+        # purpose — the mapping below is the source of truth and a seventh skill already
+        # made two hardcoded numbers here stale.)
         layout = PLUGIN_ROOT / "reference" / "repo-layout.md"
         assert layout.exists() and layout.read_text(encoding="utf-8").strip(), (
-            f"{layout} must exist and be non-empty — six skills read it at runtime to find the repository's eval tree"
+            f"{layout} must exist and be non-empty — the shipped skills read it at runtime "
+            "to find the repository's eval tree"
         )
         # Both directions, so neither the mapping nor the tree can drift silently: a
         # declared skill must ship, and a shipped skill must declare its stance.
diff --git a/tests/test_dataset_expansion.py b/tests/test_dataset_expansion.py
index 08693cfb..cbcff832 100644
--- a/tests/test_dataset_expansion.py
+++ b/tests/test_dataset_expansion.py
@@ -284,6 +284,160 @@ def test_dataset_seed_wins_over_cli_arg(self, tmp_path: Path) -> None:
         assert ids_flag == ids_yaml
 
 
+class TestExpandDatasetSplit:
+    """CLI --split: keep only rows whose dataset.split_field value matches.
+
+    The filter runs BEFORE either sampler, so a sampled split still has a
+    predictable size — sampling first would leave an unpredictable (possibly
+    zero) number of rows per split and silently destroy the tune/holdout
+    comparison the split exists to protect.
+    """
+
+    @staticmethod
+    def _split_rows() -> list[dict[str, Any]]:
+        return [
+            {"id": "t1", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "t2", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "t3", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "h1", "prompt": "p", "expected": "e", "split": "holdout"},
+            {"id": "h2", "prompt": "p", "expected": "e", "split": "holdout"},
+        ]
+
+    def test_keeps_only_matching_rows(self, tmp_path: Path) -> None:
+        task = _make_task_with_dataset(rows=self._split_rows())
+        expanded = expand_dataset(task, tmp_path, split="tune")
+        assert [t.row_id for t in expanded] == ["t1", "t2", "t3"]
+        # task_id / row_id rewriting is unchanged by the filter.
+        assert [t.task_id for t in expanded] == ["suite/t1", "suite/t2", "suite/t3"]
+        assert all(t.suite_id == "suite" for t in expanded)
+
+    def test_unlabelled_dataset_passes_through(self, tmp_path: Path) -> None:
+        # --split is global to the invocation. A run containing several
+        # dataset-backed tasks would otherwise fail every task that does not use
+        # splits, so a task with NO labelled row at all is left whole.
+        rows = [{"id": f"r{i}", "prompt": "p", "expected": "e"} for i in range(4)]
+        expanded = expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="tune")
+        assert [t.row_id for t in expanded] == ["r0", "r1", "r2", "r3"]
+
+    def test_partially_labelled_excludes_unlabelled_rows(self, tmp_path: Path) -> None:
+        # Safe direction: an unlabelled row never leaks into a named split.
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "b", "prompt": "p", "expected": "e"},
+            {"id": "c", "prompt": "p", "expected": "e", "split": ""},
+        ]
+        expanded = expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="tune")
+        assert [t.row_id for t in expanded] == ["a"]
+
+    def test_labelled_but_unmatched_raises_listing_available_splits(self, tmp_path: Path) -> None:
+        task = _make_task_with_dataset(rows=self._split_rows())
+        with pytest.raises(ValueError, match="no rows in split 'Tune'") as exc:
+            expand_dataset(task, tmp_path, split="Tune")
+        # Exact match, no case normalization — the message names what does exist.
+        assert "'holdout', 'tune'" in str(exc.value)
+
+    def test_explicit_null_split_counts_as_unlabelled(self, tmp_path: Path) -> None:
+        # `"split": null` is the natural JSONL shape for "not assigned yet", and it
+        # matches the null -> "" convention row substitution already uses. It must
+        # NOT read as the label "None" (which `str(row.get(field, ""))` would make
+        # truthy), or a half-migrated dataset fails instead of passing through and
+        # the error advertises a phantom split.
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": None},
+            {"id": "b", "prompt": "p", "expected": "e", "split": None},
+        ]
+        expanded = expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="tune")
+        assert [t.row_id for t in expanded] == ["a", "b"]
+
+    def test_null_split_rows_are_excluded_from_a_named_split(self, tmp_path: Path) -> None:
+        # The partial-labelling half of the same rule: a null-split row is unlabelled,
+        # so it is dropped from a named split rather than joining a "None" split.
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "b", "prompt": "p", "expected": "e", "split": None},
+        ]
+        expanded = expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="tune")
+        assert [t.row_id for t in expanded] == ["a"]
+
+    def test_zero_is_a_real_split_label(self, tmp_path: Path) -> None:
+        # Guards the null fix against an `or ""` implementation, which would make the
+        # falsy-but-present value 0 unlabelled. Only None/"" mean "no label".
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": 0},
+            {"id": "b", "prompt": "p", "expected": "e", "split": 1},
+        ]
+        assert [t.row_id for t in expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="0")] == ["a"]
+
+    def test_non_string_split_values_compare_by_str(self, tmp_path: Path) -> None:
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": 1},
+            {"id": "b", "prompt": "p", "expected": "e", "split": 2},
+        ]
+        expanded = expand_dataset(_make_task_with_dataset(rows=rows), tmp_path, split="1")
+        assert [t.row_id for t in expanded] == ["a"]
+
+    def test_custom_split_field(self, tmp_path: Path) -> None:
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "fold": "train"},
+            {"id": "b", "prompt": "p", "expected": "e", "fold": "test"},
+        ]
+        task = _make_task_with_dataset(rows=rows, split_field="fold")
+        assert [t.row_id for t in expand_dataset(task, tmp_path, split="test")] == ["b"]
+
+    def test_filter_runs_before_max_rows(self, tmp_path: Path) -> None:
+        # Ordering guard: every sampled row must still carry the requested split.
+        rows = self._split_rows()
+        task = _make_task_with_dataset(rows=rows)
+        expanded = expand_dataset(task, tmp_path, max_rows=2, split="tune")
+        assert len(expanded) == 2
+        assert {t.row_id for t in expanded} <= {"t1", "t2", "t3"}
+
+    def test_filter_runs_before_sample_per_stratum(self, tmp_path: Path) -> None:
+        # Same assertion on the stratified path: strata are computed WITHIN the
+        # filtered set, so a holdout row can never be drawn under --split tune.
+        rows = [
+            {"id": "t1", "prompt": "p", "expected": "e", "split": "tune", "expected_skill": "a"},
+            {"id": "t2", "prompt": "p", "expected": "e", "split": "tune", "expected_skill": "a"},
+            {"id": "t3", "prompt": "p", "expected": "e", "split": "tune", "expected_skill": "b"},
+            {"id": "h1", "prompt": "p", "expected": "e", "split": "holdout", "expected_skill": "a"},
+            {"id": "h2", "prompt": "p", "expected": "e", "split": "holdout", "expected_skill": "b"},
+        ]
+        task = _make_task_with_dataset(rows=rows, sample_per_stratum=1, sample_seed=0)
+        expanded = expand_dataset(task, tmp_path, split="tune")
+        assert len(expanded) == 2  # one per stratum, within tune only
+        assert {t.row_id for t in expanded} <= {"t1", "t2", "t3"}
+
+    def test_split_none_is_byte_identical_to_today(self, tmp_path: Path) -> None:
+        # No-regression guard: with no --split, a dataset carrying mixed split
+        # values expands exactly as it did before the filter existed.
+        task = _make_task_with_dataset(rows=self._split_rows())
+        assert [t.row_id for t in expand_dataset(task, tmp_path)] == ["t1", "t2", "t3", "h1", "h2"]
+        assert [t.row_id for t in expand_dataset(task, tmp_path, split=None)] == ["t1", "t2", "t3", "h1", "h2"]
+
+    def test_task_without_dataset_unaffected(self, tmp_path: Path) -> None:
+        task = TaskDefinition(**_base_task_dict())
+        assert expand_dataset(task, tmp_path, split="tune") == [task]
+
+    def test_duplicate_ids_across_splits_are_caught_under_a_filter(self, tmp_path: Path) -> None:
+        # Id uniqueness is a property of the DATASET, not of whichever split you selected.
+        # If the filter ran before the duplicate check, a copy-pasted row landing in the
+        # other split would validate under every --split and only blow up on a full run —
+        # and the documented optimize-skill workflow always passes --split, so it would
+        # never be seen at all.
+        rows = [
+            {"id": "a", "prompt": "p", "expected": "e", "split": "tune"},
+            {"id": "a", "prompt": "p", "expected": "e", "split": "holdout"},
+        ]
+        task = _make_task_with_dataset(rows=rows)
+        with pytest.raises(ValueError, match="Duplicate dataset row id"):
+            expand_dataset(task, tmp_path, split="tune")
+
+    def test_split_field_defaults_to_split(self) -> None:
+        task = _make_task_with_dataset(rows=[{"id": "a"}])
+        assert task.dataset is not None
+        assert task.dataset.split_field == "split"
+
+
 class TestExpandDatasetJsonl:
     def test_loads_jsonl(self, tmp_path: Path) -> None:
         ds_path = tmp_path / "rows.jsonl"
@@ -514,6 +668,98 @@ def test_max_rows_applies(self, tmp_path: Path) -> None:
         # --sample is now a random subset (count == max_rows), not a first-N slice.
         assert resolved[0].task.task_id in {"suite/row-a", "suite/row-b"}
 
+    def test_split_applies(self, tmp_path: Path) -> None:
+        # BatchRunConfig.split threads CLI -> config -> expand_dataset with no
+        # merge-layer participation (dataset expansion precedes variant resolution).
+        data = {
+            "task_id": "suite",
+            "description": "Test",
+            "initial_prompt": "Prompt: ${row.prompt}",
+            "sandbox": {"driver": "tempdir"},
+            "success_criteria": [{"type": "file_exists", "path": "out.txt", "description": "File"}],
+            "dataset": {
+                "rows": [
+                    {"id": "row-a", "prompt": "a", "split": "tune"},
+                    {"id": "row-b", "prompt": "b", "split": "holdout"},
+                ]
+            },
+        }
+        task_file = tmp_path / "suite.yaml"
+        task_file.write_text(yaml.safe_dump(data))
+        default_exp, experiment = self._make_experiment(["v1"])
+        config = BatchRunConfig(run_dir=tmp_path / "runs", split="tune")
+
+        resolved, _ = resolve_all_tasks(
+            task_files=[task_file],
+            experiment=experiment,
+            default_experiment=default_exp,
+            config=config,
+        )
+        assert [rt.task.task_id for rt in resolved] == ["suite/row-a"]
+
+    def _write_split_suite(self, tmp_path: Path, task_id: str, splits: list[str | None]) -> Path:
+        rows: list[dict[str, Any]] = []
+        for i, split in enumerate(splits):
+            row: dict[str, Any] = {"id": f"row-{i}", "prompt": str(i)}
+            if split is not None:
+                row["split"] = split
+            rows.append(row)
+        data = {
+            "task_id": task_id,
+            "description": "Test",
+            "initial_prompt": "Prompt: ${row.prompt}",
+            "sandbox": {"driver": "tempdir"},
+            "success_criteria": [{"type": "file_exists", "path": "out.txt", "description": "File"}],
+            "dataset": {"rows": rows},
+        }
+        p = tmp_path / f"{task_id}.yaml"
+        p.write_text(yaml.safe_dump(data))
+        return p
+
+    def test_split_leaves_an_unlabelled_sibling_suite_whole(self, tmp_path: Path) -> None:
+        # The actual reason unlabelled datasets pass through: --split is global to the
+        # invocation, so a run mixing a split-labelled suite with an unlabelled one must
+        # filter the first and leave the second entirely alone. Asserting this at the
+        # resolver (not on one isolated expand_dataset call) is what proves the claim.
+        labelled = self._write_split_suite(tmp_path, "labelled", ["tune", "holdout"])
+        unlabelled = self._write_split_suite(tmp_path, "plain", [None, None, None])
+        default_exp, experiment = self._make_experiment(["v1"])
+
+        resolved, skipped = resolve_all_tasks(
+            task_files=[labelled, unlabelled],
+            experiment=experiment,
+            default_experiment=default_exp,
+            config=BatchRunConfig(run_dir=tmp_path / "runs", split="tune"),
+        )
+        assert not skipped
+        assert sorted(rt.task.task_id for rt in resolved) == [
+            "labelled/row-0",  # the one tune row
+            "plain/row-0",  # unlabelled suite survives whole
+            "plain/row-1",
+            "plain/row-2",
+        ]
+
+    def test_unmatched_split_demotes_the_suite_to_a_skipped_task(self, tmp_path: Path) -> None:
+        # A labelled suite with no row in the requested split raises out of
+        # expand_dataset, but resolve_all_tasks catches ValueError and records a
+        # SkippedTask rather than aborting — so the run reports it in run.json's
+        # skipped_tasks instead of failing. Pinned because the reason string is the
+        # ONLY place a mistyped --split selector surfaces: nothing else distinguishes
+        # it from an empty run.
+        task_file = self._write_split_suite(tmp_path, "labelled", ["tune", "holdout"])
+        default_exp, experiment = self._make_experiment(["v1"])
+
+        resolved, skipped = resolve_all_tasks(
+            task_files=[task_file],
+            experiment=experiment,
+            default_experiment=default_exp,
+            config=BatchRunConfig(run_dir=tmp_path / "runs", split="holdou"),
+        )
+        assert resolved == []
+        assert len(skipped) == 1
+        assert "no rows in split 'holdou'" in skipped[0].reason
+        assert "'holdout', 'tune'" in skipped[0].reason
+
     def test_non_dataset_task_unaffected(self, tmp_path: Path) -> None:
         task_file = self._write_task_yaml(tmp_path, "plain", with_dataset=False)
         default_exp, experiment = self._make_experiment(["v1"])
diff --git a/tests/test_declared_dependencies.py b/tests/test_declared_dependencies.py
new file mode 100644
index 00000000..aea1b00c
--- /dev/null
+++ b/tests/test_declared_dependencies.py
@@ -0,0 +1,128 @@
+"""Every third-party module imported under ``src/`` must be a DECLARED dependency.
+
+Why this exists, and why the rest of the suite could not catch it: `evaluation/judge_bedrock.py`
+imported `httpx` while `pyproject.toml` never declared it. It worked for months because
+`anthropic` pulled `httpx` transitively — so `uv.lock` contained it, and every `uv sync --frozen`
+job (the whole locked half of CI, plus `make verify`) installed it and passed.
+
+Then `anthropic` 1.0.0 moved to `httpx2`. A *fresh* resolve — which is what
+`uv tool install` and `pip install coder-eval` do, because a published wheel carries no
+lockfile — stopped installing `httpx`. `criteria/llm_judge.py` imports `judge_bedrock`, so it
+raised on import; `criteria/__init__.py` catches that and logs it, and `llm_judge` simply
+vanished from the registry. The failure surfaced only later, as a task dying on
+"Missing criterion checkers", in the one CI job that installs like a real user.
+
+So the invariant is about the DECLARATION, not the installed set. Checking "can I import it"
+passes in every locked environment and is exactly the blind spot that shipped the bug; checking
+"is it in `pyproject.toml`" fails fast in the cheap locked job with no network.
+
+What counts as required is DERIVED, not listed: an import wrapped in a `try/except ImportError`
+declares its own optionality in code (`google.antigravity`, `openai`), so it is exempt. An
+UNGUARDED import is a hard requirement and must be declared. That keeps the exemption set from
+drifting out of date — adding a guarded soft dependency needs no edit here, and deleting a guard
+turns the import into one this test checks.
+
+Extras count as declared: a lazily imported optional dependency (`uipath`) is legitimate.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+import sys
+import tomllib
+from importlib.metadata import packages_distributions
+from pathlib import Path
+
+
+SRC = Path(__file__).parent.parent / "src"
+PYPROJECT = Path(__file__).parent.parent / "pyproject.toml"
+
+# Import names that are part of this project, not third-party.
+FIRST_PARTY = {"coder_eval", "tests"}
+
+
+def _normalize(name: str) -> str:
+    """PEP 503 name normalization, so `python-dotenv` and `python_dotenv` compare equal."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def _declared_distributions() -> set[str]:
+    data = tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))
+    project = data["project"]
+    specs = list(project.get("dependencies", []))
+    for extra in project.get("optional-dependencies", {}).values():
+        specs.extend(extra)
+    # Strip everything after the first version/extras/marker delimiter.
+    return {_normalize(re.split(r"[<>=!~\[;\s]", spec, maxsplit=1)[0]) for spec in specs}
+
+
+def _guards_importerror(handler: ast.ExceptHandler) -> bool:
+    """Does this `except` clause catch a missing module?"""
+    caught = handler.type
+    if caught is None:  # bare `except:` catches it too
+        return True
+    names = caught.elts if isinstance(caught, ast.Tuple) else [caught]
+    return any(isinstance(n, ast.Name) and n.id in {"ImportError", "ModuleNotFoundError"} for n in names)
+
+
+def _guarded_import_nodes(tree: ast.AST) -> set[int]:
+    """AST node ids of imports sitting inside a `try` whose handler catches a missing module."""
+    guarded: set[int] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Try) and any(_guards_importerror(h) for h in node.handlers):
+            for stmt in node.body:
+                for inner in ast.walk(stmt):
+                    if isinstance(inner, ast.Import | ast.ImportFrom):
+                        guarded.add(id(inner))
+    return guarded
+
+
+def _imported_top_level_modules() -> dict[str, list[str]]:
+    """Map each REQUIRED (unguarded) imported top-level module name -> the files importing it."""
+    found: dict[str, list[str]] = {}
+    for path in sorted(SRC.rglob("*.py")):
+        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+        guarded = _guarded_import_nodes(tree)
+        for node in ast.walk(tree):
+            if id(node) in guarded:
+                continue
+            if isinstance(node, ast.Import):
+                names = [alias.name for alias in node.names]
+            elif isinstance(node, ast.ImportFrom):
+                # level > 0 is a relative (first-party) import.
+                names = [node.module] if node.level == 0 and node.module else []
+            else:
+                continue
+            for name in names:
+                found.setdefault(name.split(".")[0], []).append(str(path.relative_to(SRC)))
+    return found
+
+
+def test_every_third_party_import_is_a_declared_dependency() -> None:
+    declared = _declared_distributions()
+    mapping = packages_distributions()
+    undeclared: list[str] = []
+    unresolvable: list[str] = []
+
+    for module, importers in sorted(_imported_top_level_modules().items()):
+        if module in FIRST_PARTY or module in sys.stdlib_module_names:
+            continue
+        dists = mapping.get(module)
+        if not dists:
+            unresolvable.append(f"{module} (imported by {importers[0]})")
+            continue
+        if not any(_normalize(d) in declared for d in dists):
+            undeclared.append(f"{module} -> {sorted(dists)} (imported by {', '.join(sorted(set(importers)))})")
+
+    assert not undeclared, (
+        "These modules are imported under src/ but their distribution is not declared in "
+        f"pyproject.toml: {undeclared}. They are reaching the environment TRANSITIVELY, so "
+        "`uv sync --frozen` installs them and this suite passes — until a direct dependency "
+        "drops or renames its own dependency, at which point a fresh `pip install coder-eval` "
+        "stops installing them and the import fails for real users. Declare each one."
+    )
+    assert not unresolvable, (
+        f"Could not map these imported modules to any installed distribution: {unresolvable}. "
+        "Either the module is missing from this environment (install it) or the import is dead."
+    )
diff --git a/uv.lock b/uv.lock
index bd9fd214..9e1c353b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -457,6 +457,7 @@ dependencies = [
     { name = "azure-monitor-opentelemetry-exporter" },
     { name = "claude-agent-sdk" },
     { name = "click" },
+    { name = "httpx" },
     { name = "jmespath" },
     { name = "jsonschema" },
     { name = "opentelemetry-sdk" },
@@ -506,6 +507,7 @@ requires-dist = [
     { name = "click", specifier = ">=8.3.3" },
     { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" },
     { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" },
+    { name = "httpx", specifier = ">=0.28.1" },
     { name = "jmespath", specifier = ">=1.1.0" },
     { name = "jsonschema", specifier = ">=4.26.0" },
     { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.28.1" },