diff --git a/.gitignore b/.gitignore index a775beb4..a582577c 100644 --- a/.gitignore +++ b/.gitignore @@ -204,3 +204,8 @@ src/tmp/ # Observability artifacts (OTLP-JSON traces + per-run trajectory JSON). traces/ + +/skills/private/ +/skills/**/private/ +/skills-private/ +assetops-skills-v*.zip \ No newline at end of file diff --git a/README.md b/README.md index a988ea51..afc603ba 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Or jump in instantly: - 🚀 **[Run on Colab](https://colab.research.google.com/github/IBM/AssetOpsBench/blob/main-0.x/notebook/LLM_Agent.ipynb)** — no install required (illustration of LLM Agent) - 🎮 **[Try the HF Playground](https://huggingface.co/spaces/ibm-research/AssetOps-Bench)** — interactive demo - 📖 **[Read INSTRUCTIONS.md](./INSTRUCTIONS.md)** — full setup, MCP servers, plan-execute runner +- 🧠 **[Running with Skills](./docs/running_with_skills.md)** — mount an operating-knowledge library, report it as a level, and measure what it changed > [!NOTE] > Active development is on `main`. The codebase used for various publication venues continues to be maintained on separate branches, for example, ACL 2026 [`IndustryAssetEQA`](https://github.com/IBM/AssetOpsBench/tree/IndustryAssetEQA) and prior experimental work is maintained on [`main-0.x`](https://github.com/IBM/AssetOpsBench/tree/main-0.x). diff --git a/docs/running_benchmark.md b/docs/running_benchmark.md index f440f281..dab7d347 100644 --- a/docs/running_benchmark.md +++ b/docs/running_benchmark.md @@ -4,9 +4,11 @@ trajectory per scenario, and scores the results. This page is everything you need to get from a fresh clone to a leaderboard report. -Related docs: [scenario_suite/README.md](scenario_suite/README.md) for scenario +Related docs: [scenario_suite/README.md](../benchmarks/scenario_suite/README.md) for scenario selectors, [../docs/stirrup-agent.md](../docs/stirrup-agent.md) for the agent -itself, [../INSTRUCTIONS.md](../INSTRUCTIONS.md) for the full environment table. +itself, [../INSTRUCTIONS.md](../INSTRUCTIONS.md) for the full environment table, +and [running_with_skills.md](running_with_skills.md) for running the same suite +with an operating-knowledge library mounted, and measuring what it changed. --- @@ -98,7 +100,7 @@ a selector — one id per line, `#` for comments: --scenario-ids my_scenarios.txt ``` -See [scenario_suite/README.md](scenario_suite/README.md) for the selector +See [scenario_suite/README.md](../benchmarks/scenario_suite/README.md) for the selector grammar (`fcc_lite`, `fcc+fmsr_all`, `lite`, `all`). ### CouchDB diff --git a/docs/running_with_skills.md b/docs/running_with_skills.md new file mode 100644 index 00000000..344c38f2 --- /dev/null +++ b/docs/running_with_skills.md @@ -0,0 +1,283 @@ +# Running the Benchmark with Skills + +An agent working this environment brings two things: a model, and whatever it +knows about industrial asset operations. The second is usually implicit, buried +in a system prompt or in whatever the backbone happens to remember about +bearings and chillers. This page makes it explicit, mountable, and reportable as +a level, so a run can say which operating knowledge it had. + +The mechanism is a **skill library** copied into the agent's code-execution +workspace, and a **K level** that says whether it was mounted. `K0` is the +unaided baseline and is byte-identical to the behaviour before any of this +existed. `K1` mounts a library. The difference between them, per task, is the +measurement. + +Related docs: [running_benchmark.md](running_benchmark.md) for the suite runner +and the leaderboard, [stirrup-agent.md](stirrup-agent.md) for the agent, +[../skills/README.md](../skills/README.md) for the library that ships here and +[../skills/CONTRACT.md](../skills/CONTRACT.md) for writing your own. + +--- + +## Quick start + +```bash +# 0. everything from running_benchmark.md first: uv sync, .env, CouchDB, code image + +# 1. prove the agent will see the skills, before spending a run +python skills/preflight.py --assetops . --skills skills/repositories + +# 2. one scenario, unaided +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k0 \ + --k-level k0 "" + +# 3. the same scenario, with the library mounted +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k1 \ + --skills-dir skills/repositories --k-level k1 "" +``` + +The preflight is worth the ten seconds. Its check 4 is the one that matters: + +``` +PASS 4 mount k0 nothing mounted, nothing appended +``` + +That proves `K0` really is unaided. A contaminated baseline invalidates every +comparison downstream of it, and it fails silently otherwise. + +--- + +## The three K levels + +| Level | What happens | Use it for | +| --- | --- | --- | +| `k0` | Mounts nothing, appends nothing to the system prompt | The baseline. This is the default | +| `k1` | Copies the library into the workspace, appends a routing block | The treatment | +| `k1-recovery` | Mounts the library but tells the agent to work unaided first and consult only after a concrete failure | Scoring the library on recovery rather than on substitution | + +`--k-level` defaults to `k0`, so nothing changes for anyone who does not pass the +new flags. + +`k1-recovery` answers a different question from `k1` and costs another full arm. +Run it only if you intend to report it. + +--- + +## What `--skills-dir` points at + +The directory holding **both** `repo-skills/` and `repo-skills-router/`. Not one +or the other: the router is the index into the graphs, and the prompt block names +only the router. + +``` +skills/repositories/ <- this is the path you pass + repo-skills/ the graphs + repo-skills-router/ the index +``` + +Two moving parts, both in `src/agent/stirrup_agent/skills_mount.py`: + +1. The library is copied into the code-execution workspace, so the agent sees it + at `/workspace/skills` under the Docker backend and `skills/` locally. +2. A block of about 650 characters is appended to the system prompt, naming the + router and the routing discipline. + +The agent already has a shell, so it reads a `SKILL.md` with `cat` and +progressive disclosure comes free: router, then one graph, then one sub-skill. +Nothing is loaded until it is chosen. **This is why the prompt cost does not grow +with the library**: a one-graph library and a forty-graph library both cost the +same 665 characters up front. + +### Using a different library + +Change one path. Nothing else, and no code change: + +```bash +--skills-dir /path/to/other-library/repositories +``` + +The library that ships here is a small reference one, complete and mountable, +covering this repository's own tool surface. A larger library, held anywhere, +mounts the same way. Validate any library before you mount it: + +```bash +python skills/tools/validate_skills.py --root /path/to/other-library/repositories +``` + +--- + +## A suite run, one arm at a time + +`--skills-dir` and `--k-level` are threaded through +`benchmark.scenario_suite_runner`, so a suite run takes them directly: + +```bash +MODEL="litellm_proxy/aws/claude-opus-5" +SKILLS=skills/repositories + +# K0 +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids lite --scenario-root benchmarks/scenario_suite \ + --agent_name stirrup_agent --model-id "$MODEL" --reasoning-effort high \ + --k-level k0 \ + --trajectory-root runs/k0/assetopsbench-trajectories \ + --reports-root runs/k0/assetopsbench-reports \ + --stirrup-workspace-root runs/k0/ws --preserve-workspaces + +# K1, identical except for the two skill flags and the output roots +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids lite --scenario-root benchmarks/scenario_suite \ + --agent_name stirrup_agent --model-id "$MODEL" --reasoning-effort high \ + --skills-dir "$SKILLS" --k-level k1 \ + --trajectory-root runs/k1/assetopsbench-trajectories \ + --reports-root runs/k1/assetopsbench-reports \ + --stirrup-workspace-root runs/k1/ws --preserve-workspaces +``` + +> **Give each arm its own output roots.** The suite runner names trajectory files +> by scenario id alone, so two arms sharing a root means the second silently +> overwrites the first, and the pairing below then compares an arm against +> itself. This is the single easiest way to waste a suite of runs. + +Keep everything else identical between arms: model, reasoning effort, +temperature, scenario selector, and **the commit**. A model or a code change +between arms is a confound the analysis cannot detect, because both arms still +look structurally fine. + +### Check the mount reached the agent, once + +`--preserve-workspaces` exists for this. Three commands, one time, and you never +again wonder: + +```bash +ls runs/k0/ws/stirrup_agent/*/*/skills 2>/dev/null # must be empty +ls runs/k1/ws/stirrup_agent/*/*/skills # repo-skills, repo-skills-router +grep -l "repo-skills" runs/k1/assetopsbench-trajectories/*/*/*.json +``` + +--- + +## Measuring the difference + +Per task, `s(t) = score_K1(t) - score_K0(t)`. Two tools do the work, and neither +needs anything instrumented: the benchmark already writes the score and the +operational metrics to `_aggregate.json`, and the trajectory already records +which `SKILL.md` files the agent opened. + +### 1. Build a run manifest + +```bash +python skills/tools/build_run_manifest.py --k-level k0 \ + --reports-root runs/k0/assetopsbench-reports \ + --trajectory-root runs/k0/assetopsbench-trajectories \ + --out runs/manifest.jsonl --expect-no-skills + +python skills/tools/build_run_manifest.py --k-level k1 \ + --reports-root runs/k1/assetopsbench-reports \ + --trajectory-root runs/k1/assetopsbench-trajectories \ + --out runs/manifest.jsonl --append --expect-skills +``` + +`--expect-no-skills` and `--expect-skills` check the arm label against what the +trajectories actually show. A mislabelled arm produces a clean-looking manifest +and a meaningless result, which is the kind of mistake you find months later. + +Add `--asset-class-map map.json`, a small JSON file mapping scenario id to asset +class, to enable the per-class breakdown. + +### 2. Run the analysis + +```bash +python skills/tools/gate5_counterfactual.py --runs runs/manifest.jsonl \ + --per-graph --emit gate5-admission.json +``` + +What it reports, and why each part is there: + +| Output | Why | +| --- | --- | +| Mean `s`, paired bootstrap interval, sign test | The resampling unit is the task, not the run, so repetitions of one scenario do not masquerade as independent observations | +| Regression count and rate, with a budget | Never netted into the mean. A library that helps on average while poisoning one asset class is worse than no library | +| Per asset class | Where that poisoning becomes visible behind a positive mean | +| Per graph | Restricted to the tasks where the agent actually opened that graph, corrected across graphs by Benjamini-Hochberg | +| Minimum detectable effect | So a null reads as "no effect" or "not enough runs", which are different findings | +| Spearman of `s` against extra tokens, steps, tool calls | Pre-empts "you just spent more compute" | +| Contamination check on the recorded `k0` runs | The preflight proves the k0 *code path* mounts nothing; this proves the k0 *runs that were scored* consulted nothing | + +The verdict is one of `ADMITTED`, `NOT_SHOWN_TO_HELP`, `UNDERPOWERED`, +`HARMFUL` or `VOID`. Exit code is 0 only when the effect is admitted and nothing +else failed, so it can sit in CI as a release gate once a baseline exists. + +`python skills/tools/gate5_counterfactual.py --self-test` plants known effects +and checks they are recovered, that a null library is refused, and that a +contaminated baseline voids the run. Run it before spending a suite on it. + +### How many tasks you need + +Minimum detectable mean `s` at 80 percent power, two-sided alpha 0.05: + +| Paired tasks | sd 0.15 | sd 0.25 | sd 0.35 | +| --- | ---: | ---: | ---: | +| 3 (`open`) | 0.243 | 0.404 | 0.566 | +| 50 (`lite`) | 0.059 | 0.099 | 0.139 | +| 215 (`all`) | 0.029 | 0.048 | 0.067 | + +The per-graph table is the harder constraint: a graph the agent opens on 20 of +215 tasks needs roughly a 0.16 effect to clear its own interval, so expect +`INSUFFICIENT_POWER` on many graphs even at full scale. That is reported rather +than hidden, and how often each graph was opened is itself a finding about the +suite's coverage. + +Start with `open` (3 scenarios) to shake out the plumbing. It cannot answer the +research question and is not meant to; a verdict of `UNDERPOWERED` there is the +correct result. + +--- + +## Before every evaluation + +```bash +# frontmatter, licence, self-containment, routing metadata, industrial axes +python skills/tools/validate_skills.py --root skills/repositories + +# the leakage audit, which needs your answer set +python skills/tools/validate_skills.py --root skills/repositories \ + --answers-hf ibm-research/AssetOpsBench +``` + +> **Treat a leakage hit as blocking.** A skill library sits closer to the answers +> than anything else an agent reads. The audit fails any eight-word sequence +> shared between a skill and the answer set, and names the scenario each hit came +> from so it can be triaged rather than argued with. Pointing it at the checkout +> proves nothing: `benchmarks/scenario_suite/*.yaml` holds scenario **ids** only. +> Use `--answers-hf` for the published dataset, or `--answers-dir` for whatever +> your harness exports. + +A `leakage-class: solution` skill fails outright rather than warning. + +--- + +## Recording a run + +Put the **K level** and the **library version** in every results row, beside the +model id. A library bump moves the leaderboard exactly as a model change does, +and a run that does not record which library it read cannot be compared with one +that read another. The library version is `metadata.library-version` in the +frontmatter, and the taxonomy it was routed against is `taxonomy_sha256` in each +graph's `repo-routing-metadata.json`. + +Keep the transcript. Per-graph attribution reads it for the skill paths the agent +opened, so without it every other part of the analysis still works and the +per-graph table is empty. + +--- + +## What this does not tell you + +Mounting a library and measuring a delta says whether *that* library helped *this* +suite at *this* power. It does not say operating knowledge helps in general, and +it says nothing at all about a graph no run ever opened. + +Until a paired suite has actually run, the honest description of any library is +**constructed and gated**, not shown to help. The gate exists so that claim can +become a measured one; running it is what changes the wording. diff --git a/skills/CONTRACT.md b/skills/CONTRACT.md new file mode 100644 index 00000000..f52a5b25 --- /dev/null +++ b/skills/CONTRACT.md @@ -0,0 +1,161 @@ +# Skill contract + +What a skill graph must look like to mount and validate here. The graph in +`repositories/repo-skills/assetopsbench/` is a worked example of every rule +below; read it alongside this file. + +The format follows the AREX repository-skill contract, with three additions the +physical-asset setting forces: an asset-class axis alongside the capability +axis, a leakage class, and a rule that a script must refuse something. + +## Layout + +``` +repositories/ + repo-skills-router/ + SKILL.md the index; the prompt names this file + references/entry.md one page: what the library covers, and the route + references/areas/.md one page per area + repo-skills/ + / + SKILL.md root router, 80 to 150 lines with frontmatter + references/ + repo-provenance.md required, schema below + repo-routing-metadata.json required, schema below + .md whatever backs the numbers in the skill + scripts/.py 0 to 2 graph-level gates + sub-skills// + SKILL.md 80 to 250 lines with frontmatter + scripts/.py usually one +``` + +Every relative link must resolve inside the library. Sub-skill ids are unique +across the whole library, not just within a graph. + +## Frontmatter + +Required on every `SKILL.md`: + +```yaml +--- +name: +description: "" +disable-model-invocation: true +license: +metadata: + disco-role: operating + capability-family: + asset-class: + leakage-class: ops + library-version: 0.1.0 +--- +``` + +The router's own `SKILL.md` is the exception: it must not set +`disable-model-invocation`, because it is the file the agent is told to open. + +**Capability families.** C1 asset and sensor discovery; C2 time-series retrieval +and conditioning; C3 data-quality triage and instrument faults; C4 signal +processing and vibration; C5 anomaly and change point; C6 forecasting; C7 +failure-mode reasoning and sensor mapping; C8 health, degradation and RUL; C9 +root-cause isolation and diagnostic chaining; C10 maintenance planning and work +orders; C11 control, setpoint and energy efficiency; C12 evidence assembly and +reporting. + +**Asset classes.** `A0` (asset-agnostic), `chiller-hvac`, `ahu`, `pumps`, +`motors-drives`, `fans-blowers`, `compressors`, `bearings-gearboxes`, +`wind-turbine`, `transformers-electrical`. + +Two axes rather than one, because a capability and the machine it is applied to +come apart: envelope analysis is a capability, a gearbox is an asset, and the +useful skill lives at the intersection. A single axis forces either twelve +bloated skills or a hundred duplicated ones. + +**`leakage-class`** is `ops` for anything that ships. `solution` marks a skill +derived from answers, and the validator fails it outright rather than warning. + +## `references/repo-routing-metadata.json` + +```json +{ + "schema_version": "2.0", + "repo_id": " for a graph with no upstream>", + "skill_id": "", + "taxonomy_sha256": "sha256:<64 lowercase hex characters>", + "routing_status": "classified", + "assignments": [{ "area": "", "family": "" }] +} +``` + +The router's area pages are generated from these files, so a graph cannot be +routable and undeclared, or declared and unroutable. `taxonomy_sha256` pins +which taxonomy version the assignment was made against; without it a library can +be re-routed silently and two runs that read "the same" library stop being +comparable. + +**Write the digest in URI form**, `sha256:` followed by the hex, the same shape +OCI image references and Subresource Integrity use. This is not decoration. A +bare 64-character hex string is indistinguishable from a credential to an +entropy scanner: this repository runs `detect-secrets`, whose +`HexHighEntropyString` plugin flags a bare digest at 64, 32 and even 16 +characters and blocks the commit. The prefix clears every scanner tested and +names the algorithm at the point of use, so it is the better representation +regardless of the scanner. The validator enforces the form and says so by name +if it finds a bare digest. + +## `references/repo-provenance.md` + +Opens with ` schema: disco.repo-provenance.v1`, then the fields shown in the +example graph: `graph_kind`, the sources you actually read, `inspection_method`, +`license`. Then two sections that carry the weight: + +- **Evidence.** Every API you cite, with its real signature, read from the + installed distribution or cloned source during construction. Then how any + measured result was produced: the generator, the sample counts, where the + numbers live. +- **Excluded.** What you did not consult and why. Benchmark payloads. Packages + that failed to install and what you used instead. Standards text. + +## Rules the validator enforces + +1. **No absolute paths.** No `/home/...`, no `/Users/...`, no `site-packages`, + no environment activation. A library is copied into a fresh workspace on + every run and must work there. +2. **No benchmark leakage.** Nothing derived from scenario payloads, scorer + logic, ground truth or expected outputs. Exclude at gather time; auditing + leakage out of a finished skill is strictly worse than never letting it in. +3. **Line limits.** Root 80 to 150, sub-skill 80 to 250. Detail goes to + `references/`, which is loaded only when a sub-skill points at it. +4. **One licence per graph.** A library may span licences; a single graph may + not. + +## Rules the validator cannot enforce, and which matter more + +**Never write an API you have not verified.** Install the package in a throwaway +virtualenv and introspect it, or clone and read the source. Record the exact +version. This single rule is where most of a library's value comes from, and +skipping it produces something that reads like a README and is wrong in the +specifics. + +**Never write a number you did not measure.** If a skill says a method loses +three percent, someone computed three percent in the session that wrote it. An +unmeasured quantitative claim is the worst thing a library can ship, because a +reader will check it and everything else becomes suspect at once. + +**A script is a gate, not a demo.** It takes a claim or a computed result and +returns a pass or a named rejection. Ship it with a `--self-test` that builds +both a passing and a failing case, and make the failing case the input that +would otherwise have slipped through. A gate that passes everything is not a +gate, and a self-test written to confirm the gate's intent rather than probe its +boundary will not tell you which one you have. + +**Every script must respond to `--help` on a bare interpreter.** Import optional +dependencies inside the function that needs them and exit with a one-line +`install X` message, never a traceback. + +**Lead with the mistake.** Each sub-skill opens with the error it prevents, then +the procedure, then the gate. State the precondition under which the method +stops working. That precondition is the part a capable model does not already +know, and it is the reason the skill exists. diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 00000000..6c82a627 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,110 @@ +# Skills + +Operating knowledge for agents working this benchmark, mounted into the agent's +workspace and reported as a controlled variable rather than baked into a prompt. + +This directory holds two separate things, and the distinction is the point: + +1. **The interface.** The mount, the K-level control, the skill contract, the + validator and the router. All public, all in this repository. +2. **A library.** One reference graph, `assetopsbench`, covering this + repository's own tool surface. Complete and mountable, deliberately small. + +A larger library, held anywhere, mounts through the same interface with no code +change. That is what makes the interface worth publishing on its own. + +## The split, and why it mirrors the scenario split + +The scenario suite in this repository is public; a held-out suite is not. Skills +work the same way, for the same reason. + +| | Public, here | Held out | +| --- | --- | --- | +| Scenarios | The released suite | The evaluation suite | +| Skills | The interface, and the `assetopsbench` graph | A larger domain library | + +A skill library is an experimental condition. If the library an agent reads is +published alongside the tasks it is scored on, then the library can be tuned to +the tasks, and a result no longer measures whether operating knowledge helps. It +measures whether that knowledge was fitted to that suite. Holding a library out +is the same control as holding scenarios out, and it is why `--k-level` reports +which condition a run used rather than leaving it implicit. + +Nothing about the mechanism is secret. Anyone can build a library against the +contract below and run the same three arms. + +## Running it + +[docs/running_with_skills.md](../docs/running_with_skills.md) is the full guide: +the three arms, a suite run, and how to measure the difference. The short version +is below. + +```bash +# K0: unaided baseline. Mounts nothing, appends nothing to the prompt. +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k0 \ + --k-level k0 "" + +# K1: this repository's reference library +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k1 \ + --skills-dir skills/repositories --k-level k1 "" + +# K1 with a different library: change one path, nothing else +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k1 \ + --skills-dir /path/to/other-library/repositories --k-level k1 "" + +# K1-recovery: unaided first, skills consulted only after a concrete failure +uv run python -m agent.stirrup_agent.cli --workspace-dir ./ws-k1r \ + --skills-dir skills/repositories --k-level k1-recovery "" +``` + +`--skills-dir` points at the directory holding **both** `repo-skills/` and +`repo-skills-router/`. It is not one or the other: the router is the index into +the graphs, and the prompt block names only the router. + +**K0 is byte-identical to the behaviour before any of this existed.** It mounts +nothing and appends nothing, which is what makes the comparison honest. Record +the K level and the library version in every results row; a library change moves +the leaderboard exactly as a model change does. + +## How a skill reaches the agent + +Two moving parts, both in `src/agent/stirrup_agent/skills_mount.py`: + +1. The library is copied into the code-execution workspace, so the agent sees it + at `/workspace/skills` under the Docker backend and `skills/` locally. +2. A short block is appended to the system prompt naming the router and the + routing discipline. It is about 650 characters and it names one file, not the + library, because the collection is routed rather than enumerated. + +The agent already has a shell, so it reads a `SKILL.md` with `cat` and +progressive disclosure comes free: router, then one graph, then one sub-skill. +Nothing is loaded until it is chosen. This is why a library of forty graphs +costs the same prompt budget as a library of one. + +## Checking a library before you spend a run + +```bash +python skills/tools/validate_skills.py --root skills/repositories +``` + +`skills/tools/` also holds `build_run_manifest.py`, which turns the benchmark's +own reports and trajectories into a run manifest, and `gate5_counterfactual.py`, +which measures `s(t) = score_K1(t) - score_K0(t)` from it. Both are described in +[docs/running_with_skills.md](../docs/running_with_skills.md). + +Frontmatter contract, per-tree licence consistency, self-containment, and a +leakage audit. Add `--answers` to point the leakage half at your answer set; +without it that half does not run and says so. + +The leakage check exists because a skill library sits closer to the answers than +anything else an agent reads. `leakage-class: solution` fails outright, and any +eight-word sequence shared between a skill and the answer set is a failure that +names the scenario it came from. + +## Contributing a graph + +`CONTRACT.md` has the layout, the frontmatter, and the rules. The short version: +install or clone what you are describing and read it, rather than writing from +memory; make every script a gate that refuses something, with a `--self-test` +that proves it refuses; and lead each sub-skill with the mistake it prevents, +because that is the part a capable model does not already know. diff --git a/skills/preflight.py b/skills/preflight.py new file mode 100644 index 00000000..2c76476b --- /dev/null +++ b/skills/preflight.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Preflight for a skill library inside AssetOpsBench. + +Run this after installing, before spending a benchmark run. It answers the one +question that matters at handoff: will the agent actually see the skills. + + python skills/preflight.py --assetops . --skills skills/repositories + +Seven checks, in the order that a failure would block you: + + 1. Skill tree the collection is present, well-formed, and countable + 2. Patch the Stirrup plug is applied to the target checkout + 3. Import `skills_mount` imports and exposes the expected contract + 4. Mount k0 mounts nothing and appends nothing, so the baseline is intact + 5. Mount k1 copies the tree into a workspace and returns a prompt block + 6. Router the mounted tree's entry point and router resolve + 7. Runner wiring StirrupAgentRunner accepts `skills_dir` and `k_level` + +Exit codes: 0 ready to run, 1 a check failed, 2 bad invocation. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + +ROWS: list[tuple[str, str, str]] = [] + + +def ok(name: str, detail: str = "") -> None: + ROWS.append(("PASS", name, detail)) + + +def bad(name: str, detail: str) -> None: + ROWS.append(("FAIL", name, detail)) + + +def warn(name: str, detail: str) -> None: + ROWS.append(("WARN", name, detail)) + + +def check_tree(skills: pathlib.Path) -> int: + graphs_dir = skills / "repo-skills" + router = skills / "repo-skills-router" / "SKILL.md" + if not graphs_dir.is_dir(): + bad("1 skill tree", f"no repo-skills directory under {skills}") + return 0 + graphs = [p for p in graphs_dir.iterdir() if p.is_dir() and (p / "SKILL.md").exists()] + subs = sum(len(list((g / "sub-skills").glob("*/SKILL.md"))) for g in graphs) + total = len(graphs) + subs + if not router.exists(): + bad("1 skill tree", "repo-skills-router/SKILL.md missing, routing will not work") + return total + entry = skills / "repo-skills-router" / "references" / "entry.md" + if not entry.exists(): + warn("1 skill tree", "router references/entry.md missing; agents will route without the one-page entry") + ok("1 skill tree", f"{len(graphs)} graphs, {total} skills, router present") + # The mount copies every SKILL.md, which is the graph and sub-skill count + # plus the router itself. Return the file count so check 5 compares like + # with like. + return total + 1 + + +def check_patch(aob: pathlib.Path) -> bool: + mount = aob / "src" / "agent" / "stirrup_agent" / "skills_mount.py" + runner = aob / "src" / "agent" / "stirrup_agent" / "runner.py" + if not runner.exists(): + bad("2 patch", f"not an AssetOpsBench checkout: {runner} missing") + return False + if not mount.exists(): + bad("2 patch", "skills_mount.py missing; apply patches/stirrup_skills_plug.diff") + return False + body = runner.read_text(errors="replace") + missing = [t for t in ("skills_mount", "skills_dir", "k_level") if t not in body] + if missing: + bad("2 patch", f"runner.py lacks {missing}; the patch is not applied") + return False + ok("2 patch", "skills_mount.py present and runner.py wired") + return True + + +def check_import(aob: pathlib.Path): + sys.path.insert(0, str(aob / "src" / "agent" / "stirrup_agent")) + try: + import skills_mount # type: ignore + except Exception as exc: # noqa: BLE001 + bad("3 import", f"{type(exc).__name__}: {exc}") + return None + for attr in ("mount_skills", "K_LEVELS"): + if not hasattr(skills_mount, attr): + bad("3 import", f"skills_mount has no `{attr}`") + return None + ok("3 import", f"K_LEVELS = {tuple(skills_mount.K_LEVELS)}") + return skills_mount + + +def check_mounts(sm, skills: pathlib.Path, total: int) -> None: + with tempfile.TemporaryDirectory() as td: + ws0 = pathlib.Path(td) / "k0" + ws0.mkdir() + try: + block = sm.mount_skills(skills, ws0, k_level="k0", code_backend="docker") + except Exception as exc: # noqa: BLE001 + bad("4 mount k0", f"{type(exc).__name__}: {exc}") + return + if block is not None: + bad("4 mount k0", "k0 returned a prompt block; the baseline is not clean") + elif any(ws0.iterdir()): + bad("4 mount k0", "k0 wrote files into the workspace") + else: + ok("4 mount k0", "nothing mounted, nothing appended") + + ws1 = pathlib.Path(td) / "k1" + ws1.mkdir() + try: + block = sm.mount_skills(skills, ws1, k_level="k1", code_backend="docker") + except Exception as exc: # noqa: BLE001 + bad("5 mount k1", f"{type(exc).__name__}: {exc}") + return + landed = list((ws1 / "skills").rglob("SKILL.md")) + if not block: + bad("5 mount k1", "no prompt block returned") + elif not landed: + bad("5 mount k1", "no SKILL.md landed in the workspace") + else: + if len(landed) != total: + warn("5 mount k1", f"{len(landed)} SKILL.md landed, tree has {total}") + ok("5 mount k1", f"{len(landed) - 1} skills plus the router mounted, " + f"prompt block {len(block)} chars") + + router = ws1 / "skills" / "repo-skills-router" / "SKILL.md" + if not router.exists(): + bad("6 router", "router did not survive the mount") + elif "/workspace/skills" not in (block or ""): + bad("6 router", "prompt block does not name the docker mount path") + else: + ok("6 router", "router mounted and named in the prompt block") + + +def check_runner(aob: pathlib.Path) -> None: + """Parse runner.py rather than importing it, so no heavy deps are needed.""" + src = (aob / "src" / "agent" / "stirrup_agent" / "runner.py").read_text(errors="replace") + try: + tree = ast.parse(src) + except SyntaxError as exc: + bad("7 runner wiring", f"runner.py does not parse: {exc}") + return + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == "StirrupAgentRunner": + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == "__init__": + args = {a.arg for a in item.args.args} | {a.arg for a in item.args.kwonlyargs} + missing = {"skills_dir", "k_level"} - args + if missing: + bad("7 runner wiring", f"__init__ lacks {sorted(missing)}") + else: + ok("7 runner wiring", "StirrupAgentRunner accepts skills_dir and k_level") + return + bad("7 runner wiring", "StirrupAgentRunner.__init__ not found") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--assetops", type=pathlib.Path, required=True, + help="path to the AssetOpsBench checkout with the patch applied") + ap.add_argument("--skills", type=pathlib.Path, + default=pathlib.Path(__file__).parent / "repositories", + help="path to the skill collection (the directory holding repo-skills/)") + ap.add_argument("--json", action="store_true") + a = ap.parse_args() + + total = check_tree(a.skills.resolve()) + if check_patch(a.assetops.resolve()): + sm = check_import(a.assetops.resolve()) + if sm is not None: + check_mounts(sm, a.skills.resolve(), total) + check_runner(a.assetops.resolve()) + + failed = any(r[0] == "FAIL" for r in ROWS) + if a.json: + print(json.dumps({"ready": not failed, + "checks": [{"status": s, "check": c, "detail": d} for s, c, d in ROWS]}, + indent=2)) + else: + for status, name, detail in ROWS: + print(f"{status:<5} {name:<18} {detail}") + print() + print("READY: run the benchmark" if not failed + else "NOT READY: fix the failures above before spending a run") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/repositories/repo-skills-router/SKILL.md b/skills/repositories/repo-skills-router/SKILL.md new file mode 100644 index 00000000..e2f7f052 --- /dev/null +++ b/skills/repositories/repo-skills-router/SKILL.md @@ -0,0 +1,42 @@ +--- +name: repo-skills-router +description: "Routes a request to the skill graph that owns the capability, narrowing area, then family, then graph, then sub-skill. Read this first, before opening any graph, so that only the relevant branch is loaded." +license: Apache 2.0 +metadata: + disco-role: operating +--- + +# Skill router + +## Purpose + +Narrow before you load. Open an area page, then the graph it names, then that +graph's sub-skill for the step you are on. Read one sub-skill at a time, and +open a reference file only when a sub-skill points at it. + +This discipline matters more as the library grows. The routing cost is one page +whether the library holds one graph or forty, which is the whole reason the +prompt names this file and nothing else. + +## Areas + +| Area | Graphs | +| --- | ---: | +| [Industrial asset operations](references/areas/industrial-asset-operations.md) | 1 | + +## Start here + +Read [`references/entry.md`](references/entry.md). It is one page: what this +library covers, the route from what a request asks for to the graph that answers +it, and the rule for when to call an MCP tool versus when to run code in the +workspace. + +## What this library is + +This is the reference library shipped in the AssetOpsBench repository. It holds +one graph, `assetopsbench`, covering the benchmark's own tool surface and the +evidence discipline it scores on. It is complete and mountable as it stands. + +It is also the worked example for the skill contract. A larger library, public +or private, mounts in exactly the same way and replaces this one: see +`skills/README.md` and `skills/CONTRACT.md`. diff --git a/skills/repositories/repo-skills-router/references/areas/industrial-asset-operations.md b/skills/repositories/repo-skills-router/references/areas/industrial-asset-operations.md new file mode 100644 index 00000000..2aa16284 --- /dev/null +++ b/skills/repositories/repo-skills-router/references/areas/industrial-asset-operations.md @@ -0,0 +1,16 @@ +# Industrial asset operations + +1 family, 1 skill graph assigned. + +Read a family page only after confirming the family scope matches the capability +the current step needs. + +| Family | Graphs | +| --- | ---: | +| Asset and sensor discovery | 1 | + +## Asset and sensor discovery + +| Graph | Covers | +| --- | --- | +| [`assetopsbench`](../../../repo-skills/assetopsbench/SKILL.md) | The benchmark's six MCP servers, 85 tools, and the evidence discipline that applies to every answer | diff --git a/skills/repositories/repo-skills-router/references/entry.md b/skills/repositories/repo-skills-router/references/entry.md new file mode 100644 index 00000000..9d6a9c89 --- /dev/null +++ b/skills/repositories/repo-skills-router/references/entry.md @@ -0,0 +1,34 @@ +# Entry page + +One page. Read this before opening a graph. + +## What this library covers + +One graph, `assetopsbench`: which of the six MCP servers owns which capability, +and the evidence discipline this environment scores on. It does not cover +domain judgement, which is what a larger library adds. + +## Route + +| The request is about | Open | +| --- | --- | +| Which server or tool to use, or a server refusing | `assetopsbench` then `server-routing` | +| Whether an answer is supportable, or an underspecified request | `assetopsbench` then `evidence-and-abstention` | +| Anything else | Nothing here covers it. Say so rather than stretching a skill to fit | + +That last row is not filler. A library that always has an answer is a library +that is guessing, and routing to a graph that does not cover the step is worse +than routing to nothing, because it lends unearned confidence. + +## MCP tool or code workspace + +Call an MCP tool when the environment holds the thing: assets, sensors, +telemetry, failure modes, spectra, work orders, runs. + +Use the code workspace when the step is computation over things you already +retrieved: arithmetic across two results, a unit conversion, a statistic no +server exposes, a plot. Doing it in code is correct and it is recorded. + +Do neither, and say so, when the step needs a value that no call returned and no +computation can produce. That is an abstention, and it is a scored outcome here +rather than a failure to answer. diff --git a/skills/repositories/repo-skills/assetopsbench/SKILL.md b/skills/repositories/repo-skills/assetopsbench/SKILL.md new file mode 100644 index 00000000..ac2a2f50 --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/SKILL.md @@ -0,0 +1,109 @@ +--- +name: assetopsbench +description: "Operates the AssetOpsBench MCP surface: six stdio FastMCP servers holding + 85 tools across asset and sensor discovery, failure-mode reasoning, time-series + modelling, vibration diagnostics, work-order management and reference catalogs. + Route here when a request concerns a physical asset at a site, its sensors or + telemetry, its failure modes, a forecast or anomaly check on its signals, a + vibration spectrum, or a maintenance work order. Read this before calling any + tool, so the server that owns the capability is chosen rather than guessed, and + so the evidence discipline this environment scores on is applied from the first + call rather than reconstructed afterwards." +disable-model-invocation: true +license: Apache 2.0 +metadata: + disco-role: operating + capability-family: C1, C2, C12 + asset-class: A0 + leakage-class: ops + library-version: 0.1.0 +--- + +# AssetOpsBench tool surface + +## Purpose + +AssetOpsBench exposes an industrial asset operations environment as six stdio +MCP servers holding 85 tools. This graph is the map of that surface: which +server owns which capability, how to reach it, and the evidence discipline that +applies to every answer here. + +This is the reference skill graph shipped with the repository. It is complete +and mountable on its own, and it is deliberately small. See `skills/README.md` +for how a larger library is mounted in its place. + +## The one thing to get right first + +Answers in this environment are judged on the execution record, not only on the +claim. A conclusion that no executed tool call supports is not a weaker answer, +it is an unsupported one, and it scores as one. Two consequences that change +what you do before you have any results: + +1. **Retrieve before you assert.** If you cannot name the call that produced a + number, do not put the number in the answer. +2. **Abstain rather than interpolate.** Reporting that the evidence is + insufficient is a correct answer when it is true. Filling the gap with a + plausible value is not a partially correct answer, it is a wrong one that is + harder to detect. + +## Server access + +The six servers are launched as stdio subprocesses. The launch contract lives +in `src/mcphub/__init__.py`: + +```python +DEFAULT_SERVERS = {n: ["uv", "run", f"{n}-mcp-server"] + for n in ["iot", "utilities", "fmsr", "wo", "tsfm", "vibration"]} +``` + +Before anything else in a fresh environment, prove the surface is reachable and +is the surface this skill documents: + +```bash +python scripts/check_servers.py --json +``` + +It completes the MCP handshake, calls `tools/list`, and asserts that the +documented tool names are the live tool names. A server that fails the handshake +is unavailable, not empty. Do not work around it by guessing values; say the +server is down and stop. + +`AOB_READONLY=1` removes the six work-order mutation tools. Check whether it is +set before planning any write, because a plan that ends in a write you cannot +perform is a plan you have to redo. + +## Which server owns what + +| Server | Tools | Owns | +| --- | ---: | --- | +| `iot` | 12 | Sites, assets, sensors, and telemetry history | +| `fmsr` | 3 | Failure modes and their sensor relationships | +| `tsfm` | 41 | Forecasting, anomaly detection, data quality, recipes and runs | +| `vibration` | 8 | Spectra, envelope analysis, bearing frequencies | +| `wo` | 15 | Work orders: history, distribution, generation and updates | +| `utilities` | 6 | Reference catalogs and lookups | + +Full tool-by-tool inventory: `references/server-capability-map.md`. + +## Sub-skills + +Open one, for the step you are on. Do not read both up front. + +| Sub-skill | Open it when | +| --- | --- | +| [`server-routing`](sub-skills/server-routing/SKILL.md) | You know what you need and not which server has it, or a server is refusing, or you are about to write | +| [`evidence-and-abstention`](sub-skills/evidence-and-abstention/SKILL.md) | You are about to state a conclusion, or you suspect the evidence does not reach it | + +## Failure modes of this skill + +- **It maps the surface, not the domain.** It tells you `vibration` owns + envelope analysis. It does not tell you whether the sampling rate resolved the + harmonic you are about to name. That judgement lives in a domain library. +- **The tool counts are pinned to a commit.** If `check_servers.py` reports + `SKILL_GAP`, the skill is stale and the server is right. + +## Stop conditions + +Stop and report rather than proceeding if a server fails its handshake, if a +requested asset or sensor does not resolve, or if the only path to an answer is +a value no call returned. diff --git a/skills/repositories/repo-skills/assetopsbench/references/repo-provenance.md b/skills/repositories/repo-skills/assetopsbench/references/repo-provenance.md new file mode 100644 index 00000000..49e20257 --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/references/repo-provenance.md @@ -0,0 +1,40 @@ + schema: disco.repo-provenance.v1 + +- graph_kind: tool-surface +- lane_a_source: IBM/AssetOpsBench, this repository, read at the commit this + file ships with. The tool surface was extracted from source by AST rather + than from documentation, so a tool named here is a tool that is registered. +- lane_b_libraries: none. This graph documents an MCP surface and needs no + third-party distribution to do it. +- lane_c_standards: none reproduced. This graph makes no reference to any + standards text, table or threshold. +- inspection_method: AST extraction of the six FastMCP server modules plus + execution of the stdio handshake against each server +- license: Apache 2.0 + +## Evidence + +Six stdio FastMCP servers, launched by the contract in `src/mcphub/__init__.py`: + +```python +DEFAULT_SERVERS = {n: ["uv", "run", f"{n}-mcp-server"] + for n in ["iot", "utilities", "fmsr", "wo", "tsfm", "vibration"]} +``` + +Registered tool counts, extracted from the server modules: `iot` 12, `fmsr` 3, +`tsfm` 41, `vibration` 8, `utilities` 6, `wo` 15. Total 85. + +`AOB_READONLY=1` removes six work-order mutation tools from the `wo` surface. +Verified by launching `wo` with and without the variable and comparing +`tools/list`. + +`scripts/check_servers.py` performs the same handshake at runtime and asserts +the documented names against the live names, so this file cannot drift silently +past the code it describes. + +## Excluded + +- No benchmark scenario payload, scorer, reference answer or expected output was + consulted. This graph describes the tool surface only. +- `materialize_iot` is a test helper rather than a registered tool and is + therefore absent from the inventory, although a naive grep would find it. diff --git a/skills/repositories/repo-skills/assetopsbench/references/repo-routing-metadata.json b/skills/repositories/repo-skills/assetopsbench/references/repo-routing-metadata.json new file mode 100644 index 00000000..7863525f --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/references/repo-routing-metadata.json @@ -0,0 +1,13 @@ +{ + "schema_version": "2.0", + "repo_id": "IBM/AssetOpsBench", + "skill_id": "assetopsbench", + "routing_status": "classified", + "taxonomy_sha256": "sha256:3195427e04469614ee241cb6d95acc96ec9c8af92e52fe49d1d17781444c1f7b", + "assignments": [ + { + "area": "industrial-asset-operations", + "family": "asset-and-sensor-discovery" + } + ] +} diff --git a/skills/repositories/repo-skills/assetopsbench/references/server-capability-map.md b/skills/repositories/repo-skills/assetopsbench/references/server-capability-map.md new file mode 100644 index 00000000..f377b8c1 --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/references/server-capability-map.md @@ -0,0 +1,128 @@ +# Server capability map + +Extracted from the six FastMCP server modules by AST, so a tool named here is a +tool that is registered. `scripts/check_servers.py` asserts these names against +the live surface; when the two disagree, the server is right and this file is +stale. + +## `iot` (12 tools) + +- `iot.sites()` -> `SitesResult` +- `iot.asset_ids(site_name: str)` -> `Union[AssetsResult, ErrorResult]` +- `iot.asset_detail(site_name: str, asset_id: str)` -> `Union[AssetDetail, ErrorResult]` +- `iot.measured_sensors(site_name: str, asset_id: str)` -> `Union[SensorsResult, ErrorResult]` +- `iot.installed_sensors(site_name: str, asset_id: str)` -> `Union[SensorsResult, ErrorResult]` +- `iot.assets(site_name: str, assettype: Optional[str])` -> `Union[AssetsWithMetadataResult, ErrorResult]` +- `iot.find_assets_by_sensors(site_name: str, sensors: List[str], match: str, substring: bool, source: str)` -> `Union[FindAssetsResult, ErrorResult]` +- `iot.stream_extent(site_name: str, asset_id: str, sensor: Optional[str], start: Optional[str], end: Optional[str])` -> `Union[StreamExtentResult, ErrorResult]` +- `iot.history(site_name: str, asset_id: str, start: Optional[str], end: Optional[str], sensors: Optional[List[str]], limit: int, cursor: Optional[str])` -> `Union[HistoryResult, ErrorResult]` +- `iot.latest_reading(site_name: str, asset_id: str, sensor: Optional[str])` -> `Union[LatestReadingResult, ErrorResult]` +- `iot.sensor_coverage(site_name: str, asset_id: str)` -> `Union[SensorCoverageResult, ErrorResult]` +- `iot.sensor_stats(site_name: str, asset_id: str, sensor: Optional[str], start: Optional[str], end: Optional[str])` -> `Union[SensorStatsResult, ErrorResult]` + +## `fmsr` (3 tools) + +- `fmsr.get_failure_modes(asset_class: str)` -> `Union[FailureModesResult, ErrorResult]` +- `fmsr.generate_failure_modes(asset_class: str, max_modes: int)` -> `Union[GenerateFailureModesResult, ErrorResult]` +- `fmsr.add_failure_modes(asset_class: str, failure_modes: List[str], exhaustive: Optional[bool], source: Optional[str])` -> `Union[AddFailureModesResult, ErrorResult]` + +## `tsfm` (41 tools) + +- `tsfm.list_tasks()` -> `Union[TasksResult, ErrorResult]` +- `tsfm.profile_series(dataset_path: str, timestamp_column: Optional[str], channels: Optional[List[str]])` -> `Union[ProfileResult, ErrorResult]` +- `tsfm.characterize_series(dataset_path: str, timestamp_column: Optional[str], channels: Optional[List[str]], groups: Optional[dict], group_rules: Optional[str])` -> `Union[CharacterizeResult, ErrorResult]` +- `tsfm.data_quality(dataset_path: str, timestamp_column: str)` -> `Union[DataQualityResult, ErrorResult]` +- `tsfm.list_features(kind: Optional[str], status: Optional[str])` -> `Union[FeaturesResult, ErrorResult]` +- `tsfm.list_models(task_id: Optional[str], domain: Optional[str], status: str)` -> `Union[ModelsResult, ErrorResult]` +- `tsfm.search_models(text: str, tags: Optional[List[str]], status: str)` -> `Union[ModelsResult, ErrorResult]` +- `tsfm.find_models(task_id: str, min_context_length: Optional[int], prediction_length: Optional[int], domain: Optional[str], top_k: int)` -> `Union[ModelsResult, ErrorResult]` +- `tsfm.describe_candidates(task_id: str, top_k: int, domain: Optional[str])` -> `Union[CandidatesResult, ErrorResult]` +- `tsfm.describe_models(model_ids: List[str])` -> `Union[DescribeModelsResult, ErrorResult]` +- `tsfm.count_models()` -> `Union[ModelCountResult, ErrorResult]` +- `tsfm.list_domains(task_id: Optional[str])` -> `Union[DomainsResult, ErrorResult]` +- `tsfm.get_model_lineage(model_id: str)` -> `Union[LineageResult, ErrorResult]` +- `tsfm.register_model(model: dict)` -> `Union[RegisterResult, ErrorResult]` +- `tsfm.model_template()` -> `ModelTemplateResult` +- `tsfm.register_finetuned(model_id: str, checkpoint_path: str, base_model_id: str, context_length: int, prediction_length: int, description: str, domain: str)` -> `Union[CardResult, ErrorResult]` +- `tsfm.update_model(model_id: str, fields: dict)` -> `Union[CardResult, ErrorResult]` +- `tsfm.deprecate_model(model_id: str, reason: Optional[str])` -> `Union[CardResult, ErrorResult]` +- `tsfm.new_model_version(model_id: str, fields: dict, new_model_id: Optional[str])` -> `Union[CardResult, ErrorResult]` +- `tsfm.resolve_model(model_id: str)` -> `Union[ResolveResult, ErrorResult]` +- `tsfm.hf_stats(model_id: Optional[str], hf_repo: Optional[str])` -> `Union[HfStatsResult, ErrorResult]` +- `tsfm.count_features()` -> `Union[FeatureCountResult, ErrorResult]` +- `tsfm.describe_features(names: List[str])` -> `Union[DescribeFeaturesResult, ErrorResult]` +- `tsfm.extract_features(dataset_path: str, extractors: List[str], target_columns: List[str], timestamp_column: Optional[str], window: Optional[int])` -> `Union[ExtractResult, ErrorResult]` +- `tsfm.select_features(dataset_path: str, channel: str, extractors: List[str], timestamp_column: Optional[str], reference_feature: str, cd_margin: float)` -> `Union[FeatureSelectionResult, ErrorResult]` +- `tsfm.search_features(text: str, tags: Optional[List[str]], status: Optional[str])` -> `Union[FeaturesResult, ErrorResult]` +- `tsfm.get_feature(feature_id: str)` -> `Union[CardResult, ErrorResult]` +- `tsfm.register_feature(feature: dict, overwrite: bool)` -> `Union[RegisterResult, ErrorResult]` +- `tsfm.update_feature(feature_id: str, fields: dict)` -> `Union[CardResult, ErrorResult]` +- `tsfm.deprecate_feature(feature_id: str, reason: Optional[str])` -> `Union[CardResult, ErrorResult]` +- `tsfm.new_feature_version(feature_id: str, fields: Optional[dict], new_feature_id: Optional[str])` -> `Union[CardResult, ErrorResult]` +- `tsfm.get_feature_lineage(feature_id: str)` -> `Union[LineageResult, ErrorResult]` +- `tsfm.recipe_template()` -> `RecipeTemplateResult` +- `tsfm.run_recipe(dataset_path: str, timestamp_column: str, target_columns: List[str], recipe: dict, asset_id: str, parent_run_id: Optional[str])` -> `Union[RecipeResult, ErrorResult]` +- `tsfm.run_tabular_recipe(dataset_path: str, recipe: dict, label_column: Optional[str], asset_id: str)` -> `Union[TabularResult, ErrorResult]` +- `tsfm.run_plan(plan_spec: dict, asset_id: str, scenario_id: Optional[str])` -> `Union[PlanResult, ErrorResult]` +- `tsfm.evaluate(recipe: dict, configs: List[dict])` -> `Union[EvaluateResult, ErrorResult]` +- `tsfm.get_result(task_type: str, result_id: str)` -> `Union[ResultRecord, ErrorResult]` +- `tsfm.list_results(task_type: str, asset_id: Optional[str], scenario_id: Optional[str])` -> `ResultsListResult` +- `tsfm.get_run(run_id: str)` -> `Union[RunRecord, ErrorResult]` +- `tsfm.list_runs(asset_id: Optional[str])` -> `RunsResult` + +## `vibration` (8 tools) + +- `vibration.get_vibration_data(site_name: str, asset_id: str, sensor_name: str, start: str, final: Optional[str])` -> `Union[dict, ErrorResult]` +- `vibration.list_vibration_sensors(site_name: str, asset_id: str)` -> `Union[dict, ErrorResult]` +- `vibration.compute_fft_spectrum(data_id: str, window: str, top_n: int)` -> `Union[dict, ErrorResult]` +- `vibration.compute_envelope_spectrum(data_id: str, band_low_hz: Optional[float], band_high_hz: Optional[float], top_n: int)` -> `Union[dict, ErrorResult]` +- `vibration.assess_vibration_severity(rms_velocity_mm_s: float, machine_group: str)` -> `dict` +- `vibration.calculate_bearing_frequencies(rpm: float, n_balls: int, ball_diameter_mm: float, pitch_diameter_mm: float, contact_angle_deg: float, bearing_name: str)` -> `dict` +- `vibration.list_known_bearings()` -> `dict` +- `vibration.diagnose_vibration(data_id: str, rpm: Optional[float], bearing_designation: Optional[str], bearing_n_balls: Optional[int], bearing_ball_dia_mm: Optional[float], bearing_pitch_dia_mm: Optional[float], bearing_contact_angle_deg: float, bpfo_hz: Optional[float], bpfi_hz: Optional[float], bsf_hz: Optional[float], ftf_hz: Optional[float], machine_group: str, machine_description: str)` -> `Union[dict, ErrorResult]` + +## `utilities` (6 tools) + +- `utilities.json_reader(file_name: str)` -> `str` +- `utilities.get_sensor_catalog(sensor: Optional[str])` -> `Union[CatalogResult, ErrorResult]` +- `utilities.get_asset_catalog(asset: Optional[str], category: Optional[str])` -> `Union[CatalogResult, ErrorResult]` +- `utilities.get_failure_mode_catalog(failure_mode: Optional[str], category: Optional[str])` -> `Union[CatalogResult, ErrorResult]` +- `utilities.current_date_time()` -> `DateTimeResult` +- `utilities.current_time_english()` -> `TimeEnglishResult` + +## `wo` (15 tools) + +- `wo.list_workorders` +- `wo.get_workorder` +- `wo.get_workorder_tasks` +- `wo.get_workorder_costs` +- `wo.get_workorder_actuals_vs_planned` +- `wo.get_workorder_kpis` +- `wo.get_schedule_calendar` +- `wo.get_my_assigned_workorders` +- `wo.get_failure_codes` +- `wo.generate_work_order` +- `wo.update_workorder` +- `wo.approve_workorder` +- `wo.assign_technician` +- `wo.close_workorder` +- `wo.cancel_workorder` + +## What has no MCP tool and must be done in the code track + + + +The servers retrieve, catalog and run recipes. They do not do the following, so +these belong in the terminal agent's workspace using the library's own scripts: + +- responsible-variable attribution (contribution plots, RBC, SHAP) +- propagation direction (lead-lag, Granger, transfer entropy) +- spectral admissibility (Nyquist, resolution, defect separation) +- bearing defect frequencies from geometry when the bearing is not in the database +- refrigerant-side thermodynamics (superheat, subcooling, approach, cycle COP) +- heat-exchanger UA and fouling attribution +- Weibull and survival fitting, PM interval optimisation, P-F detection probability +- work-order code-quality auditing and crosswalk loss +- RPN lattice auditing and criticality ranking +- alarm rate, flood, chattering and Pareto metrics +- health-indicator suitability screening (monotonicity, trendability, prognosability) diff --git a/skills/repositories/repo-skills/assetopsbench/scripts/check_servers.py b/skills/repositories/repo-skills/assetopsbench/scripts/check_servers.py new file mode 100644 index 00000000..4a95ac7c --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/scripts/check_servers.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Reachability and tool-surface check for the AssetOpsBench MCP servers. + +This is the MCP equivalent of the "minimal import check" that a Python +repository skill would carry. It completes the stdio handshake, calls +``tools/list``, and asserts that the tool names the skill documents are the +tool names the server actually exposes. + +Usage +----- + python check_servers.py # all six servers + python check_servers.py --server iot # one server + python check_servers.py --json # machine-readable result + +Exit codes +---------- + 0 every checked server passed + 1 at least one server failed the handshake or the surface assertion + 2 the MCP client library is unavailable + +Outcome vocabulary matches the DisCo native-check classes: +PASS, SKILL_GAP, NATIVE_FAIL, SKIP_UNSAFE. + + PASS handshake succeeded and the expected tools are all present + SKILL_GAP handshake succeeded but the documented surface disagrees with + the live surface; the skill is stale, not the server + NATIVE_FAIL the server could not be launched or did not complete the + handshake + SKIP_UNSAFE the server was not selected for this run +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys + +# Expected surface, as documented in references/mcp-servers.md. +# Work-order write tools are conditional on AOB_READONLY. +EXPECTED: dict[str, list[str]] = { + "iot": [ + "sites", "asset_ids", "asset_detail", "measured_sensors", + "installed_sensors", "assets", "find_assets_by_sensors", + "stream_extent", "history", "latest_reading", "sensor_coverage", + "sensor_stats", + ], + "fmsr": ["get_failure_modes", "generate_failure_modes", "add_failure_modes"], + "vibration": [ + "get_vibration_data", "list_vibration_sensors", "compute_fft_spectrum", + "compute_envelope_spectrum", "assess_vibration_severity", + "calculate_bearing_frequencies", "list_known_bearings", + "diagnose_vibration", + ], + "utilities": [ + "json_reader", "get_sensor_catalog", "get_asset_catalog", + "get_failure_mode_catalog", "current_date_time", "current_time_english", + ], + "wo": [ + "list_workorders", "get_workorder", "get_workorder_tasks", + "get_workorder_costs", "get_workorder_actuals_vs_planned", + "get_workorder_kpis", "get_schedule_calendar", + "get_my_assigned_workorders", "get_failure_codes", + ], + # tsfm exposes 41 tools; assert a representative spine rather than all of + # them, so a catalog addition upstream does not read as a regression. + "tsfm": [ + "list_tasks", "profile_series", "data_quality", "list_models", + "find_models", "extract_features", "run_recipe", "run_plan", + "list_results", "list_runs", + ], +} + +WO_WRITE = [ + "generate_work_order", "update_workorder", "approve_workorder", + "assign_technician", "close_workorder", "cancel_workorder", +] + +LAUNCH = {name: ["uv", "run", f"{name}-mcp-server"] for name in EXPECTED} + + +async def check_one(name: str, timeout: float) -> dict: + from mcp import ClientSession, StdioServerParameters + from mcp.client.stdio import stdio_client + + expected = list(EXPECTED[name]) + if name == "wo" and os.environ.get("AOB_READONLY") != "1": + expected += WO_WRITE + + params = StdioServerParameters( + command=LAUNCH[name][0], args=LAUNCH[name][1:], env=dict(os.environ) + ) + try: + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await asyncio.wait_for(session.initialize(), timeout=timeout) + listed = await asyncio.wait_for(session.list_tools(), timeout=timeout) + except Exception as exc: # noqa: BLE001 - the failure class is the result + return { + "server": name, + "status": "NATIVE_FAIL", + "error": f"{type(exc).__name__}: {exc}", + "tools_found": 0, + } + + found = sorted(t.name for t in listed.tools) + missing = sorted(set(expected) - set(found)) + status = "PASS" if not missing else "SKILL_GAP" + return { + "server": name, + "status": status, + "tools_found": len(found), + "missing_expected": missing, + "unexpected_extra": sorted(set(found) - set(expected)) if name != "tsfm" else [], + } + + +async def main_async(servers: list[str], timeout: float) -> list[dict]: + return [await check_one(name, timeout) for name in servers] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", action="append", choices=sorted(EXPECTED), + help="check one server; repeatable; default is all six") + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + try: + import mcp # noqa: F401 + except ImportError: + print("mcp client library not importable; install the project first", + file=sys.stderr) + return 2 + + servers = args.server or sorted(EXPECTED) + results = asyncio.run(main_async(servers, args.timeout)) + skipped = [{"server": s, "status": "SKIP_UNSAFE"} + for s in sorted(EXPECTED) if s not in servers] + + if args.json: + print(json.dumps({"results": results + skipped}, indent=2)) + else: + for r in results: + line = f"{r['status']:<12} {r['server']:<10} tools={r['tools_found']}" + if r.get("missing_expected"): + line += f" missing={','.join(r['missing_expected'])}" + if r.get("error"): + line += f" {r['error']}" + print(line) + for r in skipped: + print(f"{r['status']:<12} {r['server']}") + + return 0 if all(r["status"] == "PASS" for r in results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/repositories/repo-skills/assetopsbench/sub-skills/evidence-and-abstention/SKILL.md b/skills/repositories/repo-skills/assetopsbench/sub-skills/evidence-and-abstention/SKILL.md new file mode 100644 index 00000000..c466be82 --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/sub-skills/evidence-and-abstention/SKILL.md @@ -0,0 +1,102 @@ +--- +name: evidence-and-abstention +description: "Decides whether the evidence actually reaches the conclusion about to be + stated, and what to do when it does not. Open this before writing any answer that + carries a number, an identifier, a date or a diagnosis, when a request is + underspecified about which asset, sensor or window it means, when part of an answer + would have to be inferred rather than retrieved, or when a tool returned less than + was asked for and the temptation is to fill the rest in. Abstention is a scored + outcome in this environment, not a failure to answer, and the distinction between + what was retrieved and what was assumed is the thing being measured." +disable-model-invocation: true +license: Apache 2.0 +metadata: + disco-role: operating + capability-family: C12 + asset-class: A0 + leakage-class: ops + library-version: 0.1.0 +--- + +# Evidence, and when to decline + +## The mistake this prevents + +Writing a complete-looking answer in which one element was retrieved and the +rest was reconstructed from what would be reasonable. The reconstructed parts +are indistinguishable from the retrieved parts in the prose, which is exactly +why the execution record is scored and not only the claim. + +The failure has a signature. An answer that names a temperature, a work-order +id, or a date that appears in no tool result is not a small inaccuracy in an +otherwise good answer. It is the specific thing this environment is built to +detect. + +## Preconditions + +- [ ] You can name, for each factual element of the answer, the call that + produced it. +- [ ] Identifiers in the answer were returned by a call, not composed. +- [ ] Any arithmetic was performed, in a tool or in the code workspace, not + estimated. + +If you cannot tick these, the answer is not ready and the fix is another +retrieval or a narrower claim, not better prose. + +## Procedure + +1. **Separate the request into what was asked and what was withheld.** Operator + requests routinely omit the site, the sensor, or the window. That omission is + part of the task: the investigation is yours to do. It is not licence to pick + a plausible default silently. + +2. **Do the retrieval you can, then look at what is left.** Three outcomes, and + they are different answers: + - Everything resolved. State the conclusion and cite the calls. + - The gap is closable by another call. Make it. + - The gap is not closable. Go to step 3. + +3. **When the gap is not closable, choose between asking and declining.** + - **Ask** when one specific missing fact would unblock everything and only the + requester has it: which of three assets they meant, which window matters. + Ask for that one fact, not for a restatement of the request. + - **Decline the specific claim** when the environment cannot supply the + evidence at all: the sensor is not instrumented, the stream does not cover + the window, the server is down. Say which claim you are declining and why, + and give the part of the answer that does hold. + +4. **Never let the shape of the question dictate the shape of the answer.** A + question phrased as "which failure mode is this" invites a named mode. If the + evidence supports a set of two modes and not one, the answer is the set. A + confident single mode drawn from an ambiguous signature is wrong even when it + happens to be right, because the reasoning does not carry. + +5. **Write the answer so the evidence is traceable.** For each claim, the call + that supports it. This is not ceremony; it is what makes the answer checkable + by someone who was not watching, and it is what an evidence-scored evaluation + reads. + +## Interpretation + +| What you have | What the answer is | +| --- | --- | +| Every element retrieved | The conclusion, with its calls | +| The conclusion holds, one supporting detail does not | The conclusion, with the unsupported detail removed rather than softened | +| Two conclusions fit the evidence equally | Both, named as a pair, with the test that would separate them | +| The window or asset is ambiguous and it changes the answer | One question naming the specific ambiguity | +| The evidence does not exist in this environment | An explicit decline for that claim, plus whatever else holds | + +## Failure modes of this skill + +- **Abstention can be overused.** Declining when a further retrieval would have + closed the gap is a failure too, and a lazier one. Exhaust the retrievals + before you decline. +- **It does not tell you whether a result is physically possible.** An + efficiency above one is fully supported by the calls that produced it and + still wrong. Admissibility is a domain judgement and lives elsewhere. + +## Stop conditions + +Stop and report rather than completing the answer if a required identifier never +resolved, if the only remaining route to a number is to assume it, or if the +question cannot be answered without a fact the environment does not hold. diff --git a/skills/repositories/repo-skills/assetopsbench/sub-skills/server-routing/SKILL.md b/skills/repositories/repo-skills/assetopsbench/sub-skills/server-routing/SKILL.md new file mode 100644 index 00000000..9d4ed320 --- /dev/null +++ b/skills/repositories/repo-skills/assetopsbench/sub-skills/server-routing/SKILL.md @@ -0,0 +1,96 @@ +--- +name: server-routing +description: "Chooses the AssetOpsBench MCP server that owns a capability before any + tool is called, and handles the three situations where a naive choice goes wrong: a + capability that looks like it belongs to one server and is served by another, a + server that fails its handshake, and a step that mutates state when the environment + is read-only. Open this when you know what you need but not where it lives, when a + call returns an ErrorResult you did not expect, when you are about to write a work + order or a failure mode, or when a step has no tool at all and belongs in the code + workspace instead." +disable-model-invocation: true +license: Apache 2.0 +metadata: + disco-role: operating + capability-family: C1, C2 + asset-class: A0 + leakage-class: ops + library-version: 0.1.0 +--- + +# Routing to the server that owns the capability + +## The mistake this prevents + +Guessing the server from the word in the request. "Show me the vibration on +P-101" contains the word vibration, and the first call is almost always an `iot` +call, because `vibration` operates on a signal you have not retrieved yet. The +servers are split by **what they hold**, not by what the question is about, and +those come apart constantly. + +## Preconditions + +- [ ] `python scripts/check_servers.py --json` passed, or you know which servers + are down. +- [ ] You know whether `AOB_READONLY=1` is set. +- [ ] You have a site and an asset identifier, or your first call is the one that + resolves them. + +## Procedure + +1. **Resolve identity before capability.** Nothing downstream works on an asset + you have not resolved. `iot.sites()`, then `iot.asset_ids(site_name)`, then + `iot.asset_detail(site_name, asset_id)`. A request naming an asset in prose is + not a resolved identifier; assets have registry ids and prose names are not + guaranteed to match them. + +2. **Route by what is held, using this table.** + + | You need | Server | Note | + | --- | --- | --- | + | What assets and sensors exist | `iot` | `installed_sensors` is the registry, `measured_sensors` is the stream. They disagree more often than you expect, and the disagreement is itself a finding | + | Raw telemetry over a window | `iot` | `history` is paged; `stream_extent` first, so you know what you are asking for | + | Failure modes for an asset class | `fmsr` | `get_failure_modes` reads, `generate_failure_modes` invents. Do not confuse them in an answer | + | Forecast, anomaly, data quality, a run | `tsfm` | The largest server, 41 tools. It owns the analysis lifecycle, not just models | + | A spectrum or envelope | `vibration` | Operates on a signal you supply, so an `iot` retrieval comes first | + | Work-order history or a new order | `wo` | Six of its fifteen tools mutate | + | A catalog or lookup | `utilities` | Reference data, not asset data | + +3. **Check the read half before the write half.** Every write in this + environment has a read that should precede it. Generating a work order + without having read the asset's work-order history produces an order that + duplicates one already open, and nothing in the tool surface will stop you. + +4. **Handle a refusing server as a stop, not a detour.** Every tool returns + `Union[Result, ErrorResult]`. An `ErrorResult` is information: it usually + means the identifier did not resolve. A failed handshake is different and + means the server is not running. Neither is a licence to supply the value + yourself. + +5. **Recognise the steps that have no tool.** Some work has no MCP tool and + belongs in the code workspace: arithmetic across two retrievals, a unit + conversion, a plot, a statistic the server does not compute. Doing it in code + is correct. Asserting it without doing it anywhere is not. + +## Interpretation + +| Situation | What it means | Do this | +| --- | --- | --- | +| `installed_sensors` lists a tag `measured_sensors` does not | The registry claims a sensor the stream never reports | Report the gap. It often explains why a mode is undiagnosable | +| `stream_extent` returns a span shorter than the window asked for | The data does not cover the question | Narrow the claim to the covered span, or say so | +| A tool returns `ErrorResult` on a name from the request | The prose name is not the registry id | Resolve through `iot`, do not retry with variants | +| `AOB_READONLY=1` and the task needs a write | The environment cannot complete the task | Produce the plan and say the write was not performed | + +## Failure modes of this skill + +- **It routes, it does not sequence.** Knowing that `tsfm` owns anomaly + detection does not tell you that a data-quality pass belongs before it. Order + of operations is a workflow concern. +- **Tool counts are pinned to a commit.** Trust `check_servers.py` over this + file when they disagree. + +## Stop conditions + +Stop and report if a server fails its handshake, if an identifier will not +resolve after being looked up through `iot`, or if the remaining path to the +answer requires a value that no call returned and no code step can compute. diff --git a/skills/tools/build_run_manifest.py b/skills/tools/build_run_manifest.py new file mode 100644 index 00000000..b939ffd9 --- /dev/null +++ b/skills/tools/build_run_manifest.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Build a Gate 5 run manifest from AssetOpsBench evaluation output. + +`gate5_counterfactual.py` wants one JSONL line per recorded run. The benchmark +already writes everything it needs, in two places, so nothing has to be +instrumented: `_aggregate.json` under the reports root carries the score and the +operational metrics, and the trajectory JSON carries the record of what the +agent actually opened. This joins them. + + # one arm at a time, appending to the same manifest + python skills/tools/build_run_manifest.py --k-level k0 \\ + --reports-root runs/k0/assetopsbench-reports \\ + --trajectory-root runs/k0/assetopsbench-trajectories \\ + --out runs/manifest.jsonl + + python skills/tools/build_run_manifest.py --k-level k1 \\ + --reports-root runs/k1/assetopsbench-reports \\ + --trajectory-root runs/k1/assetopsbench-trajectories \\ + --out runs/manifest.jsonl --append + + python skills/tools/gate5_counterfactual.py --runs runs/manifest.jsonl --per-graph + +Both roots are the ones passed to `benchmark.scenario_suite_runner` as +`--reports-root` and `--trajectory-root`, and both nest as +`///`. Every arm must be written to a **separate** +pair of roots, because the file names carry the scenario id and nothing else: run +k0 and k1 into the same directory and the second overwrites the first. + +The K level is supplied here rather than read from the output, because nothing +in the benchmark's own records it. That is the one place this join can go wrong +and it is worth an explicit check: pass `--expect-skills` on a k1 arm and +`--expect-no-skills` on k0, and the builder will fail if the trajectories +disagree with the label. A mislabelled arm produces a clean-looking manifest and +a meaningless result, so it is worth ten seconds. + +Exit codes: 0 written, 1 a check failed or nothing was found, 2 bad invocation. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys + +AGGREGATE = "_aggregate.json" +# A path into the mounted collection, as it appears in a code-exec argument. +CONSULT_RE = re.compile(r"repo-skills(?:-router)?/") + + +def find_aggregates(root: pathlib.Path) -> list[pathlib.Path]: + return sorted(root.rglob(AGGREGATE)) + + +def load_results(path: pathlib.Path) -> list[dict]: + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"{path}: {type(exc).__name__}: {exc}") + results = doc.get("results") + if not isinstance(results, list): + raise SystemExit(f"{path}: no `results` list; is this an EvalReport?") + return results + + +def trajectory_for(traj_root: pathlib.Path, runner: str, + scenario_id: str) -> pathlib.Path | None: + """The suite runner writes `_.json` under `//`. + + Matched on the file name rather than on the model directory, so a manifest + can still be built when the reports and the trajectories were written under + slightly different model slugs. + """ + exact = list(traj_root.rglob(f"{runner}_{scenario_id}.json")) + if exact: + return exact[0] + loose = list(traj_root.rglob(f"*_{scenario_id}.json")) + return loose[0] if len(loose) == 1 else None + + +def consulted(path: pathlib.Path | None) -> bool: + if path is None or not path.exists(): + return False + return bool(CONSULT_RE.search(path.read_text(encoding="utf-8", errors="replace"))) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--reports-root", type=pathlib.Path, required=True) + ap.add_argument("--trajectory-root", type=pathlib.Path, required=True) + ap.add_argument("--k-level", required=True, choices=("k0", "k1", "k1-recovery")) + ap.add_argument("--out", type=pathlib.Path, required=True) + ap.add_argument("--append", action="store_true", + help="append to --out instead of replacing it") + ap.add_argument("--repetition", type=int, default=0, + help="repetition index, when the same arm is run more than once") + ap.add_argument("--asset-class-map", type=pathlib.Path, + help="optional JSON mapping scenario_id to asset class, which " + "enables the per-asset-class breakdown in gate 5") + ap.add_argument("--expect-skills", action="store_true", + help="fail if any trajectory does NOT reference the collection") + ap.add_argument("--expect-no-skills", action="store_true", + help="fail if ANY trajectory references the collection; use on k0") + a = ap.parse_args() + + if a.expect_skills and a.expect_no_skills: + ap.error("--expect-skills and --expect-no-skills are mutually exclusive") + for p in (a.reports_root, a.trajectory_root): + if not p.is_dir(): + print(f"not a directory: {p}", file=sys.stderr) + return 2 + + classes = {} + if a.asset_class_map: + classes = json.loads(a.asset_class_map.read_text(encoding="utf-8")) + + aggregates = find_aggregates(a.reports_root) + if not aggregates: + print(f"no {AGGREGATE} under {a.reports_root}", file=sys.stderr) + return 1 + + lines, missing_traj, with_skills, without_skills = [], [], [], [] + for agg in aggregates: + for r in load_results(agg): + sid = str(r.get("scenario_id", "")).strip() + if not sid: + continue + runner = str(r.get("runner", "")).strip() or "stirrup_agent" + score = r.get("score") or {} + ops = r.get("ops") or {} + traj = trajectory_for(a.trajectory_root, runner, sid) + if traj is None: + missing_traj.append(sid) + (with_skills if consulted(traj) else without_skills).append(sid) + + rec = { + "task_id": sid, + "k_level": a.k_level, + "score": float(score.get("score", 0.0)), + "repetition": a.repetition, + "passed": bool(score.get("passed", False)), + "model": r.get("model", ""), + "tokens": int(ops.get("tokens_in", 0)) + int(ops.get("tokens_out", 0)), + "steps": int(ops.get("turn_count", 0)), + "tool_calls": int(ops.get("tool_call_count", 0)), + } + if sid in classes: + rec["asset_class"] = classes[sid] + if traj is not None: + rec["trajectory"] = str(traj.resolve()) + lines.append(json.dumps(rec)) + + mode = "a" if a.append else "w" + a.out.parent.mkdir(parents=True, exist_ok=True) + with a.out.open(mode, encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + print(f"{len(lines)} run(s) written to {a.out} " + f"({'appended' if a.append else 'replaced'})") + print(f" arm {a.k_level}, repetition {a.repetition}") + print(f" aggregates read {len(aggregates)}") + print(f" consulted skills {len(with_skills)}") + print(f" consulted nothing {len(without_skills)}") + if missing_traj: + print(f" WARN no trajectory for {len(missing_traj)} run(s), so those rows " + f"cannot carry per-graph attribution: {missing_traj[:8]}") + + failed = False + if a.expect_no_skills and with_skills: + print(f"\nFAIL labelled {a.k_level} but {len(with_skills)} trajectory(ies) " + f"reference the skill collection: {with_skills[:8]}") + print(" the baseline is contaminated, or the arms were mislabelled") + failed = True + if a.expect_skills and without_skills: + print(f"\nFAIL labelled {a.k_level} but {len(without_skills)} trajectory(ies) " + f"reference no skill at all: {without_skills[:8]}") + print(" the mount did not reach the agent, or it chose never to look. " + "Check one workspace for a skills/ directory before trusting the arm") + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/tools/gate5_counterfactual.py b/skills/tools/gate5_counterfactual.py new file mode 100644 index 00000000..2e832e45 --- /dev/null +++ b/skills/tools/gate5_counterfactual.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Gate 5: counterfactual utility of the skill library, measured per graph. + +Gates 1 to 4 ask whether a skill is well-formed, self-contained, physically +admissible and free of benchmark answers. None of them asks whether it helps. +This is the gate that does, and it is the only one whose input is runs rather +than files. + + # 1. record runs at both K levels, then + python skills/tools/gate5_counterfactual.py --runs runs.jsonl + + # 2. attribute the delta to the graphs the agent actually opened + python skills/tools/gate5_counterfactual.py --runs runs.jsonl --per-graph + + # 3. stamp the result into a machine-readable admission record + python skills/tools/gate5_counterfactual.py --runs runs.jsonl --per-graph \ + --emit gate5-admission.json + + python skills/tools/gate5_counterfactual.py --self-test + +The design, and why each piece is there: + +**Paired, task by task.** Scenario difficulty varies far more than the library +effect does, so an unpaired comparison of two group means measures the task mix. +The unit of analysis is `s(t) = score_K1(t) - score_K0(t)` for the same task, and +the resampling unit is the task, not the run. + +**Regressions counted separately, never netted.** A library that helps on average +while poisoning one asset class is worse than no library. Mean `s` cannot show +that and is not asked to. + +**A clean-baseline check on the recorded runs, not on the code.** `preflight.py` +proves the k0 code path mounts nothing. This proves the k0 runs that were +actually scored consulted nothing, by reading their trajectories. A contaminated +baseline invalidates every number below it, so it is a hard failure and it is +checked first. + +**Power reported alongside every null.** With one suite and many graphs, most +graphs are consulted on a handful of tasks. "No effect detected" and "not enough +runs to detect one" are different findings and are reported as different +verdicts. A graph consulted on too few tasks is `INSUFFICIENT_POWER`, never +`NEUTRAL`. + +**Multiplicity controlled.** Per-graph p-values are corrected across the graphs +actually tested, by Benjamini-Hochberg. Testing 38 graphs at alpha 0.05 and +reporting the two that cleared it is how a library gets admitted on noise. + +Input is a run manifest, JSONL, one recorded run per line: + + {"task_id": "s-014", "k_level": "k0", "score": 0.0, "repetition": 0, + "tokens": 18422, "steps": 11, "tool_calls": 7, + "asset_class": "chiller-hvac", "trajectory": "runs/s-014-k0-0.jsonl"} + +`task_id`, `k_level` and `score` are required. Everything else is optional and +enables a further section of the report. `trajectory` may be a path to a +recorded trajectory (JSON, JSONL or transcript text) or the transcript inline +under `transcript`; it is read only to discover which `SKILL.md` files were +opened, which is what makes per-graph attribution possible. + +Exit codes: 0 the library is admitted at the collection level (or the self-test +passed), 1 a hard failure, a regression breach, or a collection-level null, +2 bad invocation. +""" + +from __future__ import annotations + +import argparse +import json +import math +import pathlib +import random +import re +import statistics +import sys +from collections import defaultdict + +K_BASELINE = "k0" +K_TREATMENT = "k1" +K_RECOVERY = "k1-recovery" +KNOWN_K = {K_BASELINE, K_TREATMENT, K_RECOVERY} + +# A path into the mounted collection, as it appears in a shell command, a file +# read or a transcript line. Both mount layouts are matched, and so is a bare +# relative reference, because the agent's own cwd varies. +CONSULT_RE = re.compile( + r"repo-skills/([a-z0-9][a-z0-9-]{0,63})" + r"(?:/sub-skills/([a-z0-9][a-z0-9-]{0,63}))?" +) +ROUTER_RE = re.compile(r"repo-skills-router") + +# 80 percent power, two-sided alpha 0.05: z(0.975) + z(0.80). +Z_MDE = 1.959964 + 0.841621 +MIN_TASKS_FOR_A_VERDICT = 8 + + +# -------------------------------------------------------------------------- +# statistics, stdlib only so the harness runs wherever the benchmark runs +# -------------------------------------------------------------------------- + +def mean(xs: list[float]) -> float: + return sum(xs) / len(xs) if xs else float("nan") + + +def bootstrap_ci(deltas: list[float], reps: int = 10000, alpha: float = 0.05, + seed: int = 20260905) -> tuple[float, float]: + """Percentile bootstrap over the paired per-task deltas. + + The resampling unit is the task. Resampling runs instead would treat two + repetitions of one scenario as two independent observations, which they are + not, and would report a confidence interval that is too narrow. + """ + if len(deltas) < 2: + return (float("nan"), float("nan")) + rng = random.Random(seed) + n = len(deltas) + means = [] + for _ in range(reps): + means.append(sum(deltas[rng.randrange(n)] for _ in range(n)) / n) + means.sort() + lo = means[int(math.floor((alpha / 2) * reps))] + hi = means[min(reps - 1, int(math.ceil((1 - alpha / 2) * reps)) - 1)] + return (lo, hi) + + +def sign_test_p(deltas: list[float]) -> float: + """Exact two-sided sign test. Zero deltas are dropped, which is the + conservative convention: a task the library did not change is not evidence + that it helped.""" + pos = sum(1 for d in deltas if d > 0) + neg = sum(1 for d in deltas if d < 0) + n = pos + neg + if n == 0: + return 1.0 + k = min(pos, neg) + tail = sum(math.comb(n, i) for i in range(0, k + 1)) / (2 ** n) + return min(1.0, 2 * tail) + + +def _midranks(xs: list[float]) -> list[float]: + order = sorted(range(len(xs)), key=lambda i: xs[i]) + ranks = [0.0] * len(xs) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and xs[order[j + 1]] == xs[order[i]]: + j += 1 + r = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = r + i = j + 1 + return ranks + + +def spearman(xs: list[float], ys: list[float]) -> float: + """Spearman rho with midranks, so ties do not inflate it.""" + if len(xs) < 3: + return float("nan") + rx, ry = _midranks(xs), _midranks(ys) + mx, my = mean(rx), mean(ry) + num = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) + dx = math.sqrt(sum((a - mx) ** 2 for a in rx)) + dy = math.sqrt(sum((b - my) ** 2 for b in ry)) + return num / (dx * dy) if dx and dy else float("nan") + + +def benjamini_hochberg(pvals: dict[str, float], q: float = 0.05) -> set[str]: + """Return the keys rejected at false-discovery rate q.""" + if not pvals: + return set() + items = sorted(pvals.items(), key=lambda kv: kv[1]) + m = len(items) + cut = 0 + for i, (_, p) in enumerate(items, start=1): + if p <= q * i / m: + cut = i + return {k for k, _ in items[:cut]} + + +def mde(deltas: list[float]) -> float: + """Minimum effect this many paired tasks could detect at 80 percent power. + Reported next to every null so a null is readable.""" + if len(deltas) < 2: + return float("nan") + sd = statistics.stdev(deltas) + return Z_MDE * sd / math.sqrt(len(deltas)) + + +# -------------------------------------------------------------------------- +# reading the manifest +# -------------------------------------------------------------------------- + +def load_runs(path: pathlib.Path) -> list[dict]: + runs = [] + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = line.strip() + if not line or line.startswith("#"): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError as exc: + raise SystemExit(f"{path}:{lineno}: not JSON: {exc}") + for field in ("task_id", "k_level", "score"): + if field not in rec: + raise SystemExit(f"{path}:{lineno}: missing required field `{field}`") + if rec["k_level"] not in KNOWN_K: + raise SystemExit(f"{path}:{lineno}: unknown k_level `{rec['k_level']}`; " + f"expected one of {sorted(KNOWN_K)}") + try: + rec["score"] = float(rec["score"]) + except (TypeError, ValueError): + raise SystemExit(f"{path}:{lineno}: score is not numeric") + runs.append(rec) + if not runs: + raise SystemExit(f"{path}: no runs") + return runs + + +def consulted_graphs(rec: dict, base: pathlib.Path | None) -> set[str]: + """Which skill graphs this run opened, read from its trajectory. + + Attribution is by what the agent actually read, not by what the router + might have offered it. A graph nobody opened is untested, and saying so is + the point of the `UNTESTED` verdict. + """ + text = rec.get("transcript") + if text is None: + traj = rec.get("trajectory") + if not traj: + return set() + p = pathlib.Path(traj) + if not p.is_absolute() and base is not None: + p = base / p + if not p.exists(): + return set() + text = p.read_text(encoding="utf-8", errors="replace") + found = set() + for graph, sub in CONSULT_RE.findall(text): + if graph in {"repo-skills", "sub-skills"}: + continue + found.add(graph) + if sub: + found.add(f"{graph}/{sub}") + if ROUTER_RE.search(text): + found.add("repo-skills-router") + return found + + +# -------------------------------------------------------------------------- +# the gate +# -------------------------------------------------------------------------- + +def pair_runs(runs: list[dict], treatment: str) -> tuple[dict, list[str]]: + """Collapse repetitions to a per-task mean at each K level, then pair. + + A task present at only one level cannot contribute a delta and is reported + rather than dropped silently, because a systematically missing arm is the + most common way a paired comparison goes wrong. + """ + by = defaultdict(lambda: defaultdict(list)) + for r in runs: + by[r["task_id"]][r["k_level"]].append(r) + paired, unpaired = {}, [] + for task, levels in by.items(): + if K_BASELINE in levels and treatment in levels: + paired[task] = { + K_BASELINE: levels[K_BASELINE], + treatment: levels[treatment], + } + else: + have = sorted(levels) + unpaired.append(f"{task}: only {have}") + return paired, sorted(unpaired) + + +def analyse(runs: list[dict], treatment: str, base: pathlib.Path | None, + per_graph: bool, regression_budget: float, + reps: int) -> dict: + out: dict = {"treatment_arm": treatment, "hard_failures": [], + "warnings": []} + + # Hard check first: the baseline must be clean in the recorded runs, not + # only in the code path. Everything downstream is void if it is not. + contaminated = [] + for r in runs: + if r["k_level"] != K_BASELINE: + continue + if consulted_graphs(r, base): + contaminated.append(r["task_id"]) + if contaminated: + out["hard_failures"].append({ + "code": "CONTAMINATED_BASELINE", + "detail": f"{len(contaminated)} k0 run(s) reference the skill " + f"collection; the baseline is not unaided", + "tasks": sorted(set(contaminated))[:20], + }) + + paired, unpaired = pair_runs(runs, treatment) + out["tasks_paired"] = len(paired) + out["tasks_unpaired"] = unpaired + if len(paired) < 2: + out["hard_failures"].append({ + "code": "NO_PAIRED_TASKS", + "detail": "fewer than two tasks have runs at both K levels", + }) + return out + + deltas, meta = {}, {} + for task, levels in paired.items(): + s0 = mean([r["score"] for r in levels[K_BASELINE]]) + s1 = mean([r["score"] for r in levels[treatment]]) + deltas[task] = s1 - s0 + meta[task] = { + "asset_class": levels[treatment][0].get("asset_class"), + "d_tokens": _delta_field(levels, treatment, "tokens"), + "d_steps": _delta_field(levels, treatment, "steps"), + "d_tool_calls": _delta_field(levels, treatment, "tool_calls"), + "consulted": sorted(set().union(*[consulted_graphs(r, base) + for r in levels[treatment]])), + } + + d = list(deltas.values()) + lo, hi = bootstrap_ci(d, reps=reps) + p = sign_test_p(d) + regressions = {t: v for t, v in deltas.items() if v < 0} + improvements = {t: v for t, v in deltas.items() if v > 0} + out["headline"] = { + "mean_s": mean(d), + "median_s": statistics.median(d), + "ci95": [lo, hi], + "sign_test_p": p, + "n_tasks": len(d), + "improved": len(improvements), + "unchanged": len(d) - len(improvements) - len(regressions), + "regressed": len(regressions), + "regression_rate": len(regressions) / len(d), + "mde_at_80_power": mde(d), + } + out["worst_regressions"] = sorted(regressions.items(), key=lambda kv: kv[1])[:10] + + # Regressions are a budget, not a footnote. + if len(regressions) / len(d) > regression_budget: + out["hard_failures"].append({ + "code": "REGRESSION_BUDGET_EXCEEDED", + "detail": f"{len(regressions)}/{len(d)} tasks regressed " + f"({len(regressions)/len(d):.1%}), budget is " + f"{regression_budget:.1%}", + }) + + # Per asset class, because a mean can hide a class the library poisons. + by_class = defaultdict(list) + for t, v in deltas.items(): + if meta[t]["asset_class"]: + by_class[meta[t]["asset_class"]].append(v) + out["by_asset_class"] = { + k: {"n": len(v), "mean_s": mean(v), + "regressed": sum(1 for x in v if x < 0)} + for k, v in sorted(by_class.items()) + } + + # The compute confound. If s tracks extra tokens, the finding is "we spent + # more", and a reviewer will say so before we do. + conf = {} + for field in ("d_tokens", "d_steps", "d_tool_calls"): + xs = [(deltas[t], meta[t][field]) for t in deltas + if meta[t][field] is not None] + if len(xs) >= 3: + conf[field] = { + "spearman_rho": spearman([a for a, _ in xs], [b for _, b in xs]), + "n": len(xs), + "mean_delta": mean([b for _, b in xs]), + } + out["compute_confound"] = conf + + if per_graph: + out["per_graph"] = _per_graph(deltas, meta, reps) + + out["verdict"] = _verdict(out) + # The gate passes only when the effect is admitted AND nothing else failed. + # Keeping the two apart means a library that helps on average but breaches + # the regression budget reads as what it is, rather than as a null. + out["gate"] = "PASS" if (out["verdict"] == "ADMITTED" + and not out["hard_failures"]) else "FAIL" + return out + + +def _delta_field(levels: dict, treatment: str, field: str): + a = [r[field] for r in levels[K_BASELINE] if isinstance(r.get(field), (int, float))] + b = [r[field] for r in levels[treatment] if isinstance(r.get(field), (int, float))] + if not a or not b: + return None + return mean(b) - mean(a) + + +def _per_graph(deltas: dict, meta: dict, reps: int) -> dict: + """Admission per graph, restricted to the tasks where it was opened. + + This is the part that makes Gate 5 a gate rather than a headline. A library + can post a positive mean while a third of its graphs do nothing, and only + per-graph attribution shows which third. + """ + tasks_by_graph = defaultdict(list) + for t in deltas: + for g in meta[t]["consulted"]: + if "/" in g or g == "repo-skills-router": + continue # graph level only; sub-skills roll up + tasks_by_graph[g].append(t) + + rows, pvals = {}, {} + for g, ts in sorted(tasks_by_graph.items()): + d = [deltas[t] for t in ts] + row = { + "n_tasks": len(d), + "mean_s": mean(d), + "regressed": sum(1 for x in d if x < 0), + "mde_at_80_power": mde(d), + } + if len(d) >= MIN_TASKS_FOR_A_VERDICT: + row["ci95"] = list(bootstrap_ci(d, reps=reps)) + row["sign_test_p"] = sign_test_p(d) + pvals[g] = row["sign_test_p"] + rows[g] = row + + rejected = benjamini_hochberg(pvals) + for g, row in rows.items(): + if row["n_tasks"] < MIN_TASKS_FOR_A_VERDICT: + row["verdict"] = "INSUFFICIENT_POWER" + elif row["mean_s"] < 0 and g in rejected: + row["verdict"] = "REGRESSION" + elif row["mean_s"] > 0 and g in rejected: + row["verdict"] = "ADMITTED" + else: + row["verdict"] = "NEUTRAL" + row["bh_rejected"] = g in rejected + return rows + + +#: Failures that make the comparison meaningless rather than merely negative. +#: A contaminated baseline is not a bad result, it is no result. A regression +#: breach is a real result and a gate failure, so it must not be allowed to +#: overwrite the effect verdict with `VOID`. +INVALIDATING = {"CONTAMINATED_BASELINE", "NO_PAIRED_TASKS"} + + +def _verdict(out: dict) -> str: + if any(f["code"] in INVALIDATING for f in out["hard_failures"]): + return "VOID" + if "headline" not in out: + return "VOID" + h = out["headline"] + if h["ci95"][0] > 0: + return "ADMITTED" + if h["ci95"][1] < 0: + return "HARMFUL" + if h["mde_at_80_power"] > abs(h["mean_s"]) * 2 and h["n_tasks"] < 40: + return "UNDERPOWERED" + return "NOT_SHOWN_TO_HELP" + + +# -------------------------------------------------------------------------- +# reporting +# -------------------------------------------------------------------------- + +def render(out: dict, untested: list[str]) -> None: + for f in out["hard_failures"]: + print(f"FAIL {f['code']}: {f['detail']}") + if f.get("tasks"): + print(f" tasks: {', '.join(f['tasks'])}") + if out["hard_failures"]: + print() + if "headline" not in out: + return + + h = out["headline"] + print(f"Arm: {K_BASELINE} versus {out['treatment_arm']}, " + f"{h['n_tasks']} paired tasks") + if out["tasks_unpaired"]: + print(f" {len(out['tasks_unpaired'])} task(s) had only one arm and " + f"were excluded") + print() + print(f" mean s {h['mean_s']:+.4f}") + print(f" 95% CI [{h['ci95'][0]:+.4f}, {h['ci95'][1]:+.4f}]") + print(f" median s {h['median_s']:+.4f}") + print(f" sign test p {h['sign_test_p']:.4f}") + print(f" detectable at 80% {h['mde_at_80_power']:.4f}") + print(f" improved {h['improved']}") + print(f" unchanged {h['unchanged']}") + print(f" regressed {h['regressed']} ({h['regression_rate']:.1%})") + if out["worst_regressions"]: + print() + print(" worst regressions") + for t, v in out["worst_regressions"]: + print(f" {t:<28}{v:+.4f}") + + if out.get("by_asset_class"): + print() + print(" by asset class") + for k, v in out["by_asset_class"].items(): + print(f" {k:<28}n={v['n']:<4}mean {v['mean_s']:+.4f} " + f"regressed {v['regressed']}") + + if out.get("compute_confound"): + print() + print(" compute confound (Spearman of s against extra compute)") + for k, v in out["compute_confound"].items(): + rho = v["spearman_rho"] + shown = "no variance" if rho != rho else f"{rho:+.3f}" + print(f" {k:<28}rho {shown:<12}n={v['n']} " + f"mean delta {v['mean_delta']:+.1f}") + + if out.get("per_graph"): + print() + print(" per graph, on the tasks where the graph was actually opened") + print(f" {'graph':<44}{'n':>4} {'mean s':>8} {'verdict':<20}") + for g, row in sorted(out["per_graph"].items(), + key=lambda kv: -kv[1]["mean_s"]): + print(f" {g:<44}{row['n_tasks']:>4} {row['mean_s']:>+8.4f} " + f"{row['verdict']:<20}") + if untested: + print() + print(f" {len(untested)} graph(s) never opened by any run: " + f"UNTESTED") + for g in untested[:12]: + print(f" {g}") + if len(untested) > 12: + print(f" ... and {len(untested) - 12} more") + + print() + print(f"VERDICT: {out['verdict']} GATE 5: {out.get('gate', 'FAIL')}") + if VERDICT_MEANING.get(out["verdict"]): + print(f" {VERDICT_MEANING[out['verdict']]}") + + +VERDICT_MEANING = { + "VOID": "a hard failure invalidates the comparison", + "ADMITTED": "the confidence interval on mean s excludes zero from above", + "HARMFUL": "the confidence interval excludes zero from below", + "UNDERPOWERED": "the effect this many tasks could detect is larger than the " + "effect observed; run more tasks before concluding anything", + "NOT_SHOWN_TO_HELP": "the interval spans zero at adequate power", +} + + +# -------------------------------------------------------------------------- +# self-test +# -------------------------------------------------------------------------- + +def _synth(tmp: pathlib.Path, effect: float, n: int = 60, seed: int = 7, + contaminate: bool = False, poison_class: str | None = None) -> pathlib.Path: + """Build a manifest with a known planted effect, so the harness can be + checked against an answer it cannot see.""" + rng = random.Random(seed) + classes = ["chiller-hvac", "pumps", "compressors", "bearings-gearboxes"] + graphs = ["assetops-domain", "rca-and-responsible-variable", + "compressor-diagnosis", "pint-units-for-assets"] + lines = [] + for i in range(n): + task = f"s-{i:03d}" + cls = classes[i % len(classes)] + base = rng.uniform(0.1, 0.8) + eff = effect + if poison_class and cls == poison_class: + eff = -abs(effect) * 2 + lines.append(json.dumps({ + "task_id": task, "k_level": "k0", "score": round(base, 4), + "tokens": 15000 + rng.randint(-2000, 2000), "steps": 10, + "tool_calls": 6, "asset_class": cls, + "transcript": "opened nothing" if not contaminate + else "cat repo-skills/assetops-domain/SKILL.md", + })) + lines.append(json.dumps({ + "task_id": task, "k_level": "k1", + "score": round(min(1.0, max(0.0, base + eff + rng.gauss(0, 0.05))), 4), + "tokens": 17000 + rng.randint(-2000, 2000), "steps": 12, + "tool_calls": 8, "asset_class": cls, + "transcript": f"cat repo-skills/{graphs[i % len(graphs)]}/SKILL.md", + })) + p = tmp / f"runs-{effect}-{seed}-{contaminate}-{poison_class}.jsonl" + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + return p + + +def self_test() -> int: + import tempfile + fails = [] + with tempfile.TemporaryDirectory() as td: + tmp = pathlib.Path(td) + + # 1. a real effect is recovered and admitted + r = analyse(load_runs(_synth(tmp, 0.12)), K_TREATMENT, tmp, True, 0.35, 2000) + if r["verdict"] != "ADMITTED": + fails.append(f"planted +0.12 gave {r['verdict']}, expected ADMITTED") + if not (0.08 < r["headline"]["mean_s"] < 0.16): + fails.append(f"planted +0.12 recovered as {r['headline']['mean_s']:.4f}") + + # 2. a null library is not admitted + r = analyse(load_runs(_synth(tmp, 0.0, seed=11)), K_TREATMENT, tmp, True, 0.6, 2000) + if r["verdict"] == "ADMITTED": + fails.append("a null effect was admitted; the gate passes anything") + + # 3. a harmful library is caught + # The budget is set to 1.0 so the effect verdict is isolated: a harmful + # library also breaches any sane regression budget, and the point of + # this case is that the effect itself is named. + r = analyse(load_runs(_synth(tmp, -0.15, seed=13)), K_TREATMENT, tmp, True, 1.0, 2000) + if r["verdict"] != "HARMFUL": + fails.append(f"planted -0.15 gave {r['verdict']}, expected HARMFUL") + if r["gate"] != "FAIL": + fails.append("a harmful library passed the gate") + + # 4. a contaminated baseline voids the run + r = analyse(load_runs(_synth(tmp, 0.12, seed=17, contaminate=True)), + K_TREATMENT, tmp, True, 0.35, 500) + if r["verdict"] != "VOID": + fails.append(f"contaminated baseline gave {r['verdict']}, expected VOID") + if not any(f["code"] == "CONTAMINATED_BASELINE" for f in r["hard_failures"]): + fails.append("contamination was not named as the failure") + + # 5. a class the library poisons is visible even when the mean is up + r = analyse(load_runs(_synth(tmp, 0.20, seed=19, poison_class="pumps")), + K_TREATMENT, tmp, True, 0.9, 2000) + if r["headline"]["mean_s"] <= 0: + fails.append("poisoned-class fixture did not produce a positive mean") + if r["by_asset_class"].get("pumps", {}).get("mean_s", 0) >= 0: + fails.append("the poisoned class did not show a negative class mean") + + # 6. the regression budget bites + r = analyse(load_runs(_synth(tmp, 0.20, seed=19, poison_class="pumps")), + K_TREATMENT, tmp, True, 0.10, 500) + if not any(f["code"] == "REGRESSION_BUDGET_EXCEEDED" for f in r["hard_failures"]): + fails.append("a 25% regression rate did not breach a 10% budget") + + # 7. small n is called underpowered, not neutral + r = analyse(load_runs(_synth(tmp, 0.01, n=10, seed=23)), K_TREATMENT, + tmp, True, 0.9, 2000) + for g, row in r.get("per_graph", {}).items(): + if row["n_tasks"] < MIN_TASKS_FOR_A_VERDICT and row["verdict"] != "INSUFFICIENT_POWER": + fails.append(f"{g} with n={row['n_tasks']} was called {row['verdict']}") + + # 8. the statistics themselves + if abs(spearman([1, 2, 3, 4, 5], [5, 4, 3, 2, 1]) + 1.0) > 1e-9: + fails.append("spearman of a perfect inversion is not -1") + if abs(sign_test_p([1, 1, 1, 1, 1]) - 2 / 32) > 1e-12: + fails.append("exact sign test disagrees with 2/2^5") + if benjamini_hochberg({"a": 0.001, "b": 0.9, "c": 0.8}) != {"a"}: + fails.append("BH rejected the wrong set") + # One graph at p=0.04 among nineteen nulls must NOT survive: nominal + # significance on one of twenty tests is exactly what multiplicity + # control exists to refuse. (Twenty graphs all at 0.04 is a different + # situation and BH does reject them, correctly.) + lonely = {"hit": 0.04} + lonely.update({f"g{i}": 0.9 for i in range(19)}) + if benjamini_hochberg(lonely) != set(): + fails.append("BH admitted one nominal hit among nineteen nulls") + if benjamini_hochberg({f"g{i}": 0.04 for i in range(20)}) != {f"g{i}" for i in range(20)}: + fails.append("BH failed to reject twenty consistent hits") + + for f in fails: + print(f"SELF-TEST FAIL {f}") + if not fails: + print("self-test passed: 8 checks, planted effects recovered, " + "null and contaminated fixtures correctly refused") + return 1 if fails else 0 + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--runs", type=pathlib.Path, + help="JSONL run manifest, one recorded run per line") + ap.add_argument("--root", type=pathlib.Path, + default=pathlib.Path("skills/repositories"), + help="collection root, used to list graphs never opened") + ap.add_argument("--base", type=pathlib.Path, + help="directory that relative `trajectory` paths are relative to " + "(default: the manifest's own directory)") + ap.add_argument("--arm", default=K_TREATMENT, choices=sorted(KNOWN_K - {K_BASELINE}), + help="which treatment arm to compare against k0") + ap.add_argument("--per-graph", action="store_true", + help="attribute the delta to the graphs each run opened") + ap.add_argument("--regression-budget", type=float, default=0.15, + help="fraction of tasks allowed to regress before the gate fails") + ap.add_argument("--bootstrap", type=int, default=10000) + ap.add_argument("--emit", type=pathlib.Path, + help="write the machine-readable admission record here") + ap.add_argument("--json", action="store_true") + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args() + + if a.self_test: + return self_test() + if not a.runs: + ap.error("--runs is required unless --self-test is given") + if not a.runs.exists(): + print(f"not found: {a.runs}", file=sys.stderr) + return 2 + + base = a.base or a.runs.parent + runs = load_runs(a.runs) + out = analyse(runs, a.arm, base, a.per_graph, a.regression_budget, a.bootstrap) + + untested: list[str] = [] + if a.per_graph and (a.root / "repo-skills").is_dir(): + present = {p.name for p in (a.root / "repo-skills").iterdir() + if p.is_dir() and (p / "SKILL.md").exists()} + untested = sorted(present - set(out.get("per_graph", {}))) + out["untested_graphs"] = untested + + out["verdict_meaning"] = VERDICT_MEANING.get(out["verdict"], "") + if a.json: + print(json.dumps(out, indent=2, default=str)) + else: + render(out, untested) + + if a.emit: + a.emit.write_text(json.dumps(out, indent=2, default=str), encoding="utf-8") + print(f"\nadmission record written to {a.emit}") + + return 0 if out["verdict"] == "ADMITTED" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/tools/validate_skills.py b/skills/tools/validate_skills.py new file mode 100644 index 00000000..4ad8e35f --- /dev/null +++ b/skills/tools/validate_skills.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Static and leakage gates for an AssetOpsBench skill library. + +Run this against any library before mounting it, whether it is the reference +library in this repository or one you built yourself. It checks the contract in +`skills/CONTRACT.md`: frontmatter, per-graph licence consistency, +self-containment, and the industrial axes. + +Gate 3 is the one specific to a benchmark. A skill library sits closer to the +answers than anything else an agent reads, so a `leakage-class: solution` skill +fails outright and any eight-word sequence shared with the answer set is a +failure that names the scenario it came from. + + python skills/tools/validate_skills.py --root skills/repositories + +The answer set is not in the repository: `benchmarks/scenario_suite/*.yaml` hold +scenario ids only, so an audit pointed at the checkout proves nothing. Point it +at where the answers actually live, by any of three routes: + + # a file the evaluation harness exported + ... --answers /path/to/scenarios_with_answers.jsonl + + # a directory of them, walked recursively + ... --answers-dir /path/to/exported_answers/ + + # the published dataset, every config and split by default + ... --answers-hf ibm-research/AssetOpsBench + +Exit codes: 0 all gates pass, 1 a gate failed, 2 bad invocation. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys + +REQUIRED_FIELDS = ("name", "description", "license", "metadata") +# Industrial extension to the AREX contract. Domain skills carry the index axes +# and the leakage class; tool-surface skills do not need the asset axis. +CAPABILITY_FAMILIES = {f"C{i}" for i in range(1, 13)} +ASSET_CLASSES = { + "A0", "chiller-hvac", "ahu", "pumps", "motors-drives", "fans-blowers", + "compressors", "bearings-gearboxes", "wind-turbine", "transformers-electrical", +} +LEAKAGE_CLASSES = {"ops", "solution"} +LEAK_PATTERNS = [ + (re.compile(r"/home/[a-z0-9_.-]+/", re.I), "absolute home path"), + (re.compile(r"/Users/[a-z0-9_.-]+/", re.I), "absolute macOS home path"), + (re.compile(r"site-packages"), "installed-package path"), + (re.compile(r"conda activate|micromamba activate|source .*/bin/activate"), "environment activation"), + (re.compile(r"\.disco/agent"), "DisCo managed path"), +] +FORBIDDEN_EVIDENCE = [ + "benchmarks/scenario_suite", + "src/evaluation/scorers", + "src/scenarios/", +] +#: Populated in main() from whichever answer source was given; None means no +#: source was supplied, which is a warning rather than a pass. +_ANSWER_BLOBS: list[tuple[str, str]] | None = None +DEBRIS = ("__pycache__", ".pyc", ".ipynb_checkpoints", ".DS_Store") +ROOT_LINES = (80, 150) +SUB_LINES = (80, 250) + + +def parse_frontmatter(text: str) -> tuple[dict, str] | tuple[None, str]: + if not text.startswith("---\n"): + return None, "no frontmatter block" + end = text.find("\n---\n", 4) + if end == -1: + return None, "unterminated frontmatter block" + block = text[4:end] + data: dict = {} + key = None + # A double-quoted YAML scalar may span lines. Join continuations onto the + # value before parsing, otherwise a perfectly valid multi-line description + # is reported as unquoted, which is a bug in the checker and not in the + # skill it is checking. + lines, joined = block.split("\n"), [] + for line in lines: + if (joined and isinstance(joined[-1], str) and line.startswith(" ") + and joined[-1].count('"') == 1 and '"' in joined[-1]): + joined[-1] = joined[-1].rstrip() + " " + line.strip() + continue + joined.append(line) + for line in joined: + if not line.strip(): + continue + if line.startswith(" ") and key: + k, _, v = line.strip().partition(":") + data.setdefault(key, {}) + if isinstance(data[key], dict): + data[key][k.strip()] = v.strip() + continue + k, _, v = line.partition(":") + key = k.strip() + data[key] = v.strip() if v.strip() else {} + return data, "" + + +class Report: + def __init__(self) -> None: + self.rows: list[tuple[str, str, str]] = [] + + def add(self, level: str, where: str, msg: str) -> None: + self.rows.append((level, where, msg)) + + @property + def failed(self) -> bool: + return any(r[0] == "FAIL" for r in self.rows) + + def print(self) -> None: + for level, where, msg in self.rows: + print(f"{level:<5} {where}: {msg}") + fails = sum(1 for r in self.rows if r[0] == "FAIL") + warns = sum(1 for r in self.rows if r[0] == "WARN") + print(f"\n{fails} failures, {warns} warnings, {len(self.rows)} findings") + + +def _tree_of(skill_md: pathlib.Path, root: pathlib.Path) -> str: + """The skill graph a file belongs to: /.""" + rel = skill_md.relative_to(root).parts + return "/".join(rel[:2]) if len(rel) > 1 else rel[0] + + +def gate_frontmatter(root: pathlib.Path, rep: Report) -> None: + """Gate 1: frontmatter contract and per-tree licence consistency.""" + licences: dict[str, set[str]] = {} + for skill_md in sorted(root.rglob("SKILL.md")): + rel = skill_md.relative_to(root).as_posix() + text = skill_md.read_text(encoding="utf-8") + fm, err = parse_frontmatter(text) + if fm is None: + rep.add("FAIL", rel, err) + continue + for field in REQUIRED_FIELDS: + if field not in fm: + rep.add("FAIL", rel, f"missing required frontmatter field `{field}`") + name = fm.get("name", "") + if name != skill_md.parent.name: + rep.add("FAIL", rel, f"name `{name}` does not equal directory `{skill_md.parent.name}`") + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,63}", str(name)): + rep.add("FAIL", rel, f"name `{name}` violates the id pattern") + desc = fm.get("description", "") + if not (isinstance(desc, str) and desc.startswith('"') and desc.rstrip().endswith('"')): + rep.add("FAIL", rel, "description must be a double-quoted string") + lic = fm.get("license", "") + if not isinstance(lic, str) or not lic.strip(): + rep.add("FAIL", rel, "license must be a non-empty single-line value") + else: + licences.setdefault(_tree_of(skill_md, root), set()).add(lic.strip()) + meta = fm.get("metadata", {}) + role = meta.get("disco-role") if isinstance(meta, dict) else None + if role != "operating": + rep.add("FAIL", rel, f"metadata.disco-role must be `operating`, found `{role}`") + # Industrial extension: any skill declaring a capability family must + # declare a valid asset class and a leakage class, and only `ops` ships. + if isinstance(meta, dict) and "capability-family" in meta: + fams = {x.strip() for x in str(meta["capability-family"]).split(",")} + bad = fams - CAPABILITY_FAMILIES + if bad: + rep.add("FAIL", rel, f"unknown capability family: {sorted(bad)}") + classes = {x.strip() for x in str(meta.get("asset-class", "")).split(",")} + bad = classes - ASSET_CLASSES + if bad: + rep.add("FAIL", rel, f"unknown asset class: {sorted(bad)}") + lk = str(meta.get("leakage-class", "")).strip() + if lk not in LEAKAGE_CLASSES: + rep.add("FAIL", rel, f"leakage-class must be one of {sorted(LEAKAGE_CLASSES)}, found `{lk}`") + elif lk == "solution": + rep.add("FAIL", rel, "a `solution` class skill must never ship to an evaluated agent") + + is_router = skill_md.parent.name == "repo-skills-router" + dmi = str(fm.get("disable-model-invocation", "")).lower() + if is_router and dmi == "true": + rep.add("FAIL", rel, "the router must not set disable-model-invocation") + if not is_router and dmi != "true": + rep.add("FAIL", rel, "disable-model-invocation: true is required") + + n = len(text.splitlines()) + lo, hi = ROOT_LINES if skill_md.parent.parent.name == "repo-skills" else SUB_LINES + if n > hi: + rep.add("WARN", rel, f"{n} lines exceeds the {hi}-line target; move detail to references/") + elif n < lo and not is_router: + rep.add("WARN", rel, f"{n} lines is below the {lo}-line target; likely underspecified") + + for tree, lics in sorted(licences.items()): + if len(lics) > 1: + rep.add("FAIL", tree, + f"inconsistent licences within one skill tree: {sorted(lics)}") + + +ROUTING_REQUIRED = ("schema_version", "repo_id", "skill_id", + "taxonomy_sha256", "routing_status", "assignments") +#: The digest is stored in URI form (`sha256:<64 hex>`). A bare hex digest is +#: indistinguishable from a credential to an entropy scanner: `detect-secrets`' +#: `HexHighEntropyString` flags one at 64, 32 and even 16 characters and blocks +#: the commit. The prefix clears every scanner tested and names the algorithm at +#: the point of use, so it is the better representation regardless. +DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def gate_routing(root: pathlib.Path, rep: Report) -> None: + """Gate 1b: every graph declares where it routes, and against which taxonomy. + + The router's area pages are generated from these files, so a graph cannot be + routable and undeclared. `taxonomy_sha256` is the pin saying which taxonomy + version the assignment was made against; without it a library can be + re-routed silently and two runs that read "the same" library stop being + comparable. + """ + graphs_dir = root / "repo-skills" + if not graphs_dir.is_dir(): + return + for graph in sorted(p for p in graphs_dir.iterdir() + if p.is_dir() and (p / "SKILL.md").exists()): + meta = graph / "references" / "repo-routing-metadata.json" + rel = meta.relative_to(root).as_posix() + if not meta.exists(): + rep.add("FAIL", graph.name, "missing references/repo-routing-metadata.json") + continue + try: + data = json.loads(meta.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + rep.add("FAIL", rel, f"not valid JSON: {exc}") + continue + for field in ROUTING_REQUIRED: + if field not in data or data[field] in ("", None, []): + rep.add("FAIL", rel, f"missing required routing field `{field}`") + if data.get("skill_id") not in (None, graph.name): + rep.add("FAIL", rel, f"skill_id `{data['skill_id']}` does not equal " + f"directory `{graph.name}`") + sha = str(data.get("taxonomy_sha256", "")) + if sha and not DIGEST_RE.fullmatch(sha): + if re.fullmatch(r"[0-9a-f]{64}", sha): + rep.add("FAIL", rel, "taxonomy_sha256 is a bare hex digest; write it " + "as `sha256:` so entropy scanners do not " + "read it as a credential") + else: + rep.add("FAIL", rel, "taxonomy_sha256 must be `sha256:` followed by " + "64 lowercase hex characters") + for i, asn in enumerate(data.get("assignments") or []): + if not isinstance(asn, dict) or not asn.get("area") or not asn.get("family"): + rep.add("FAIL", rel, f"assignment {i} needs both `area` and `family`") + + +def gate_static(root: pathlib.Path, rep: Report) -> None: + """Gate 2: self-containment, leakage of local paths, artifact debris.""" + for path in sorted(root.rglob("*")): + rel = path.relative_to(root).as_posix() + if any(d in rel for d in DEBRIS): + rep.add("FAIL", rel, "build or editor debris inside the runtime tree") + continue + if not path.is_file() or path.suffix not in {".md", ".py", ".json", ".jsonl"}: + continue + text = path.read_text(encoding="utf-8", errors="replace") + for pattern, label in LEAK_PATTERNS: + m = pattern.search(text) + if m: + rep.add("FAIL", rel, f"{label} leaked into a runtime file: {m.group(0)!r}") + for link in re.findall(r"\]\(([^)]+)\)", text): + if link.startswith(("http://", "https://", "#")): + continue + target = (path.parent / link).resolve() + try: + target.relative_to(root.resolve()) + except ValueError: + rep.add("FAIL", rel, f"link escapes the skill tree: {link}") + continue + if not target.exists(): + rep.add("FAIL", rel, f"broken link: {link}") + + +def _record_label(obj, index: int) -> str: + """A name for an answer record, so a leakage hit can be triaged rather than + only counted.""" + if isinstance(obj, dict): + for key in ("id", "scenario_id", "task_id", "name", "uid", "utterance_id"): + if key in obj: + return f"{key}={obj[key]}" + return f"record#{index}" + + +def load_answer_blobs(answers: pathlib.Path | None, + answers_dir: pathlib.Path | None, + hf_dataset: str | None, + hf_configs: list[str] | None, + hf_split: str | None, + rep: Report) -> list[tuple[str, str]] | None: + """Collect the benchmark's answer text from wherever it actually lives. + + Three sources, because the answers are not in the repository. The in-repo + `benchmarks/scenario_suite/*.yaml` files hold scenario ids only, so an audit + pointed at the checkout proves nothing. The real surfaces are the published + dataset and whatever export the evaluation harness writes. + + Returns a list of (label, text), or None if no source was given. + """ + blobs: list[tuple[str, str]] = [] + + def eat_file(p: pathlib.Path) -> None: + raw = p.read_text(encoding="utf-8", errors="replace") + if p.suffix == ".jsonl": + for i, line in enumerate(raw.splitlines()): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + blobs.append((f"{p.name}:{_record_label(obj, i)}", json.dumps(obj))) + except json.JSONDecodeError: + blobs.append((f"{p.name}:line{i}", line)) + elif p.suffix == ".json": + try: + obj = json.loads(raw) + except json.JSONDecodeError: + blobs.append((p.name, raw)) + return + if isinstance(obj, list): + for i, o in enumerate(obj): + blobs.append((f"{p.name}:{_record_label(o, i)}", json.dumps(o))) + else: + blobs.append((p.name, json.dumps(obj))) + else: + blobs.append((p.name, raw)) + + if answers is not None: + if not answers.exists(): + rep.add("FAIL", "collection", f"answers file not found: {answers}") + return [] + eat_file(answers) + + if answers_dir is not None: + if not answers_dir.is_dir(): + rep.add("FAIL", "collection", f"answers directory not found: {answers_dir}") + return [] + found = [p for p in sorted(answers_dir.rglob("*")) + if p.is_file() and p.suffix in {".json", ".jsonl", ".yaml", ".yml", + ".txt", ".csv", ".md"}] + if not found: + rep.add("FAIL", "collection", f"no answer files under {answers_dir}") + return [] + for p in found: + eat_file(p) + + if hf_dataset is not None: + try: + from datasets import get_dataset_config_names, load_dataset + except ImportError: + rep.add("FAIL", "collection", + "--answers-hf needs the `datasets` package; " + "install it with: pip install datasets") + return [] + try: + configs = hf_configs or list(get_dataset_config_names(hf_dataset)) + except Exception as exc: # noqa: BLE001 + rep.add("FAIL", "collection", + f"could not list configs of {hf_dataset}: " + f"{type(exc).__name__}: {exc}") + return [] + if not configs: + configs = [None] + for cfg in configs: + try: + ds = load_dataset(hf_dataset, cfg) if cfg else load_dataset(hf_dataset) + except Exception as exc: # noqa: BLE001 + rep.add("FAIL", "collection", + f"could not load {hf_dataset} config {cfg}: " + f"{type(exc).__name__}: {exc}") + continue + splits = [hf_split] if hf_split else list(ds.keys()) + for sp in splits: + if sp not in ds: + continue + for i, row in enumerate(ds[sp]): + blobs.append((f"{hf_dataset}/{cfg}/{sp}:{_record_label(row, i)}", + json.dumps(row, default=str))) + + if answers is None and answers_dir is None and hf_dataset is None: + return None + return blobs + + +def gate_leakage(root: pathlib.Path, answers: pathlib.Path | None, rep: Report) -> None: + """Gate 3: no benchmark answer content, and no evidence from excluded paths.""" + runtime_text: dict[str, str] = {} + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix in {".md", ".py", ".json", ".jsonl"}: + runtime_text[path.relative_to(root).as_posix()] = path.read_text( + encoding="utf-8", errors="replace") + + # 3a: excluded evidence paths must not be cited by any runtime instruction. + for rel, text in runtime_text.items(): + if rel.endswith("repo-provenance.md"): + continue # provenance records the exclusion itself + for bad in FORBIDDEN_EVIDENCE: + if bad in text: + rep.add("FAIL", rel, f"cites an excluded evidence path: {bad}") + + # 3b: n-gram overlap with the answer set, when one is supplied. + blobs = _ANSWER_BLOBS + if blobs is None: + rep.add("WARN", "collection", + "no answer source supplied (--answers, --answers-dir or " + "--answers-hf); the n-gram leakage audit did not run") + return + if not blobs: + return # the loader already recorded why + + def shingles(s: str, k: int = 8) -> set[str]: + words = re.findall(r"[a-z0-9_]+", s.lower()) + return {" ".join(words[i:i + k]) for i in range(max(0, len(words) - k + 1))} + + # Keep the owning record for each shingle, so a hit names the scenario it + # came from. A leakage failure that cannot be traced back gets argued with + # instead of fixed. + owner: dict[str, str] = {} + for label, b in blobs: + for sh in shingles(b): + owner.setdefault(sh, label) + + rep.add("INFO", "collection", + f"leakage audit ran against {len(blobs)} answer records, " + f"{len(owner)} distinct eight-word sequences") + + for rel, text in runtime_text.items(): + hits = shingles(text) & owner.keys() + if hits: + sample = sorted(hits)[:3] + sources = sorted({owner[h] for h in hits})[:3] + rep.add("FAIL", rel, + f"{len(hits)} eight-word sequences shared with the answer set " + f"(from {sources}), e.g. {sample}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", type=pathlib.Path, default=pathlib.Path("skills/repositories")) + ap.add_argument("--answers", type=pathlib.Path, default=None, + help="scenario file containing reference answers, for the leakage audit") + ap.add_argument("--answers-dir", type=pathlib.Path, default=None, + help="directory of answer files to audit against, walked recursively") + ap.add_argument("--answers-hf", default=None, metavar="REPO_ID", + help="HuggingFace dataset holding the answers, " + "e.g. ibm-research/AssetOpsBench; needs `pip install datasets`") + ap.add_argument("--hf-config", action="append", default=None, metavar="NAME", + help="restrict --answers-hf to this config; repeatable, " + "default is every config the dataset publishes") + ap.add_argument("--hf-split", default=None, + help="restrict --answers-hf to this split, default is every split") + a = ap.parse_args() + if not a.root.is_dir(): + print(f"root not found: {a.root}", file=sys.stderr) + return 2 + + rep = Report() + global _ANSWER_BLOBS + _ANSWER_BLOBS = load_answer_blobs(a.answers, a.answers_dir, a.answers_hf, + a.hf_config, a.hf_split, rep) + gate_frontmatter(a.root, rep) + gate_routing(a.root, rep) + gate_static(a.root, rep) + gate_leakage(a.root, a.answers, rep) + rep.print() + return 1 if rep.failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/agent/stirrup_agent/cli.py b/src/agent/stirrup_agent/cli.py index c2b56932..c249c53a 100644 --- a/src/agent/stirrup_agent/cli.py +++ b/src/agent/stirrup_agent/cli.py @@ -126,6 +126,26 @@ def _build_parser() -> argparse.ArgumentParser: "Supported with docker/local backends." ), ) + parser.add_argument( + "--skills-dir", + type=Path, + default=None, + metavar="PATH", + help=( + "Skill collection to mount into the code-execution workspace. " + "Point at the directory holding repo-skills/ and repo-skills-router/." + ), + ) + parser.add_argument( + "--k-level", + choices=("k0", "k1", "k1-recovery"), + default="k0", + help=( + "Operating-knowledge level. k0 mounts nothing (unaided baseline), " + "k1 mounts the collection, k1-recovery mounts it but instructs the " + "agent to attempt the task unaided first." + ), + ) return parser @@ -138,6 +158,8 @@ async def _run(args: argparse.Namespace) -> None: code_backend=args.code_backend, workspace_dir=args.workspace_dir, preserve_workspace=args.preserve_workspace, + skills_dir=args.skills_dir, + k_level=args.k_level, max_turns=args.max_turns, temperature=args.temperature, reasoning_effort=args.reasoning_effort, diff --git a/src/agent/stirrup_agent/runner.py b/src/agent/stirrup_agent/runner.py index 07ae5961..c2cae104 100644 --- a/src/agent/stirrup_agent/runner.py +++ b/src/agent/stirrup_agent/runner.py @@ -44,6 +44,7 @@ from .finish_tool import ASSETOPS_FINISH_TOOL from .trajectory import build_trajectory, classify_tool, final_answer from .handoff_tools import build_handoff_tools +from .skills_mount import copy_skills_into, resolve_skills_source, skills_prompt _log = logging.getLogger(__name__) @@ -110,6 +111,33 @@ def _copy_workspace_contents(source: Path, destination: Path) -> None: shutil.copy2(item, target) +def _skill_mounting_provider_class(provider_cls): + """Copy the skill library into the exec directory once it exists. + + The provider creates ``temp_dir`` under ``temp_base_dir`` when it is + entered, and that child is what the sandbox exposes as ``/workspace``. The + copy therefore has to happen here, not in ``__init__``. + """ + + class _SkillMountingCodeExecToolProvider(provider_cls): + def __init__(self, *args, skills_source: Path, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._assetops_skills_source = skills_source + + async def __aenter__(self): + result = await super().__aenter__() + temp_dir = self.temp_dir + if temp_dir is None or not Path(temp_dir).is_dir(): + raise RuntimeError( + "code-exec provider exposed no temp_dir after entry, so the " + "skill library cannot be mounted where the agent reads it" + ) + copy_skills_into(self._assetops_skills_source, temp_dir) + return result + + return _SkillMountingCodeExecToolProvider + + def _preserving_provider_class(provider_cls): class _PreservingCodeExecToolProvider(provider_cls): def __init__(self, *args, preserve_dir: Path, **kwargs) -> None: @@ -165,6 +193,8 @@ def __init__( code_backend: str = "docker", workspace_dir: Path | str | None = None, preserve_workspace: bool = False, + skills_dir: Path | str | None = None, + k_level: str = "k0", max_turns: int = 30, temperature: float | None = None, reasoning_effort: str | None = None, @@ -187,6 +217,18 @@ def __init__( "preserve_workspace is only supported with docker or local code backends" ) self._preserve_workspace = preserve_workspace + self._k_level = k_level + self._skills_source = resolve_skills_source(skills_dir, k_level=k_level) + if self._skills_source is not None and not code_enabled: + raise ValueError( + "skills mount into the code-execution workspace; " + f"k_level={k_level} requires the code track, not --no-code" + ) + self._skills_prompt = skills_prompt( + self._skills_source, + k_level=k_level, + code_backend=code_backend, + ) self._max_turns = max_turns self._temperature = temperature self._reasoning_effort = reasoning_effort @@ -264,25 +306,28 @@ def _build_code_provider(self): from stirrup.tools.code_backends.local import LocalCodeExecToolProvider provider_cls = LocalCodeExecToolProvider + args: tuple = () kwargs = {"temp_base_dir": self._workspace_dir} - if self._preserve_workspace: - provider_cls = _preserving_provider_class(provider_cls) - kwargs["preserve_dir"] = self._workspace_dir - return provider_cls(**kwargs) - from stirrup.tools.code_backends.docker import DockerCodeExecToolProvider + else: + from stirrup.tools.code_backends.docker import DockerCodeExecToolProvider + + # K0 keeps the original construction path untouched. + if not self._preserve_workspace and self._skills_source is None: + return DockerCodeExecToolProvider.from_image( + _DEFAULT_CODE_IMAGE, + temp_base_dir=self._workspace_dir, + ) + provider_cls = DockerCodeExecToolProvider + args = (_DEFAULT_CODE_IMAGE,) + kwargs = {"is_dockerfile": False, "temp_base_dir": self._workspace_dir} if self._preserve_workspace: - provider_cls = _preserving_provider_class(DockerCodeExecToolProvider) - return provider_cls( - _DEFAULT_CODE_IMAGE, - is_dockerfile=False, - temp_base_dir=self._workspace_dir, - preserve_dir=self._workspace_dir, - ) - return DockerCodeExecToolProvider.from_image( - _DEFAULT_CODE_IMAGE, - temp_base_dir=self._workspace_dir, - ) + provider_cls = _preserving_provider_class(provider_cls) + kwargs["preserve_dir"] = self._workspace_dir + if self._skills_source is not None: + provider_cls = _skill_mounting_provider_class(provider_cls) + kwargs["skills_source"] = self._skills_source + return provider_cls(*args, **kwargs) def _build_tools(self) -> list: if not self._code_enabled: @@ -296,16 +341,21 @@ def _build_tools(self) -> list: ] def _build_system_prompt(self) -> str: - """Append code-execution guidance when the code track is enabled.""" + """Append code-execution guidance, then the skill router block.""" if not self._code_enabled: - return AGENT_SYSTEM_PROMPT - - backend_prompt = ( - _DOCKER_CODE_EXEC_SYSTEM_PROMPT - if self._code_backend == "docker" - else _LOCAL_CODE_EXEC_SYSTEM_PROMPT - ) - return f"{AGENT_SYSTEM_PROMPT}\n{_CODE_EXEC_SYSTEM_PROMPT}\n{backend_prompt}" + prompt = AGENT_SYSTEM_PROMPT + else: + backend_prompt = ( + _DOCKER_CODE_EXEC_SYSTEM_PROMPT + if self._code_backend == "docker" + else _LOCAL_CODE_EXEC_SYSTEM_PROMPT + ) + prompt = ( + f"{AGENT_SYSTEM_PROMPT}\n{_CODE_EXEC_SYSTEM_PROMPT}\n{backend_prompt}" + ) + if self._skills_prompt: + prompt = f"{prompt}\n{self._skills_prompt}" + return prompt # -- run --------------------------------------------------------------- @@ -331,12 +381,13 @@ async def run(self, question: str) -> AgentResult: ) _log.info( - "StirrupAgentRunner: starting (model=%s, code=%s, backend=%s, workspace=%s, preserve=%s)", + "StirrupAgentRunner: starting (model=%s, code=%s, backend=%s, workspace=%s, preserve=%s, k_level=%s)", self._model_id, self._code_enabled, self._code_backend, self._workspace_dir, self._preserve_workspace, + self._k_level, ) async with agent.session() as session: diff --git a/src/agent/stirrup_agent/skills_mount.py b/src/agent/stirrup_agent/skills_mount.py new file mode 100644 index 00000000..6f33e0e8 --- /dev/null +++ b/src/agent/stirrup_agent/skills_mount.py @@ -0,0 +1,160 @@ +"""Skill mounting for the Stirrup runner (Plug A). + +Stirrup has no skill mechanism: `StirrupAgentRunner` builds its system prompt +from `AGENT_SYSTEM_PROMPT` plus the code-execution blocks, and serves tools +through the workspace-bridged MCP provider. This module adds the smallest thing +that makes a skill collection usable there. + +The mechanism is deliberately plain. The skill tree is copied into the +code-execution workspace base, so the agent sees it at `/workspace/skills` under +the Docker backend and at `skills/` under the local backend, and a short block +is appended to the system prompt telling it the entry point and the routing +discipline. Progressive disclosure then comes free, because the agent chooses +which file to read with the shell it already has. + +Install this file at `src/agent/stirrup_agent/skills_mount.py` and apply +`patches/stirrup_runner.diff`. + +Design notes +------------ +The prompt block names the router and nothing else. Listing the skills in the +prompt would defeat the purpose: the whole point of a routed collection is that +the up-front context cost is one paragraph rather than the library. + +`K_LEVEL` is the benchmark control. `k0` mounts nothing and appends nothing, so +the unaided baseline stays exactly what it was before this module existed. +`k1` mounts the collection. `k1-recovery` mounts it but instructs the agent to +attempt the task unaided first and consult the collection only after a concrete +failure, which preserves unaided difficulty measurement. +""" + +from __future__ import annotations + +import logging +import shutil +from pathlib import Path + +_log = logging.getLogger(__name__) + +K_LEVELS = ("k0", "k1", "k1-recovery") + +_SKILLS_PROMPT = """\ +A skill collection is mounted at {mount}. It holds operating knowledge for this +environment: which server owns which capability, the order of operations that +avoids the common failure patterns, and the preconditions a claim needs before +it is defensible. + +Route before you act. Read {mount}/repo-skills-router/SKILL.md, follow it to the +repository skill, then open that skill's sub-skill for the step you are on. Read +one sub-skill at a time and open a reference file only when the sub-skill points +at it. Do not read the whole collection. + +The skills describe this environment's tools and conventions. They do not +contain answers to your task. +""" + +_RECOVERY_PROMPT = """\ +Attempt the task on your own first. Consult the skill collection at {mount} only +after a concrete failure: a tool error you cannot resolve, an identifier that +will not resolve, or a result you cannot defend. When that happens, route +through {mount}/repo-skills-router/SKILL.md rather than browsing. +""" + + +_IGNORE = shutil.ignore_patterns( + "__pycache__", "*.pyc", ".git", "tests", "reports", "test-cases" +) + + +def resolve_skills_source( + skills_source: Path | str | None, k_level: str = "k1" +) -> Path | None: + """Validate the requested library and return it, or None when unused. + + Returns None for ``k0``. Raises when ``k1``/``k1-recovery`` is requested + without a usable library, so a run can never be labelled K1 while silently + behaving as K0. + """ + if k_level not in K_LEVELS: + raise ValueError(f"k_level must be one of {K_LEVELS}, got {k_level!r}") + if k_level == "k0": + if skills_source is not None: + _log.warning("k_level=k0 ignores --skills-dir %s", skills_source) + return None + if skills_source is None: + raise ValueError( + f"k_level={k_level} requires a skill library; pass --skills-dir " + "at the directory holding repo-skills/ and repo-skills-router/" + ) + source = Path(skills_source).expanduser().resolve() + if not source.is_dir(): + raise ValueError(f"skills source is not a directory: {source}") + if not (source / "repo-skills-router" / "SKILL.md").is_file(): + raise ValueError( + f"no repo-skills-router/SKILL.md under {source}; --skills-dir must " + "point at the directory holding repo-skills/ and repo-skills-router/" + ) + return source + + +def skills_prompt( + skills_source: Path | None, + k_level: str = "k1", + code_backend: str = "docker", +) -> str | None: + """Return the system-prompt block for the mount, or None for ``k0``.""" + if k_level not in K_LEVELS: + raise ValueError(f"k_level must be one of {K_LEVELS}, got {k_level!r}") + if k_level == "k0" or skills_source is None: + return None + mount = mount_path(code_backend) + template = _RECOVERY_PROMPT if k_level == "k1-recovery" else _SKILLS_PROMPT + return template.format(mount=mount) + + +def mount_path(code_backend: str = "docker") -> str: + """The path the agent sees, which is the exec directory, not its parent.""" + return "/workspace/skills" if code_backend == "docker" else "skills" + + +def copy_skills_into(skills_source: Path | str, exec_dir: Path | str) -> int: + """Copy the library into the live code-execution directory. + + ``exec_dir`` is the directory the sandbox exposes as ``/workspace``. It is + the provider's ``temp_dir``, a child of ``temp_base_dir``, and it does not + exist until the provider is entered. Copying into ``temp_base_dir`` instead + puts the library one level above the mount, where the agent cannot see it. + """ + source = Path(skills_source).expanduser().resolve() + destination = Path(exec_dir).expanduser().resolve() / "skills" + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(source, destination, ignore=_IGNORE) + # The sandbox may run as a different uid than the process doing the copy. + for path in destination.rglob("*"): + path.chmod(0o755 if path.is_dir() else 0o644) + destination.chmod(0o755) + n = sum(1 for _ in destination.rglob("SKILL.md")) + _log.info("mounted %d skills from %s into %s", n, source, destination) + return n + + +def mount_skills( + skills_source: Path | str | None, + workspace_dir: Path | None, + k_level: str = "k1", + code_backend: str = "docker", +) -> str | None: + """Deprecated. Copies beside the exec directory, so the agent never sees it. + + Kept only so out-of-tree callers fail loudly rather than silently mounting + into the wrong directory. Use :func:`resolve_skills_source`, + :func:`skills_prompt` and :func:`copy_skills_into`. + """ + raise NotImplementedError( + "mount_skills copied the library into temp_base_dir, which is the " + "parent of the directory exposed as /workspace. Use " + "resolve_skills_source() + skills_prompt() at construction time and " + "copy_skills_into(source, provider.temp_dir) after the provider is " + "entered." + ) diff --git a/src/agent/stirrup_agent/tests/test_skills_mount.py b/src/agent/stirrup_agent/tests/test_skills_mount.py new file mode 100644 index 00000000..6823e832 --- /dev/null +++ b/src/agent/stirrup_agent/tests/test_skills_mount.py @@ -0,0 +1,159 @@ +"""The skill library must land where the agent reads it, not beside it. + +The failure this file exists to prevent: the library was copied into +``temp_base_dir`` while the sandbox exposed a *child* of that directory as +``/workspace``, so ``/workspace/skills`` never existed and every K1 run scored +as an unaided K0 run while being labelled K1. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from agent.stirrup_agent.skills_mount import ( + copy_skills_into, + mount_path, + resolve_skills_source, + skills_prompt, +) + + +@pytest.fixture +def library(tmp_path: Path) -> Path: + root = tmp_path / "library" + (root / "repo-skills-router").mkdir(parents=True) + (root / "repo-skills-router" / "SKILL.md").write_text("router\n") + (root / "repo-skills" / "demo").mkdir(parents=True) + (root / "repo-skills" / "demo" / "SKILL.md").write_text("demo\n") + (root / "repo-skills" / "demo" / "__pycache__").mkdir() + (root / "repo-skills" / "demo" / "__pycache__" / "x.pyc").write_text("junk") + return root + + +def test_k1_without_a_library_raises(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="requires a skill library"): + resolve_skills_source(None, k_level="k1") + + +def test_k1_with_a_non_library_directory_raises(tmp_path: Path) -> None: + (tmp_path / "empty").mkdir() + with pytest.raises(ValueError, match="repo-skills-router"): + resolve_skills_source(tmp_path / "empty", k_level="k1") + + +def test_k0_mounts_nothing_and_appends_nothing(library: Path) -> None: + assert resolve_skills_source(None, k_level="k0") is None + assert resolve_skills_source(library, k_level="k0") is None + assert skills_prompt(None, k_level="k0") is None + + +def test_copy_lands_inside_the_exec_dir(library: Path, tmp_path: Path) -> None: + base = tmp_path / "ws" + exec_dir = base / "stirrup_agent" / "run-1" / "exec-1" + exec_dir.mkdir(parents=True) + + copy_skills_into(library, exec_dir) + + # What the prompt promises the agent, relative to the exec dir. + assert (exec_dir / "skills" / "repo-skills-router" / "SKILL.md").is_file() + # And nothing beside it, which is where the old code put the library. + assert not (base / "skills").exists() + + +def test_copy_drops_junk_and_reports_the_count(library: Path, tmp_path: Path) -> None: + exec_dir = tmp_path / "exec" + exec_dir.mkdir() + + assert copy_skills_into(library, exec_dir) == 2 + assert not (exec_dir / "skills" / "repo-skills" / "demo" / "__pycache__").exists() + + +def test_copy_is_idempotent(library: Path, tmp_path: Path) -> None: + exec_dir = tmp_path / "exec" + exec_dir.mkdir() + copy_skills_into(library, exec_dir) + stale = exec_dir / "skills" / "stale.md" + stale.write_text("from a previous run") + + copy_skills_into(library, exec_dir) + + assert not stale.exists() + + +@pytest.mark.parametrize( + ("backend", "expected"), [("docker", "/workspace/skills"), ("local", "skills")] +) +def test_prompt_names_the_router_at_the_mount( + library: Path, backend: str, expected: str +) -> None: + block = skills_prompt(library, k_level="k1", code_backend=backend) + assert f"{expected}/repo-skills-router/SKILL.md" in block + assert mount_path(backend) == expected + + +def test_recovery_prompt_defers_the_library(library: Path) -> None: + block = skills_prompt(library, k_level="k1-recovery", code_backend="docker") + assert "on your own first" in block + assert "/workspace/skills/repo-skills-router/SKILL.md" in block + + +def test_provider_wrapper_copies_after_entry(library: Path, tmp_path: Path) -> None: + """The wrapper must copy into temp_dir, which only exists after entry.""" + from agent.stirrup_agent.runner import _skill_mounting_provider_class + + base = tmp_path / "ws" + base.mkdir() + + class _FakeProvider: + """Mimics the Stirrup contract: temp_dir is a child, made on entry.""" + + def __init__(self, *, temp_base_dir: Path) -> None: + self._base = Path(temp_base_dir) + self.temp_dir = None + + async def __aenter__(self): + self.temp_dir = self._base / "exec-abc" + self.temp_dir.mkdir() + return self + + async def __aexit__(self, *exc) -> None: + return None + + wrapped = _skill_mounting_provider_class(_FakeProvider) + + async def _run() -> Path: + async with wrapped(temp_base_dir=base, skills_source=library) as provider: + return provider.temp_dir + + exec_dir = asyncio.run(_run()) + + assert (exec_dir / "skills" / "repo-skills-router" / "SKILL.md").is_file() + assert not (base / "skills").exists() + + +def test_wrapper_refuses_a_provider_without_a_temp_dir( + library: Path, tmp_path: Path +) -> None: + from agent.stirrup_agent.runner import _skill_mounting_provider_class + + class _NoTempDirProvider: + def __init__(self, **kwargs) -> None: + self.temp_dir = None + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc) -> None: + return None + + wrapped = _skill_mounting_provider_class(_NoTempDirProvider) + + async def _run() -> None: + async with wrapped(skills_source=library): + pass + + with pytest.raises(RuntimeError, match="no temp_dir"): + asyncio.run(_run()) diff --git a/src/benchmark/scenario_suite_runner.py b/src/benchmark/scenario_suite_runner.py index 5b92e03a..30ad9301 100644 --- a/src/benchmark/scenario_suite_runner.py +++ b/src/benchmark/scenario_suite_runner.py @@ -431,6 +431,12 @@ def build_methods(args: argparse.Namespace) -> dict[str, MethodConfig]: args, "stirrup_workspace_root", None ) is not None: stirrup_extra_args.append("--preserve-workspace") + skills_dir = getattr(args, "skills_dir", None) + if skills_dir is not None: + stirrup_extra_args.extend(["--skills-dir", str(skills_dir)]) + k_level = getattr(args, "k_level", None) + if k_level is not None: + stirrup_extra_args.extend(["--k-level", k_level]) opencode_extra_args: list[str] = [] if args.opencode_allow_files: @@ -776,6 +782,9 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Print commands without executing them.", ) + parser.add_argument("--skills-dir", type=Path, default=None) + parser.add_argument("--k-level", default="k0", + choices=("k0", "k1", "k1-recovery")) return parser