From df5ce8d62b11551fd095e99e99ceca9744eaeac4 Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:05:07 +0000 Subject: [PATCH 1/4] Resolve issue #11867 Lane-Activity: work-resolve-issue/resolve-issue --- ...work-resolve-issue-issue-11867-5d2e8c71.md | 12 ++++ plugins/work-items/README.md | 7 ++ .../case.yaml | 37 +++++++++++ .../samples/lists-issue.json | 22 +++++++ .../samples/no-pr-listing-call.json | 65 +++++++++++++++++++ .../samples/no-pr-rows.json | 38 +++++++++++ .../samples/reports-setting-false.json | 26 ++++++++ .../work-items/schemas/work-items.schema.json | 7 +- plugins/work-items/skills/triage/SKILL.md | 27 +++++++- 9 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 .changes/work-items/work-resolve-issue-issue-11867-5d2e8c71.md create mode 100644 plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/case.yaml create mode 100644 plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/lists-issue.json create mode 100644 plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-listing-call.json create mode 100644 plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-rows.json create mode 100644 plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/reports-setting-false.json diff --git a/.changes/work-items/work-resolve-issue-issue-11867-5d2e8c71.md b/.changes/work-items/work-resolve-issue-issue-11867-5d2e8c71.md new file mode 100644 index 00000000000..f97927befc9 --- /dev/null +++ b/.changes/work-items/work-resolve-issue-issue-11867-5d2e8c71.md @@ -0,0 +1,12 @@ +--- +bump: minor +--- + +### Added + +- `/work-items:triage` reads the repository's glossary, the ADRs covering the item's area and its scope statement before it evaluates an item. It names which of them it found in a `Read first:` line (#11867). +- A `triage_outside_prs` repository setting in `docs/conventions/work-items.yaml`, default `false`, says whether the repository takes requests as pull requests from outside contributors (#11867). + +### Changed + +- With `triage_outside_prs` unset or `false`, the `/work-items:triage` attention view lists no pull requests and does not fetch them. A pull request named explicitly is still triaged (#11867). diff --git a/plugins/work-items/README.md b/plugins/work-items/README.md index 20d083c9064..cf58f949401 100644 --- a/plugins/work-items/README.md +++ b/plugins/work-items/README.md @@ -190,6 +190,13 @@ when it resolves `true`, autonomous lanes start each tracker item and comment they create with the AI disclaimer. The key, its layers and the prefix are stated once in [`reference/ai-disclaimer.md`](reference/ai-disclaimer.md). +`triage_outside_prs` (default `false`) lives only in the repository file. Set it +to `true` when the repository takes requests as pull requests from outside +contributors: the `/work-items:triage` attention view then lists those pull +requests beside the issues. When it is `false`, the view lists only issues. A +pull request named explicitly, as `/work-items:triage `, is triaged +either way. + `work_priority_order` lives only in the repository file: a list of `priority:` label names, highest first, defaulting to `priority: critical`, `priority: high`, `priority: medium`, `priority: low`. `/work-items:work` ranks diff --git a/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/case.yaml b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/case.yaml new file mode 100644 index 00000000000..a6edcd1da3f --- /dev/null +++ b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/case.yaml @@ -0,0 +1,37 @@ +schema_version: "1.1" +name: tr-outside-prs-unset-lists-no-pr +description: 'Outside-PR intake is off by default: the workspace has no docs/conventions/work-items.yaml, so triage_outside_prs is unset and resolves false. The attention view lists the unlabeled issue #31 and no pull request, though the listing in the prompt also carries an unlabeled outside-contributor PR #32. A run needs --allow-tools Bash.' +tags: [triage, attention-view, outside-prs, must-not] +plugins: ["../.."] +expected_outcome: 'The run resolves triage_outside_prs to false (unset, layer default) and says so, lists #31 under the unlabeled bucket, and prints no row for a pull request: no line or table cell starts with a [PR] prefix and #32 is not listed as intake. It runs no pull-request listing command.' +execution: + prompt: "/work-items:triage\n\nInteractive session. The adapter's \"List items\" read for this repository returns two open items, both with no labels, no blockers and blocked_by_wont_do_count 0: issue #31 'CSV export drops the last row', opened 2026-10-02 by a reporter; and pull request #32 'Add a cache layer to the export', opened 2026-10-03 by an outside contributor (author_association NONE). Use this listing; there is no other tracker to read. Print the attention-view table only; build no board page." + allowed_tools: [Bash, Read, Glob, Grep] + max_turns: 15 + timeout_seconds: 240 +graders: + - name: lists-issue + type: regex + pattern: '#31\b' + target: last_message + arm: both + - name: reports-setting-false + type: regex + pattern: '\btriage_outside_prs\b[^\n]*?\bfalse\b' + flags: 'i' + target: last_message + arm: both + - name: no-pr-rows + type: regex + pattern: '^[ \t>]*(?:\|[^\n]*(?:\[PR\]|#32\b)|(?:[-*+]|\d+[.)])[ \t]+[^\n]*(?:\[PR\]|#32\b)|`?\[PR\])' + flags: 'm' + match: not_contains + target: last_message + arm: both + - name: no-pr-listing-call + type: tool_used + tool: Bash + input_match: '\bgh[ \t]+pr[ \t]+list\b|\bgh[ \t]+search[ \t]+prs\b|\bgh[ \t]+api\b[^\n]*?(?:/pulls\b|\bpullRequests\b)' + min: 0 + max: 0 + arm: both diff --git a/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/lists-issue.json b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/lists-issue.json new file mode 100644 index 00000000000..7e5e145319c --- /dev/null +++ b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/lists-issue.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "Settings: triage_outside_prs=false (default)\n\n| Bucket | Item |\n|---|---|\n| unlabeled | #31 CSV export drops the last row |", + "why": "issue row" + }, + { + "answer": "Unlabeled: #31 CSV export drops the last row (opened 2026-10-02).", + "why": "prose line" + } + ], + "fail": [ + { + "answer": "Nothing needs triage.", + "why": "drops the issue" + }, + { + "answer": "Unlabeled: #310 Rework the importer.", + "why": "another number" + } + ] +} diff --git a/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-listing-call.json b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-listing-call.json new file mode 100644 index 00000000000..3dfa482e15b --- /dev/null +++ b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-listing-call.json @@ -0,0 +1,65 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "bash \"/p/skills/triage/scripts/parse-concern-value.sh\" docs/conventions/work-items.yaml triage_outside_prs" + } + } + ], + "why": "settings read" + }, + { + "answer": [], + "why": "uses the given listing" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh pr list --state open --json number,title,author" + } + } + ], + "why": "lists PRs" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api repos/o/r/pulls?state=open" + } + } + ], + "why": "lists PRs through the API" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api \"repos/o/r/pulls?state=open\"" + } + } + ], + "why": "quoted API path" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh search prs --repo o/r --state open" + } + } + ], + "why": "PR search" + } + ] +} diff --git a/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-rows.json b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-rows.json new file mode 100644 index 00000000000..9046441d3ac --- /dev/null +++ b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/no-pr-rows.json @@ -0,0 +1,38 @@ +{ + "pass": [ + { + "answer": "Settings: triage_outside_prs=false (default)\n\n| Bucket | Item |\n|---|---|\n| unlabeled | #31 CSV export drops the last row |", + "why": "issue only" + }, + { + "answer": "| unlabeled | #31 CSV export drops the last row |\n\nOutside-PR intake is off, so no [PR] rows: #32 is not listed.", + "why": "mentions the prefix in prose, lists no PR row" + } + ], + "fail": [ + { + "answer": "| Bucket | Item |\n|---|---|\n| unlabeled | [issue] #31 CSV export drops the last row |\n| unlabeled | [PR] #32 Add a cache layer to the export |", + "why": "PR table row" + }, + { + "answer": "Unlabeled:\n- [issue] #31 CSV export drops the last row\n- [PR] #32 Add a cache layer to the export", + "why": "PR list line" + }, + { + "answer": "[PR] #32 Add a cache layer to the export", + "why": "bare PR line" + }, + { + "answer": "| Item | Type | Title |\n|---|---|---|\n| #31 | issue | CSV export drops the last row |\n| #32 | PR | Add a cache layer to the export |", + "why": "PR row through a type column" + }, + { + "answer": "1. [issue] #31 CSV export drops the last row\n2. [PR] #32 Add a cache layer to the export", + "why": "numbered list" + }, + { + "answer": "- `[PR]` #32 Add a cache layer to the export", + "why": "prefix in backticks" + } + ] +} diff --git a/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/reports-setting-false.json b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/reports-setting-false.json new file mode 100644 index 00000000000..3506b0f57b3 --- /dev/null +++ b/plugins/work-items/evals/tr-outside-prs-unset-lists-no-pr/samples/reports-setting-false.json @@ -0,0 +1,26 @@ +{ + "pass": [ + { + "answer": "Settings: triage_outside_prs=false (default)", + "why": "settings line" + }, + { + "answer": "`triage_outside_prs` is unset, so it resolves to `false` and the view lists no pull request.", + "why": "prose" + }, + { + "answer": "triage_outside_prs: false (layer default)", + "why": "colon form" + } + ], + "fail": [ + { + "answer": "Settings: triage_outside_prs=true (repository)", + "why": "wrong value" + }, + { + "answer": "Outside PRs are not listed.", + "why": "names no setting" + } + ] +} diff --git a/plugins/work-items/schemas/work-items.schema.json b/plugins/work-items/schemas/work-items.schema.json index e82f72edd0b..c3b3387279a 100644 --- a/plugins/work-items/schemas/work-items.schema.json +++ b/plugins/work-items/schemas/work-items.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/melodic-software/claude-code-plugins/main/plugins/work-items/schemas/work-items.schema.json", "title": "work-items team config", - "description": "Schema for a repository's docs/conventions/work-items.yaml, the team layer of the work-items plugin's settings (ADR 0064). A key set here wins over the same key in the plugin's userConfig. Readers: /work-items:triage for the triage keys (the /work-items:work-loop admission gate reads the objection window's end from the marker comment triage posts), every lane that follows reference/ai-disclaimer.md for ai_disclaimer, and /work-items:work for work_priority_order.", + "description": "Schema for a repository's docs/conventions/work-items.yaml, the team layer of the work-items plugin's settings (ADR 0064). A key set here wins over the same key in the plugin's userConfig. Readers: /work-items:triage for the triage keys (the /work-items:work-loop admission gate reads the objection window's end from the marker comment triage posts; triage_outside_prs has no userConfig layer), every lane that follows reference/ai-disclaimer.md for ai_disclaimer, and /work-items:work for work_priority_order.", "type": "object", "additionalProperties": false, "properties": { @@ -19,6 +19,11 @@ "default": 0, "description": "Hours after triage marks an item autonomous-eligible during which the work-loop lane does not dispatch it, so a person can object. 0 turns the window off." }, + "triage_outside_prs": { + "type": "boolean", + "default": false, + "description": "Whether the repository takes requests as pull requests from outside contributors. true: the /work-items:triage attention view lists their open pull requests beside the issues. false or absent: it lists no pull request. A pull request named explicitly is triaged either way." + }, "ai_disclaimer": { "type": "boolean", "default": false, diff --git a/plugins/work-items/skills/triage/SKILL.md b/plugins/work-items/skills/triage/SKILL.md index 3504b1470cd..aa5ae8b212d 100644 --- a/plugins/work-items/skills/triage/SKILL.md +++ b/plugins/work-items/skills/triage/SKILL.md @@ -51,7 +51,7 @@ read it before applying that stamp in step 5. Three rules bound what enters this flow: -- **A PR is an item with attached code.** An unsolicited or external PR enters the same intake as an issue: same states, same machine. Its diff is an **attachment to evaluate**: fetch it and run the relevant tests; it never creates an obligation to merge. Read the state names against the code: briefed means a brief exists for what to do with the diff; human-gated means a human should decide the merge. +- **A PR is an item with attached code.** An unsolicited or external PR enters the same intake as an issue, when it is named explicitly or the attention view lists it under `triage_outside_prs`: same states, same machine. Its diff is an **attachment to evaluate**: fetch it and run the relevant tests; it never creates an obligation to merge. Read the state names against the code: briefed means a brief exists for what to do with the diff; human-gated means a human should decide the merge. - **Never re-triage already-triaged output.** Items born triaged (published by `/work-items:decompose`, or created by a `/work-items:track add` that leaves no raw marker) never re-enter this flow; the attention view excludes them by construction (only the blocked-by-won't-do bucket can list one, as a report for the operator). The exclusion keys on **absence of the raw marker**, not authorship and not classification labels: the raw marker or being unlabeled puts an item in scope even alongside default labels (the marker wins), and a `track add` item with classification labels but no raw marker is out of scope. If someone names an already-triaged item explicitly, say it is already triaged and stop. - **Lane infrastructure is never intake.** The loop-lane convention's per-lane telemetry tracking issues, the surfaces holding that convention's sentinel-marked status comment, are lane infrastructure, not backlog: an open one is a lane operating. **Identify one the way the lane resolves its own telemetry home**, never by title alone: the issue the lane's launch config pins (`lanes[].telemetry.issue` in the `harness-ops` lane config, read from `/.work/lanes/lanes.json`, or from `/.work/lanes.json` when only that file exists), else the default `Lane telemetry: ` title (`/work-items:work-loop`, "Telemetry and durable loop state"); and, independent of both, **any issue carrying the convention's sentinel status comment** (``). **Also exclude `work-map` container items**: open issues carrying the tracker seam's container label (`WIT_CONTAINER_LABEL`, default `work-map`) are never claimable frontier work (`list-frontier` drops them unconditionally per the seam contract); their openness means the map exists. The telemetry exclusion holds **whatever labels the issue carries, the raw marker included** (filing defaults and lanes can add it at any cycle). A telemetry issue never enters the attention view, and one named explicitly is reported as lane infrastructure and stopped on, never state-machined, relabeled, or closed, since the lane reads that surface to operate. Container items are filtered from the attention view the same way. The lanes' own snapshots point here for the intake population. @@ -125,7 +125,7 @@ Report a `blocked by won't-do` item for the operator to decide (drop the edge, r Within each bucket, list the oldest item first, one line per item. -List open items and filter into buckets programmatically (adapter: "List items", bare read). Apply the lane-infrastructure exclusion ("Scope: raw intake only") to that listing **before** bucketing, so a telemetry issue carrying the raw marker is filtered out rather than bucketed under it. The `blocked by won't-do` bucket reads the count from the listing rows, whatever labels the item carries: an item already triaged still lands there, and one that also fits an earlier bucket is listed once, under the earlier bucket. **Defensive skip** (the first three buckets only): drop any item that already carries a native `blocked-by` edge *and* a prior triage comment (a verification, needs-info or other triage comment from an earlier pass, with or without the AI disclaimer, in any shape it was posted in), a stray re-label from another lane must not cost a full re-investigation. When the repo accepts requests in the form of outside PRs, list them in the same buckets with a `[PR]` or `[issue]` prefix on every line. Only PRs from outside contributors appear: a PR a collaborator is still working on is their work, not intake. That limit applies to this listing only; a PR named explicitly gets triaged whoever opened it. Present as a compact table. +List open items and filter into buckets programmatically (adapter: "List items", bare read). Apply the lane-infrastructure exclusion ("Scope: raw intake only") to that listing **before** bucketing, so a telemetry issue carrying the raw marker is filtered out rather than bucketed under it. The `blocked by won't-do` bucket reads the count from the listing rows, whatever labels the item carries: an item already triaged still lands there, and one that also fits an earlier bucket is listed once, under the earlier bucket. **Defensive skip** (the first three buckets only): drop any item that already carries a native `blocked-by` edge *and* a prior triage comment (a verification, needs-info or other triage comment from an earlier pass, with or without the AI disclaimer, in any shape it was posted in), a stray re-label from another lane must not cost a full re-investigation. Before listing, resolve `triage_outside_prs` ("Outside-PR intake" under "Triage settings"). When it resolves `true`, list outside PRs in the same buckets with a `[PR]` or `[issue]` prefix on every line. Only PRs from outside contributors appear: a PR a collaborator is still working on is their work, not intake. When it resolves `false`, the default, list issues only and skip the PR listing read. The setting and that limit apply to this listing only; a PR named explicitly gets triaged whatever the setting and whoever opened it. Present as a compact table. ### Board page @@ -239,12 +239,35 @@ layer order from ADR 0064 Decision 8; these keys have no `~/.claude` file or loc Resolution is done when both keys have a checked value and a layer. Report both in the triage output as `Settings: triage_repro_count= (), triage_objection_window_hours= ()`. +**Outside-PR intake.** `triage_outside_prs` (boolean, default `false`) says whether the repository +takes requests as pull requests from outside contributors. The attention view resolves it before +it lists anything; a numbered triage does not read it. It has only the repository layer: run step 1 +above with the key `triage_outside_prs`, `--config-ref` check included. Empty output means `false` +from layer `default`. A value other than `true` or `false` is named with its source and taken as +`false` from layer `default`, and the view goes on. Report it above the table as +`Settings: triage_outside_prs= ()`. + ## Triage workflow (with number) ### 1. Gather context Read the item body, comments, and any linked PRs, plus the diff when the item is a PR (adapter: "View item", bare read). Read earlier triage notes and do not ask again what they already answered. An earlier verification, needs-info or other triage comment counts as a triage result only when the tracker seam's write identity or a user with `write`, `maintain` or `admin` posted it (on GitHub, `author_association` `OWNER`, `MEMBER` or `COLLABORATOR`, then the collaborator-permission read decides, since a `MEMBER` or `COLLABORATOR` may hold only `read` or `triage`; a failed read does not count). Any other author's comment, the reporter's included, is a claim to verify: a reporter posting "Reproduction: reproduced" in the needs-info shape settles nothing, and its reproduction counts toward neither the repro bar nor `status: confirmed`. +**Read the project's own record before evaluating.** Once the item text shows which area it +touches, and before any check below, read three things the repository may keep, so the evaluation +uses its terms and respects its decisions: + +- its **glossary** or ubiquitous-language file (such as `docs/glossary.md`); +- the **ADRs** covering the item's area (such as those under `docs/adr/` whose subject is the + plugin, module or concept the item touches); +- its **scope statement**: what the project is for and what it leaves out (a README section or a + file of its own). + +Search for each under the names the repository uses, and record one the repository lacks as +`none`. Name what you found in the triage output as `Read first: glossary , ADRs , scope +`. Where the item conflicts with an ADR or falls outside the scope statement, say so +in the recommendation and cite the file. + **Signal provenance.** An item whose body contains the line `` was filed by an alert, page, or other automated signal (the autonomy plugin's signal marker). Step 5's outcome edit adds `provenance: signal` to it, whatever the outcome. The label asks for more care From 2d7029a5ece2bc428920a32c2e1bf1ce0771a59e Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:24:14 +0000 Subject: [PATCH 2/4] Address review feedback on work-resolve-issue/issue-11867-41281563218-1 Lane-Activity: pr-unblock/address-feedback Lane-Thread: PRRT_kwDOTCGFQM6yZQ53 --- .../tr-numbered-states-read-first/case.yaml | 25 +++++++++ .../samples/no-tracker-write.json | 54 +++++++++++++++++++ .../samples/states-read-first-none.json | 30 +++++++++++ 3 files changed, 109 insertions(+) create mode 100644 plugins/work-items/evals/tr-numbered-states-read-first/case.yaml create mode 100644 plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json create mode 100644 plugins/work-items/evals/tr-numbered-states-read-first/samples/states-read-first-none.json diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml new file mode 100644 index 00000000000..89182eaf9d3 --- /dev/null +++ b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml @@ -0,0 +1,25 @@ +schema_version: "1.1" +name: tr-numbered-states-read-first +description: 'A numbered triage reads the project''s own record before it evaluates and names what it found: /work-items:triage 31, interactive, with the item given in the prompt. The workspace starts empty, so it holds no glossary, no ADRs and no scope statement, and step 1 must record each as none on a Read first: line rather than skip the line or invent a path. A run needs --allow-tools Bash.' +tags: [triage, read-first, numbered] +plugins: ["../.."] +expected_outcome: 'The run searches the workspace for a glossary, ADRs and a scope statement, finds none, and its triage output carries the line Read first: glossary none, ADRs none, scope none before its recommendation. It presents the recommendation and waits: it runs no tracker write (no label, comment or close).' +execution: + prompt: "/work-items:triage 31\n\nInteractive session. The adapter's \"View item\" read for this repository returns issue #31 'CSV export drops the last row', opened 2026-10-02 by a reporter, with no labels, no comments and no linked pull request. Its body: 'Exporting a report with 10 rows to CSV gives a file with 9 data rows. Expected 10.' Use this item as given; there is no other tracker to read. Present your recommendation only and change nothing." + allowed_tools: [Bash, Read, Glob, Grep] + max_turns: 15 + timeout_seconds: 240 +graders: + - name: states-read-first-none + type: regex + pattern: '^[^\n]*\bRead first\b\W*glossary\W*none\b[^\n]*?\bADRs?\W*none\b[^\n]*?\bscope\W*none\b' + flags: 'mi' + target: last_message + arm: both + - name: no-tracker-write + type: tool_used + tool: Bash + input_match: '\bgh[ \t]+(?:issue|pr)[ \t]+(?:edit|comment|close)\b|\bgh[ \t]+label[ \t]+(?:create|edit|delete)\b|\bgh[ \t]+api\b[^\n]*?-(?:X|-method)[ \t=]*(?:POST|PATCH|PUT|DELETE)\b' + min: 0 + max: 0 + arm: both diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json new file mode 100644 index 00000000000..4a86b038521 --- /dev/null +++ b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json @@ -0,0 +1,54 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "ls -a docs README.md" + } + } + ], + "why": "searches the workspace" + }, + { + "answer": [], + "why": "uses the given item" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh issue edit 31 --add-label \"type: bug\"" + } + } + ], + "why": "labels the item" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh issue comment 31 --body \"Triage: reproduce first\"" + } + } + ], + "why": "comments on the item" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api -X POST repos/o/r/issues/31/labels -f labels[]=bug" + } + } + ], + "why": "labels through the API" + } + ] +} diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/samples/states-read-first-none.json b/plugins/work-items/evals/tr-numbered-states-read-first/samples/states-read-first-none.json new file mode 100644 index 00000000000..a40253c4ca4 --- /dev/null +++ b/plugins/work-items/evals/tr-numbered-states-read-first/samples/states-read-first-none.json @@ -0,0 +1,30 @@ +{ + "pass": [ + { + "answer": "Read first: glossary none, ADRs none, scope none", + "why": "template line" + }, + { + "answer": "**Read first:** glossary `none`, ADRs `none`, scope `none`\n\nRecommendation: Bug, priority: medium, needs reproduction.", + "why": "bold label and code spans" + }, + { + "answer": "- Read first: glossary: none, ADR: none, scope: none (searched README*, docs/)", + "why": "colon form with a search note" + } + ], + "fail": [ + { + "answer": "Read first: glossary docs/glossary.md, ADRs none, scope none", + "why": "names a glossary the empty workspace does not hold" + }, + { + "answer": "Recommendation: Bug, priority: medium. No glossary, ADRs or scope statement found.", + "why": "no Read first line" + }, + { + "answer": "Read first: glossary none, ADRs none", + "why": "scope left out" + } + ] +} From c55d60a2b08737dd0ac5d5443fe5af5e9fe9f012 Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:46:58 +0000 Subject: [PATCH 3/4] Address review feedback on work-resolve-issue/issue-11867-41281563218-1 Lane-Activity: pr-unblock/address-feedback Lane-Thread: PRRT_kwDOTCGFQM6yZVMt --- .../tr-numbered-states-read-first/case.yaml | 2 +- .../samples/no-tracker-write.json | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml index 89182eaf9d3..3fadfd0675b 100644 --- a/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml +++ b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml @@ -19,7 +19,7 @@ graders: - name: no-tracker-write type: tool_used tool: Bash - input_match: '\bgh[ \t]+(?:issue|pr)[ \t]+(?:edit|comment|close)\b|\bgh[ \t]+label[ \t]+(?:create|edit|delete)\b|\bgh[ \t]+api\b[^\n]*?-(?:X|-method)[ \t=]*(?:POST|PATCH|PUT|DELETE)\b' + input_match: '(?:\b|(?<=\\n))gh[ \t]+(?:issue|pr)[ \t]+(?:edit|comment|close)\b|(?:\b|(?<=\\n))gh[ \t]+label[ \t]+(?:create|edit|delete)\b|(?:\b|(?<=\\n))gh[ \t]+api\b(?:(?!\\n|;|&&|\|)[^\n])*?(?:-(?:X|-method)[ \t=]*(?:POST|PATCH|PUT|DELETE)\b|[ \t]-[fF]\b|--(?:raw-)?field\b|--input\b|\bmutation\b)' min: 0 max: 0 arm: both diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json index 4a86b038521..ae549a00a2e 100644 --- a/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json +++ b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json @@ -14,6 +14,17 @@ { "answer": [], "why": "uses the given item" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api repos/o/r/issues/31\ngrep -F CSV README.md" + } + } + ], + "why": "reads the item through the API, then searches on its own line" } ], "fail": [ @@ -49,6 +60,39 @@ } ], "why": "labels through the API" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "ls docs README.md\ngh issue edit 31 --add-label bug" + } + } + ], + "why": "labels the item on a later line" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api repos/o/r/issues/31/labels -f 'labels[]=bug'" + } + } + ], + "why": "labels through the API with an implied POST" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api graphql -f query='mutation { addLabelsToLabelable(input: {labelableId: \"I_1\", labelIds: [\"L_1\"]}) { clientMutationId } }'" + } + } + ], + "why": "labels through a GraphQL mutation" } ] } From 0bff85ab3c9007d8709065480b36515329da3267 Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:55:17 +0000 Subject: [PATCH 4/4] Address review feedback on work-resolve-issue/issue-11867-41281563218-1 Lane-Activity: pr-unblock/address-feedback Lane-Thread: PRRT_kwDOTCGFQM6yZeTg --- .../tr-numbered-states-read-first/case.yaml | 2 +- .../samples/no-tracker-write.json | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml index 3fadfd0675b..5cd64ab7406 100644 --- a/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml +++ b/plugins/work-items/evals/tr-numbered-states-read-first/case.yaml @@ -19,7 +19,7 @@ graders: - name: no-tracker-write type: tool_used tool: Bash - input_match: '(?:\b|(?<=\\n))gh[ \t]+(?:issue|pr)[ \t]+(?:edit|comment|close)\b|(?:\b|(?<=\\n))gh[ \t]+label[ \t]+(?:create|edit|delete)\b|(?:\b|(?<=\\n))gh[ \t]+api\b(?:(?!\\n|;|&&|\|)[^\n])*?(?:-(?:X|-method)[ \t=]*(?:POST|PATCH|PUT|DELETE)\b|[ \t]-[fF]\b|--(?:raw-)?field\b|--input\b|\bmutation\b)' + input_match: '(?:\b|(?<=\\n))gh[ \t]+(?:issue|pr)[ \t]+(?:edit|comment|close)\b|(?:\b|(?<=\\n))gh[ \t]+label[ \t]+(?:create|edit|delete)\b|(?:\b|(?<=\\n))gh[ \t]+api\b(?:(?:(?!\\n|;|&&|\|)[^\n])*?(?:-(?:X|-method)[ \t=]*(?:POST|PATCH|PUT|DELETE)\b|\bmutation\b)|(?!(?:(?!\\n|;|&&|\|)[^\n])*?-(?:X|-method)[ \t=]*GET\b)(?:(?:(?!\\n|;|&&|\|)[^\n])*?--input\b|(?!(?:(?!\\n|;|&&|\|)[^\n])*?[ \t]graphql\b)(?:(?!\\n|;|&&|\|)[^\n])*?(?:[ \t]-[fF]\b|--(?:raw-)?field\b)))' min: 0 max: 0 arm: both diff --git a/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json index ae549a00a2e..6b065d143f1 100644 --- a/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json +++ b/plugins/work-items/evals/tr-numbered-states-read-first/samples/no-tracker-write.json @@ -25,6 +25,28 @@ } ], "why": "reads the item through the API, then searches on its own line" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api -X GET search/issues -f q='repo:o/r is:issue CSV'" + } + } + ], + "why": "searches through the API with an explicit GET and a field" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "gh api graphql -f query='{ repository(owner:\"o\",name:\"r\"){ issue(number:31){ title labels(first:5){nodes{name}} } } }'" + } + } + ], + "why": "reads the item through a GraphQL query" } ], "fail": [