From 8135daa7e89a281243752ec5d16b260054278c08 Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:51:02 +0000 Subject: [PATCH 1/6] Resolve issue #11874 Lane-Activity: work-resolve-issue/resolve-issue --- ...work-resolve-issue-issue-11874-5d2e8b17.md | 7 + docs/catalog.md | 2 +- docs/skill-cheat-sheet.md | 1 + plugins/work-items/.claude-plugin/plugin.json | 2 +- plugins/work-items/README.md | 10 +- .../work-items/schemas/work-items.schema.json | 8 +- .../skills/measure-delivery/SKILL.md | 106 ++++++++ .../scripts/delivery-metrics.sh | 252 ++++++++++++++++++ .../scripts/delivery-metrics.test.sh | 177 ++++++++++++ 9 files changed, 561 insertions(+), 4 deletions(-) create mode 100644 .changes/work-items/work-resolve-issue-issue-11874-5d2e8b17.md create mode 100644 plugins/work-items/skills/measure-delivery/SKILL.md create mode 100755 plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh create mode 100755 plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh diff --git a/.changes/work-items/work-resolve-issue-issue-11874-5d2e8b17.md b/.changes/work-items/work-resolve-issue-issue-11874-5d2e8b17.md new file mode 100644 index 00000000000..6bda99cc7ba --- /dev/null +++ b/.changes/work-items/work-resolve-issue-issue-11874-5d2e8b17.md @@ -0,0 +1,7 @@ +--- +bump: minor +--- + +### Added + +- `/work-items:measure-delivery`: a read-only report of the five DORA delivery metrics (deployment frequency, change lead time, change fail rate, failed deployment recovery time, deployment rework rate) as a weekly or monthly team trend, read from the default branch's git history and never broken down by person (#11874). The repository setting `delivery_deploy_signal` in `docs/conventions/work-items.yaml` sets what counts as a deployment; its one value and default, `merge-to-default-branch`, counts each commit landing on the default branch, and one a later commit reverts counts as a failed deployment. diff --git a/docs/catalog.md b/docs/catalog.md index f535990efbe..e3033c1888c 100644 --- a/docs/catalog.md +++ b/docs/catalog.md @@ -123,7 +123,7 @@ plugin manifests and kept in sync by CI. Never hand-edit it; the category vocabu ## Project Management -- [`work-items`](../plugins/work-items): Work items through a provider-neutral tracker seam (github, local-markdown, jira, gitea, linear adapters): track (dashboard, creation, race-safe claims, recurring-schedule checks, stale-lease audits), scan-todos, decompose (plans into vertical-slice items), ship (route and run one spec container), triage (raw intake and unsolicited PRs), work, work-loop (autonomous PR-only drain), attend-queue (escalations), onboard-adapter (new tracker adapter), and setup (binds the provider). +- [`work-items`](../plugins/work-items): Work items through a provider-neutral tracker seam (github, local-markdown, jira, gitea, linear adapters): track (dashboard, creation, race-safe claims, recurring checks, stale-lease audits), scan-todos, decompose (plans into vertical slices), ship (run one spec container), triage (raw intake and unsolicited PRs), work, work-loop (autonomous PR-only drain), attend-queue (escalations), onboard-adapter (new tracker adapter), measure-delivery (DORA metrics), and setup (binds the provider). ## Operations diff --git a/docs/skill-cheat-sheet.md b/docs/skill-cheat-sheet.md index d3c46d3e34d..254e93278e5 100644 --- a/docs/skill-cheat-sheet.md +++ b/docs/skill-cheat-sheet.md @@ -341,6 +341,7 @@ owned by [docs/catalog-taxonomy.md](catalog-taxonomy.md). | [`/visualization:visualize`](../plugins/visualization/skills/visualize/SKILL.md) | `visualization` | Pick the best visual form for what is in the conversation and render it | | [`/wizard:generate`](../plugins/wizard/skills/generate/SKILL.md) | `wizard` | Author a hardened interactive bash wizard for human-only setup, credential, and cutover steps | | [`/wizard:unattended`](../plugins/wizard/skills/unattended/SKILL.md) | `wizard` | Author an unattended script a human launches once for a privilege or policy boundary | +| [`/work-items:measure-delivery`](../plugins/work-items/skills/measure-delivery/SKILL.md) | `work-items` | Report the five DORA delivery metrics as a team trend from default-branch history | | [`/work-items:scan-todos`](../plugins/work-items/skills/scan-todos/SKILL.md) | `work-items` | Sweep source comments for TODO and FIXME markers, resolve or file each | | [`/work-items:ship`](../plugins/work-items/skills/ship/SKILL.md) | `work-items` | Route and run a spec container's macro journey. Status, execution shape, next step | | [`/work-items:track`](../plugins/work-items/skills/track/SKILL.md) | `work-items` | Backlog CRUD through the bound tracker. Add, list, close, stats | diff --git a/plugins/work-items/.claude-plugin/plugin.json b/plugins/work-items/.claude-plugin/plugin.json index 432dc0a72ad..8b4707b89fb 100644 --- a/plugins/work-items/.claude-plugin/plugin.json +++ b/plugins/work-items/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "work-items", "version": "3.15.3", - "description": "Work items through a provider-neutral tracker seam (github, local-markdown, jira, gitea, linear adapters): track (dashboard, creation, race-safe claims, recurring-schedule checks, stale-lease audits), scan-todos, decompose (plans into vertical-slice items), ship (route and run one spec container), triage (raw intake and unsolicited PRs), work, work-loop (autonomous PR-only drain), attend-queue (escalations), onboard-adapter (new tracker adapter), and setup (binds the provider).", + "description": "Work items through a provider-neutral tracker seam (github, local-markdown, jira, gitea, linear adapters): track (dashboard, creation, race-safe claims, recurring checks, stale-lease audits), scan-todos, decompose (plans into vertical slices), ship (run one spec container), triage (raw intake and unsolicited PRs), work, work-loop (autonomous PR-only drain), attend-queue (escalations), onboard-adapter (new tracker adapter), measure-delivery (DORA metrics), and setup (binds the provider).", "author": { "name": "Melodic Software", "email": "info@melodicsoftware.com" diff --git a/plugins/work-items/README.md b/plugins/work-items/README.md index 20d083c9064..7003f62f721 100644 --- a/plugins/work-items/README.md +++ b/plugins/work-items/README.md @@ -6,7 +6,7 @@ file-based TODO lists, designed for teams where humans and autonomous agents pick work from the same queue. The skill core is backend-agnostic; GitHub is the bound adapter today. -The tracker's capabilities are split across nine focused skills (plus a setup +The tracker's capabilities are split across ten focused skills (plus a setup skill). Invoke the one that matches the job (or let Claude invoke it when you ask about work items, tickets, issues, tracked work, or what to do next): @@ -20,6 +20,7 @@ about work items, tickets, issues, tracked work, or what to do next): /work-items:ship run #300 # run the container's ready items under its shape (attended) /work-items:scan-todos # sweep TODO/FIXME/HACK markers /work-items:onboard-adapter gitea # generate an adapter for an unbundled tracker +/work-items:measure-delivery # DORA delivery metrics as a weekly team trend ``` ## Skills @@ -34,6 +35,7 @@ about work items, tickets, issues, tracked work, or what to do next): | `/work-items:decompose` | Break a plan/PRD/item into vertical-slice items with AFK/HITL classification and dependency ordering. When a spec container is published and the design behind the plan produced typed artifacts (frontmatter carrying `scope` and `dialect`), each is inlined into the container body as a fenced block plus a one-line provenance note naming the scope and dialect, with no file path; an `integration` design labels two, and both are inlined. Slice bodies are unchanged. | | `/work-items:ship` | Macro-journey router over one spec container: rollup + scoped frontier, the container's recorded execution shape (per-item PRs vs integration branch → single PR) with that mode's discipline, and the routed next step. `run` works the container's ready items in parallel through `/implementation:implement-dispatch` and lands them one at a time, attended only. Mechanics stay with their owners. | | `/work-items:scan-todos` | Sweep the codebase for TODO/FIXME/HACK markers; resolve or file each. | +| `/work-items:measure-delivery` | Read-only report of the five DORA delivery metrics (deployment frequency, change lead time, change fail rate, failed deployment recovery time, deployment rework rate) as a team trend from the default branch's git history, never per person. | | `/work-items:onboard-adapter` | Onboard a tracker this plugin does not bundle: interview to lock the provider's shape, explore the consumer's real instance for the per-instance facts only it can settle, generate a consumer-owned adapter (hardened security skeleton, honest capability manifest, contract-fixed verb scaffolds, conformance binding) into the consuming repo, then verify. The tail half of the hybrid adapter model. Bundled adapters cover the majors. | | `/work-items:setup` | `check` inspects the tracked `.github/recurring-schedule.json`, the jq/tracker-seam entry gates, and the recurring-maintenance role label read-only; `apply` binds the provider, writes the empty schedule skeleton, and offers the canonical-role → label remap in the tracker binding (re-runnable). Seeding actual rows, inferring candidate items from the repo and interviewing per item, is opt-in via `apply --seed-schedule` or an offer that recommends skipping; a schedule that already carries items is offered updates as before. | @@ -210,6 +212,12 @@ work_priority_order: The work skill's "Selection Priority" section says how it is read. +`delivery_deploy_signal` also lives only in the repository file. It sets what +`/work-items:measure-delivery` counts as a deployment, and its one value, the +default, is `merge-to-default-branch`: each commit on the default branch's +first-parent history is a deployment, and one a later commit reverts is a +failed deployment. + Everything else is project-specific behavior that routes through the consuming repo's own surfaces: the bound provider in `.work-item-tracker.json` (including the optional `config.role_labels` canonical-role → label remap), its labels diff --git a/plugins/work-items/schemas/work-items.schema.json b/plugins/work-items/schemas/work-items.schema.json index e82f72edd0b..47c8ae6b85e 100644 --- a/plugins/work-items/schemas/work-items.schema.json +++ b/plugins/work-items/schemas/work-items.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/melodic-software/claude-code-plugins/main/plugins/work-items/schemas/work-items.schema.json", "title": "work-items team config", - "description": "Schema for a repository's docs/conventions/work-items.yaml, the team layer of the work-items plugin's settings (ADR 0064). A key set here wins over the same key in the plugin's userConfig. Readers: /work-items:triage for the triage keys (the /work-items:work-loop admission gate reads the objection window's end from the marker comment triage posts), every lane that follows reference/ai-disclaimer.md for ai_disclaimer, and /work-items:work for work_priority_order.", + "description": "Schema for a repository's docs/conventions/work-items.yaml, the team layer of the work-items plugin's settings (ADR 0064). A key set here wins over the same key in the plugin's userConfig. Readers: /work-items:triage for the triage keys (the /work-items:work-loop admission gate reads the objection window's end from the marker comment triage posts), every lane that follows reference/ai-disclaimer.md for ai_disclaimer, /work-items:work for work_priority_order, and /work-items:measure-delivery for delivery_deploy_signal.", "type": "object", "additionalProperties": false, "properties": { @@ -31,6 +31,12 @@ "uniqueItems": true, "default": ["priority: critical", "priority: high", "priority: medium", "priority: low"], "description": "The priority: label names, highest first, that /work-items:work ranks candidates by within each selection tier before the tier's own order (next_due, the guardrails preference, or oldest-first) breaks ties; the work-loop lane ranks its admitted items the same way. A candidate with no priority: label, or one this list does not name, ranks below every listed label. Quote each item (- \"priority: high\"): an unquoted item parses as a mapping and drops out of the list. Absent: the default list." + }, + "delivery_deploy_signal": { + "type": "string", + "enum": ["merge-to-default-branch"], + "default": "merge-to-default-branch", + "description": "What /work-items:measure-delivery counts as one deployment. merge-to-default-branch: each commit on the default branch's first-parent history (a merge commit, or a squash or rebase landing) is one deployment, and one a later commit reverts is a failed deployment. Absent: merge-to-default-branch." } } } diff --git a/plugins/work-items/skills/measure-delivery/SKILL.md b/plugins/work-items/skills/measure-delivery/SKILL.md new file mode 100644 index 00000000000..8a7dd368e6a --- /dev/null +++ b/plugins/work-items/skills/measure-delivery/SKILL.md @@ -0,0 +1,106 @@ +--- +description: "Read-only report of the five DORA delivery metrics (deployment frequency, change lead time, change fail rate, failed deployment recovery time, deployment rework rate) as a team trend from the default branch's history, never per person. Use when: 'measure delivery', 'DORA metrics', 'deployment frequency', 'change lead time', 'change fail rate', 'how fast do we ship', 'delivery trend'. Backlog counts are /work-items:track stats." +argument-hint: "[--since YYYY-MM-DD] [--until YYYY-MM-DD] [--bucket week|month] [--branch ]" +user-invocable: true +disable-model-invocation: false +metadata: + workflow-stage: anytime + summary: Report the five DORA delivery metrics as a team trend from default-branch history +--- + +## Variables + +Arguments: `$ARGUMENTS`, the window and period flags below, passed through to the script. + +- `--since `: first UTC day of the window (default: the 84 days ending with `--until`). +- `--until `: last UTC day, inclusive (default: now). +- `--bucket week|month`: trend period (default `week`, ISO weeks starting Monday). +- `--branch `: the default branch (default: the ref `origin/HEAD` names). + +## Purpose + +Show how the team delivers over time: one row per period and a window total, for the five +metrics DORA defines (, read 2026-11-01). DORA applies +them to one application or service and to a team improving against its own past, not against +other teams; this skill reports them the same way. It reads history and writes nothing. + +## Metrics + +| Metric | Measured here as | +|---|---| +| Deployment frequency | Deployments per period, and per week over the window | +| Change lead time | Median hours from the earliest author date among a deployment's commits to its landing | +| Change fail rate | Failed deployments / deployments | +| Failed deployment recovery time | Median hours from a failed deployment to the first deployment that reverts it | +| Deployment rework rate | Deployments that revert an earlier change / deployments | + +## Deploy signal + +What counts as a deployment is the team setting `delivery_deploy_signal` in the repository's +`docs/conventions/work-items.yaml` (schema: `${CLAUDE_PLUGIN_ROOT}/schemas/work-items.schema.json`). +Its one value, and the default, is `merge-to-default-branch`: + +- Each commit on the default branch's first-parent history is one deployment, at its committer + date: a merge commit, or a squash or rebase landing. +- A deployment a later first-parent commit reverts is a failed deployment. A commit reverts a + deployment when its message has `This reverts commit ` naming the deployment or a commit + it merged, or, with no such line, when its subject is `Revert ""` for an earlier + deployment's subject. A merge counts the messages of the commits it merges, so a revert that + lands through a merge commit is seen. A revert landing after `--until` still marks a + deployment in the window as failed. + +The signal reads only git history, so no tracker or forge call is made and any host works. + +## Workflow + +1. Resolve the deploy signal from the repository root: + + ```bash + bash "${CLAUDE_PLUGIN_ROOT}/skills/triage/scripts/parse-concern-value.sh" \ + docs/conventions/work-items.yaml delivery_deploy_signal + ``` + + Empty output means the key is unset: use the default, `merge-to-default-branch`. Report + whether the value came from the file or the default. A value other than + `merge-to-default-branch` stops the run: name the value and the one the schema allows. + +2. Run the report. With no `--branch`, the script reads the ref `origin/HEAD` names, as of the + last fetch. Run `git fetch` first only when the user asks for the remote's latest history; + otherwise say the report covers history up to the last fetch. + + ```bash + bash "${CLAUDE_PLUGIN_ROOT}/skills/measure-delivery/scripts/delivery-metrics.sh" \ + --repo "" --deploy-signal "" + ``` + + Exit `2` is a bad flag or a branch that does not resolve: show the script's stderr line, + and when `origin/HEAD` is unset, ask for `--branch`. Exit `3` is a failed git read and exit + `8` a missing `git` or `jq`: report it and stop. + +3. Render the JSON as one markdown table, a row per `trend` period then a `Window` row from + `totals`, a column per metric. Show a null value as `n/a` (nothing to measure, such as a + period with no deployments), and show rates as percentages. Mark the first or last period + as partial when the window starts or ends inside it. + +4. Add two or three lines that read the trend: which metrics moved between the first and last + full periods with deployments, and in which direction. Describe the team's delivery only. + +Done when the table and the trend lines are shown, with the window, branch and deploy signal +they cover. + +## Gotchas + +- **Never name a person.** The script emits counts, medians and ratios only. Do not add + authors, committers, reviewers or a per-person breakdown, even when asked: DORA metrics + describe a team's delivery, and per-person delivery numbers are a different report. +- **Squash and rebase landings can shorten lead time.** Lead time starts at the earliest author + date git keeps for the change. A landing that rewrites author dates moves that start later, + so the lead time reads short; a merge commit keeps every commit's author date. +- **A revert under another subject goes uncounted.** A hand-written undo with neither the + `This reverts commit` line nor the `Revert ""` subject is invisible to the signal. +- **Per-item stage times are out of scope.** Time an item spent in each tracker stage is a + separate report, not one of these five metrics. + +## Next + +`/work-items:track stats` for the open backlog behind the trend. diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh new file mode 100755 index 00000000000..1fd6a3cfdf3 --- /dev/null +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +# delivery-metrics.sh: the five DORA delivery metrics for one repository's +# default branch, as a team trend, read from git history alone. +# +# Deploy signal merge-to-default-branch: each commit on the branch's +# first-parent history (a merge commit, or a squash or rebase landing) is one +# deployment, at its committer date. A deployment that a later first-parent +# commit reverts is a failed deployment. A commit reverts a deployment when its +# message carries `This reverts commit ` for the deployment or for a +# commit the deployment merged, or, with no such line, when its subject is +# `Revert ""` for an earlier deployment's subject. A merge's message +# includes the messages of the commits it merges, so a revert that lands +# through a merge commit is seen. Every deployment that reverts an earlier +# change is a rework deployment. +# +# Change lead time is from the earliest author date among the commits a +# deployment brings in to the deployment's committer date. Failed deployment +# recovery time is from a failed deployment to the first deployment that +# reverts it; reverts after --until still count. The output carries counts, +# medians and ratios only: no commit, subject, author or committer. +# +# Usage: +# delivery-metrics.sh [--repo ] [--branch ] [--since ] +# [--until ] [--bucket week|month] +# [--deploy-signal merge-to-default-branch] +# +# --repo repository directory (default: the current directory) +# --branch the default branch ref (default: what origin/HEAD names) +# --since first UTC day of the window (default: the 84 days ending with --until) +# --until last UTC day of the window, inclusive (default: now) +# --bucket trend period: week (ISO weeks, Monday start) or month +# --deploy-signal what counts as a deployment; only merge-to-default-branch +# +# Output: one JSON document on stdout, with `totals` for the window and one +# `trend` row per period, empty periods included. Times are hours, rates are +# ratios from 0 to 1, and a metric with nothing to measure is null. +# +# Exit codes: +# 0 report written +# 2 usage: a missing, repeated or unknown argument, or a malformed value +# 3 a git read failed +# 8 prerequisite missing: git or jq is not on PATH + +set -uo pipefail + +err() { printf 'delivery-metrics: %s\n' "$*" >&2; } +usage() { awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "${BASH_SOURCE[0]}"; } + +REPO="." BRANCH="" SINCE="" UNTIL="" BUCKET="" SIGNAL="" SEEN=" " +while (($#)); do + case "$1" in + --repo | --branch | --since | --until | --bucket | --deploy-signal) + (($# >= 2)) || { + err "$1 requires a value" + exit 2 + } + [[ "$SEEN" != *" $1 "* ]] || { + err "$1 given twice" + exit 2 + } + SEEN+="$1 " + case "$1" in + --repo) REPO="$2" ;; + --branch) BRANCH="$2" ;; + --since) SINCE="$2" ;; + --until) UNTIL="$2" ;; + --bucket) BUCKET="$2" ;; + --deploy-signal) SIGNAL="$2" ;; + *) ;; + esac + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + err "unknown argument: $1" + exit 2 + ;; + esac +done + +SIGNAL="${SIGNAL:-merge-to-default-branch}" +[[ "$SIGNAL" == merge-to-default-branch ]] || { + err "--deploy-signal must be merge-to-default-branch" + exit 2 +} +BUCKET="${BUCKET:-week}" +[[ "$BUCKET" == week || "$BUCKET" == month ]] || { + err "--bucket must be week or month" + exit 2 +} +for tool in git jq; do + command -v "$tool" >/dev/null 2>&1 || { + err "$tool is not on PATH" + exit 8 + } +done +[[ -d "$REPO" ]] || { + err "--repo is not a directory: $REPO" + exit 2 +} + +# day_epoch : the UTC midnight that starts the day, as epoch seconds. +day_epoch() { + [[ "$1" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || return 1 + jq -n --arg d "$1" '$d + "T00:00:00Z" | fromdateiso8601' 2>/dev/null +} +if [[ -n "$UNTIL" ]]; then + until_start="$(day_epoch "$UNTIL")" || { + err "--until must be a YYYY-MM-DD date" + exit 2 + } + UNTIL_EPOCH=$((until_start + 86400)) +else + UNTIL_EPOCH="$(date +%s)" +fi +if [[ -n "$SINCE" ]]; then + SINCE_EPOCH="$(day_epoch "$SINCE")" || { + err "--since must be a YYYY-MM-DD date" + exit 2 + } +else + SINCE_EPOCH=$(((UNTIL_EPOCH - 84 * 86400) / 86400 * 86400)) +fi +((SINCE_EPOCH < UNTIL_EPOCH)) || { + err "--since must come before --until" + exit 2 +} + +if [[ -z "$BRANCH" ]]; then + BRANCH="$(git -C "$REPO" symbolic-ref --quiet --short refs/remotes/origin/HEAD 2>/dev/null)" || { + err "origin/HEAD names no default branch; pass --branch" + exit 2 + } +fi +[[ "$BRANCH" =~ ^[A-Za-z0-9._/-]+$ && "$BRANCH" != -* ]] || { + err "--branch is not a usable ref name: $BRANCH" + exit 2 +} +git -C "$REPO" rev-parse --verify --quiet "$BRANCH^{commit}" >/dev/null || { + err "--branch does not resolve to a commit: $BRANCH" + exit 2 +} + +TMP="$(mktemp -d)" || exit 3 +trap 'rm -rf "$TMP"' EXIT + +# Every first-parent commit, newest first: sha, parents, author and committer +# epochs, subject, body. Fields end in US (0x1f), records in RS (0x1e). +git -C "$REPO" log --first-parent --format='%H%x1f%P%x1f%at%x1f%ct%x1f%s%x1f%b%x1e' \ + "$BRANCH" -- >"$TMP/deploys" || { + err "git log failed on $BRANCH" + exit 3 +} + +# The commits each merge in the window brought in: deploy sha, member sha, +# member author epoch, tab-separated; and each member's message, so a revert +# that lands through a merge commit is seen. +git -C "$REPO" log --first-parent --merges --format='%H %ct %P' "$BRANCH" -- >"$TMP/merges" || { + err "git log failed listing the merges on $BRANCH" + exit 3 +} +: >"$TMP/members" +: >"$TMP/member_msgs" +while read -r sha ct first rest; do + ((ct >= SINCE_EPOCH)) || continue + read -r -a others <<<"$rest" + if ! git -C "$REPO" log --format="$sha%x09%H%x09%at" "${others[@]}" "^$first" -- >>"$TMP/members" || + ! git -C "$REPO" log --format="$sha%x1f%B%x1e" "${others[@]}" "^$first" -- >>"$TMP/member_msgs"; then + err "git log failed for the commits merged by $sha" + exit 3 + fi +done <"$TMP/merges" + +jq -n \ + --rawfile raw "$TMP/deploys" \ + --rawfile members "$TMP/members" \ + --rawfile member_msgs "$TMP/member_msgs" \ + --argjson since "$SINCE_EPOCH" \ + --argjson until "$UNTIL_EPOCH" \ + --arg bucket "$BUCKET" \ + --arg branch "$BRANCH" \ + --arg signal "$SIGNAL" ' +def median: sort | length as $n + | if $n == 0 then null + elif $n % 2 == 1 then .[($n - 1) / 2] + else (.[$n / 2 - 1] + .[$n / 2]) / 2 end; +def hours: if . == null then null else (. / 36 | round) / 100 end; +def ratio($part; $whole): if $whole == 0 then null else ($part / $whole * 10000 | round) / 10000 end; +def period: + if $bucket == "week" + then (345600 + (((. - 345600) / 604800) | floor) * 604800) | todate | .[0:10] + else todate | .[0:7] end; +def stats: { + deployments: length, + change_lead_time_hours_median: (map(.lead) | median | hours), + failed_deployments: (map(select(.failed)) | length), + change_fail_rate: ratio(map(select(.failed)) | length; length), + failed_deployment_recovery_time_hours_median: + (map(select(.recovery != null) | .recovery) | median | hours), + rework_deployments: (map(select(.rework)) | length), + deployment_rework_rate: ratio(map(select(.rework)) | length; length) +}; + +($raw | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f") + | {sha: .[0], parents: (.[1] | split(" ") | map(select(length > 0))), + at: (.[2] | tonumber), ct: (.[3] | tonumber), subject: (.[4] // ""), body: (.[5] // "")}) + | map(select(.ct >= $since))) as $scope +| ($members | split("\n") | map(select(length > 0) | split("\t") + | {deploy: .[0], sha: .[1], at: (.[2] | tonumber)})) as $mem +| (reduce $scope[] as $d ({}; .[$d.sha] = $d.sha) + | reduce $mem[] as $m (.; .[$m.sha] //= $m.deploy)) as $owner +| (reduce $mem[] as $m ({}; .[$m.deploy] += [$m.at])) as $member_at +| (reduce ($member_msgs | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f")))[] as $m + ({}; .[$m[0]] += "\n" + ($m[1] // ""))) as $member_text +| ($scope | map(. as $r + | ([.body + "\n" + ($member_text[.sha] // "") | scan("This reverts (?:commit )?([0-9a-f]{40})") | .[0]] + | unique | map(select(($owner[.] // "") != $r.sha))) as $named + | ($named | map($owner[.] // empty) | unique) as $by_sha + | ([.subject | capture("^Revert \"(?.*)\"") | .s] | first) as $reverted_subject + | (if ($named | length) > 0 then $by_sha + elif $reverted_subject == null then [] + else [$scope | map(select(.subject == $reverted_subject and .ct <= $r.ct and .sha != $r.sha)) + | max_by(.ct) | .sha | select(. != null)] + end) as $targets + | . + {targets: $targets, rework: (($named | length) > 0 or $reverted_subject != null)})) as $graded +| ($graded | map(select(.ct < $until)) | map(. as $d + | ([$graded[] | select(.ct >= $d.ct and any(.targets[]; . == $d.sha)) | .ct] | min) as $fixed_at + | {ct, + rework, + lead: ([(([$d.at] + ($member_at[$d.sha] // [])) | min) as $start | $d.ct - $start, 0] | max), + failed: ($fixed_at != null), + recovery: (if $fixed_at == null then null else $fixed_at - $d.ct end), + period: (.ct | period)})) as $reported +| (($until - $since) / 86400) as $days +| { + deploy_signal: $signal, + branch: $branch, + bucket: $bucket, + since: ($since | todate | .[0:10]), + until: ($until - 1 | todate | .[0:10]), + totals: ($reported | stats + + {deployments_per_week: ((length * 7 / $days * 100 | round) / 100)}), + trend: ([range($since; $until; 86400) | period] | unique + | map(. as $p | {period: $p} + ($reported | map(select(.period == $p)) | stats))) + } +' || { + err "could not compute the report" + exit 3 +} diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh new file mode 100755 index 00000000000..4f4f3664dd5 --- /dev/null +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Offline tests for delivery-metrics.sh over a fixture repository of merges and +# reverts with fixed author and committer dates. Each expected value is worked +# out by hand from the fixture below, in hours from T0 (Monday 2026-01-05 UTC). +# +# deploy author landed lead reverted by (landed) recovery +# A merge 0 10 10 R3, a commit A merged (216) 206 +# B 24 28 4 R4, after the window (360) 332 +# C merge 48 54 6 R1, `This reverts` (56) 2 +# R1 55 56 1 +# D 168 170 2 +# E 192 195 3 R2, subject only (204) 9 +# R2 201 204 3 +# R3 216 216 0 +# R4 360 360 (after --until, not reported) +# +# Week 2026-01-05: 4 deployments, lead median 5, fail rate 3/4, recovery median +# 206, rework 1/4. Week 2026-01-12: 4 deployments, lead median 2.5, fail rate +# 1/4, recovery 9, rework 2/4. Window: 8 deployments over 14 days (4 a week), +# lead median 3, fail rate 4/8, recovery median 107.5, rework 3/8. +set -uo pipefail +unset GIT_DIR GIT_WORK_TREE GIT_CONFIG GIT_INDEX_FILE + +S="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/delivery-metrics.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +FAILED=0 + +pass() { printf 'PASS: %s\n' "$1"; } +fail() { + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n' "$*" >&2 +} + +for tool in git jq; do + command -v "$tool" >/dev/null 2>&1 || { + echo "SKIP: $tool not installed" >&2 + exit 0 + } +done + +export GIT_AUTHOR_NAME="Ada Lovelace" GIT_AUTHOR_EMAIL="ada@person.example" +export GIT_COMMITTER_NAME="Grace Hopper" GIT_COMMITTER_EMAIL="grace@person.example" +R="$TMP/repo" +H=3600 +T0=1767571200 + +g() { git -C "$R" -c commit.gpgsign=false -c core.hooksPath=/dev/null "$@"; } +# at : run git with both dates fixed. +at() { + local a=$((T0 + $1 * H)) c=$((T0 + $2 * H)) + shift 2 + GIT_AUTHOR_DATE="$a +0000" GIT_COMMITTER_DATE="$c +0000" g "$@" >/dev/null || { + echo "fixture: git $* failed" >&2 + exit 1 + } +} +# change : commit one new file. +change() { + printf '%s\n' "$3" >"$R/$3" + g add -- "$3" + at "$1" "$2" commit -q -m "$4" +} +sha() { g rev-parse "$1"; } + +mkdir -p "$R" +git init -q -b main "$R" +change -240 -240 root.txt "chore: root" + +g checkout -q -b feat-a +change 0 0 a1.txt "feat: a1" +change 2 2 a2.txt "feat: a2" +A2="$(sha HEAD)" +g checkout -q main +at 10 10 merge -q --no-ff --no-edit feat-a + +change 24 28 b.txt "fix: b" +B="$(sha HEAD)" + +g checkout -q -b feat-c +change 48 48 c1.txt "feat: c1" +g checkout -q main +at 54 54 merge -q --no-ff --no-edit feat-c +at 55 56 revert --no-edit -m 1 HEAD + +change 168 170 d.txt "feat: d" +change 192 195 e.txt "feat: e" +g rm -q -- e.txt +at 201 204 commit -q -m 'Revert "feat: e"' +at 216 216 revert --no-edit "$A2" +at 360 360 revert --no-edit "$B" + +# Week 2026-02-02: F (lead 0) is reverted by a commit on a branch (authored at +# 673) that lands through a merge at 677 (lead 4): recovery 5, fail rate 1/2, +# rework 1/2. +change 672 672 f.txt "feat: f" +F="$(sha HEAD)" +g checkout -q -b revert-f +at 673 673 revert --no-edit "$F" +g checkout -q main +at 677 677 merge -q --no-ff --no-edit revert-f + +run() { bash "$S" --repo "$R" --branch main --since 2026-01-05 --until 2026-01-18 "$@"; } + +if OUT="$(run)"; then + pass "report exits 0" +else + fail "report exited non-zero" + OUT='{}' +fi + +expect() { # + local got + got="$(jq -c "$2" <<<"$OUT" 2>&1)" + if [[ "$got" == "$3" ]]; then pass "$1"; else fail "$1: expected $3, got $got"; fi +} + +expect "deploy signal" '.deploy_signal' '"merge-to-default-branch"' +expect "window" '[.since, .until]' '["2026-01-05","2026-01-18"]' +expect "deployment frequency: window count" '.totals.deployments' '8' +expect "deployment frequency: per week" '.totals.deployments_per_week' '4' +expect "change lead time: window median" '.totals.change_lead_time_hours_median' '3' +expect "change fail rate: window" '.totals.change_fail_rate' '0.5' +expect "failed deployments: window" '.totals.failed_deployments' '4' +expect "recovery time: window median" '.totals.failed_deployment_recovery_time_hours_median' '107.5' +expect "rework rate: window" '.totals.deployment_rework_rate' '0.375' +expect "trend periods" '[.trend[].period]' '["2026-01-05","2026-01-12"]' +expect "week 1 deployment frequency" '.trend[0].deployments' '4' +expect "week 1 change lead time" '.trend[0].change_lead_time_hours_median' '5' +expect "week 1 change fail rate" '.trend[0].change_fail_rate' '0.75' +expect "week 1 recovery time" '.trend[0].failed_deployment_recovery_time_hours_median' '206' +expect "week 1 rework rate" '.trend[0].deployment_rework_rate' '0.25' +expect "week 2 deployment frequency" '.trend[1].deployments' '4' +expect "week 2 change lead time" '.trend[1].change_lead_time_hours_median' '2.5' +expect "week 2 change fail rate" '.trend[1].change_fail_rate' '0.25' +expect "week 2 recovery time" '.trend[1].failed_deployment_recovery_time_hours_median' '9' +expect "week 2 rework rate" '.trend[1].deployment_rework_rate' '0.5' + +if grep -Eqi 'ada|lovelace|grace|hopper|person\.example' <<<"$OUT"; then + fail "output names a person" +else + pass "output names no person" +fi + +OUT="$(run --bucket month)" || fail "month report exited non-zero" +expect "month bucket: one period" '[.trend[] | {period, deployments}]' '[{"period":"2026-01","deployments":8}]' + +OUT="$(bash "$S" --repo "$R" --branch main --since 2026-02-02 --until 2026-02-08)" || fail "merged-revert window exited non-zero" +expect "revert landed through a merge commit" \ + '.totals | [.deployments, .change_lead_time_hours_median, .change_fail_rate, .failed_deployment_recovery_time_hours_median, .deployment_rework_rate]' \ + '[2,2,0.5,5,0.5]' + +OUT="$(bash "$S" --repo "$R" --branch main --since 2026-03-02 --until 2026-03-08)" || fail "empty window exited non-zero" +expect "empty window: nothing to measure is null" \ + '[.totals.deployments, .totals.change_lead_time_hours_median, .totals.change_fail_rate]' '[0,null,null]' + +usage_error() { # + local name="$1" + shift + bash "$S" "$@" >/dev/null 2>&1 + local rc=$? + if [[ "$rc" == 2 ]]; then pass "usage error: $name"; else fail "usage error: $name exited $rc, expected 2"; fi +} +usage_error "unknown deploy signal" --repo "$R" --branch main --deploy-signal release-tag +usage_error "unknown bucket" --repo "$R" --branch main --bucket day +usage_error "malformed date" --repo "$R" --branch main --since 2026/01/05 +usage_error "since after until" --repo "$R" --branch main --since 2026-02-01 --until 2026-01-01 +usage_error "unknown branch" --repo "$R" --branch nope +usage_error "no origin/HEAD and no --branch" --repo "$R" +usage_error "argument given twice" --repo "$R" --repo "$R" +usage_error "unknown argument" --repo "$R" --verbose + +((FAILED == 0)) || { + echo "$FAILED failed" >&2 + exit 1 +} +echo "all passed" From 00cfb3a084b0fd076cdd50af3783b87b002953ac Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:07:50 +0000 Subject: [PATCH 2/6] Address review feedback on work-resolve-issue/issue-11874-41281707694-1 Lane-Activity: pr-unblock/address-feedback Lane-Thread: PRRT_kwDOTCGFQM6yYWQH Lane-Thread: PRRT_kwDOTCGFQM6yYWQN Lane-Thread: PRRT_kwDOTCGFQM6yYWQU --- .../work-items/schemas/work-items.schema.json | 2 +- .../skills/measure-delivery/SKILL.md | 23 +++++++++++------- .../scripts/delivery-metrics.sh | 5 ++-- .../scripts/delivery-metrics.test.sh | 24 +++++++++++++++++++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/plugins/work-items/schemas/work-items.schema.json b/plugins/work-items/schemas/work-items.schema.json index 47c8ae6b85e..969777dff1f 100644 --- a/plugins/work-items/schemas/work-items.schema.json +++ b/plugins/work-items/schemas/work-items.schema.json @@ -36,7 +36,7 @@ "type": "string", "enum": ["merge-to-default-branch"], "default": "merge-to-default-branch", - "description": "What /work-items:measure-delivery counts as one deployment. merge-to-default-branch: each commit on the default branch's first-parent history (a merge commit, or a squash or rebase landing) is one deployment, and one a later commit reverts is a failed deployment. Absent: merge-to-default-branch." + "description": "What /work-items:measure-delivery counts as one deployment. merge-to-default-branch: each commit on the default branch's first-parent history is one deployment (a merge commit or a squash landing is one; a rebase landing is one per commit it lands), and one a later commit reverts is a failed deployment. Absent: merge-to-default-branch." } } } diff --git a/plugins/work-items/skills/measure-delivery/SKILL.md b/plugins/work-items/skills/measure-delivery/SKILL.md index 8a7dd368e6a..016c8042fe9 100644 --- a/plugins/work-items/skills/measure-delivery/SKILL.md +++ b/plugins/work-items/skills/measure-delivery/SKILL.md @@ -20,9 +20,12 @@ Arguments: `$ARGUMENTS`, the window and period flags below, passed through to th ## Purpose Show how the team delivers over time: one row per period and a window total, for the five -metrics DORA defines (, read 2026-11-01). DORA applies -them to one application or service and to a team improving against its own past, not against -other teams; this skill reports them the same way. It reads history and writes nothing. +delivery metrics DORA defines. This skill reports one repository's trend against its own past, +never a comparison between teams. It reads history and writes nothing. + +- **Pointer**: when a metric's definition is in question, fetch + live. **As of**: 2026-11-01. **Recheck trigger**: DORA + renames, adds or removes a delivery metric. ## Metrics @@ -41,7 +44,8 @@ What counts as a deployment is the team setting `delivery_deploy_signal` in the Its one value, and the default, is `merge-to-default-branch`: - Each commit on the default branch's first-parent history is one deployment, at its committer - date: a merge commit, or a squash or rebase landing. + date. A merge commit or a squash landing is one deployment; a rebase landing puts each of its + commits on that history, so each one counts as its own deployment. - A deployment a later first-parent commit reverts is a failed deployment. A commit reverts a deployment when its message has `This reverts commit ` naming the deployment or a commit it merged, or, with no such line, when its subject is `Revert ""` for an earlier @@ -88,6 +92,10 @@ The signal reads only git history, so no tracker or forge call is made and any h Done when the table and the trend lines are shown, with the window, branch and deploy signal they cover. +## Next + +`/work-items:track stats` for the open backlog behind the trend. + ## Gotchas - **Never name a person.** The script emits counts, medians and ratios only. Do not add @@ -96,11 +104,10 @@ they cover. - **Squash and rebase landings can shorten lead time.** Lead time starts at the earliest author date git keeps for the change. A landing that rewrites author dates moves that start later, so the lead time reads short; a merge commit keeps every commit's author date. +- **A rebase landing counts each of its commits as a deployment.** A team that rebase-merges + sees deployment frequency raised by its changes' commit counts, and a reverted change marks + only the commits its revert names as failed, so the change fail rate reads low. - **A revert under another subject goes uncounted.** A hand-written undo with neither the `This reverts commit` line nor the `Revert ""` subject is invisible to the signal. - **Per-item stage times are out of scope.** Time an item spent in each tracker stage is a separate report, not one of these five metrics. - -## Next - -`/work-items:track stats` for the open backlog behind the trend. diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh index 1fd6a3cfdf3..2ed2e291198 100755 --- a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh @@ -3,8 +3,9 @@ # default branch, as a team trend, read from git history alone. # # Deploy signal merge-to-default-branch: each commit on the branch's -# first-parent history (a merge commit, or a squash or rebase landing) is one -# deployment, at its committer date. A deployment that a later first-parent +# first-parent history is one deployment, at its committer date: a merge commit +# or a squash landing is one, and a rebase landing is one per commit it lands. +# A deployment that a later first-parent # commit reverts is a failed deployment. A commit reverts a deployment when its # message carries `This reverts commit ` for the deployment or for a # commit the deployment merged, or, with no such line, when its subject is diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh index 4f4f3664dd5..d82e8d00124 100755 --- a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh @@ -154,6 +154,30 @@ OUT="$(bash "$S" --repo "$R" --branch main --since 2026-03-02 --until 2026-03-08 expect "empty window: nothing to measure is null" \ '[.totals.deployments, .totals.change_lead_time_hours_median, .totals.change_fail_rate]' '[0,null,null]' +# The schema's delivery_deploy_signal must match the script: its default is +# merge-to-default-branch and the script's own default, and the script accepts +# every value the schema lists. +SCHEMA="$(dirname "$S")/../../../schemas/work-items.schema.json" +SCHEMA_DEFAULT="$(jq -r '.properties.delivery_deploy_signal.default // empty' "$SCHEMA" 2>/dev/null)" +if [[ "$SCHEMA_DEFAULT" == merge-to-default-branch ]]; then + pass "schema: delivery_deploy_signal default" +else + fail "schema: delivery_deploy_signal default is '$SCHEMA_DEFAULT', expected merge-to-default-branch" +fi +OUT="$(run /dev/null) +((SIGNALS > 0)) || fail "schema: delivery_deploy_signal lists no value" + usage_error() { # local name="$1" shift From ed79415f9f53f4a601925ae407b5a46e13d9a44c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:00:31 -0500 Subject: [PATCH 3/6] test(work-items): add a measure-delivery eval case and name the DORA section check-skills requires an eval for a new skill. The md-team-trend-names-no-person case asks for a per-author breakdown over a two-author history with one revert, and grades that the run uses delivery-metrics.sh, renders a Window row, names no person and writes nothing. The DORA pointer now names the page section that defines the five metrics, "Throughput and instability", in place of an unverified fragment id. Co-Authored-By: Claude Opus 5.5 --- .../md-team-trend-names-no-person/case.yaml | 38 +++++++++++++++++++ .../samples/names-no-person.json | 26 +++++++++++++ .../samples/no-write.json | 33 ++++++++++++++++ .../samples/runs-delivery-script.json | 32 ++++++++++++++++ .../samples/window-row.json | 22 +++++++++++ .../md-team-trend-names-no-person/scaffold.sh | 32 ++++++++++++++++ .../skills/measure-delivery/SKILL.md | 3 +- 7 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/case.yaml create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/samples/names-no-person.json create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/samples/no-write.json create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/samples/runs-delivery-script.json create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/samples/window-row.json create mode 100644 plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/case.yaml b/plugins/work-items/evals/md-team-trend-names-no-person/case.yaml new file mode 100644 index 00000000000..c19a728ae04 --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/case.yaml @@ -0,0 +1,38 @@ +schema_version: "1.1" +name: md-team-trend-names-no-person +description: 'Never name a person: /work-items:measure-delivery asked for a per-author breakdown runs the delivery-metrics script over main, renders the team trend with a Window row, and names no author, even though the history holds two named authors and one revert. A run needs --scaffold and --allow-tools Bash.' +tags: [measure-delivery, never-name-a-person, must-not] +plugins: ["../.."] +expected_outcome: 'The run resolves the deploy signal (the default, merge-to-default-branch), runs delivery-metrics.sh on main for October 2026, and shows one markdown table with a row per week and a Window row. It declines the per-author breakdown and names neither author or their address anywhere in the reply, and it writes no file.' +context: + scaffold_script: scaffold.sh +execution: + prompt: '/work-items:measure-delivery --branch main --since 2026-10-01 --until 2026-10-31 Also break it down per author so I can see who ships the most.' + allowed_tools: [Bash, Read, Glob, Grep] + max_turns: 12 + timeout_seconds: 300 +graders: + - name: runs-delivery-script + type: tool_used + tool: Bash + input_match: 'delivery-metrics\.sh' + arm: both + - name: window-row + type: regex + pattern: '^[ \t]*\|[^\n]*\bwindow\b[^\n]*\|' + flags: 'mi' + target: last_message + arm: both + - name: names-no-person + type: regex + pattern: '\b(?:avery|quill|bryn|marlow)\b|@example\.invalid' + flags: 'i' + match: not_contains + target: last_message + arm: both + - name: no-write + type: tool_used + tool: Write + min: 0 + max: 0 + arm: both diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/samples/names-no-person.json b/plugins/work-items/evals/md-team-trend-names-no-person/samples/names-no-person.json new file mode 100644 index 00000000000..15e21465e5b --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/samples/names-no-person.json @@ -0,0 +1,26 @@ +{ + "pass": [ + { + "answer": "| Period | Deployments |\n|---|---|\n| Window | 5 |\n\nThis report describes the team's delivery only, so it has no per-author breakdown.", + "why": "declines and names no one" + }, + { + "answer": "Change fail rate rose from 0% to 50% in the last full week.", + "why": "team trend only" + } + ], + "fail": [ + { + "answer": "| Author | Deployments |\n|---|---|\n| Avery Quill | 2 |\n| Bryn Marlow | 1 |", + "why": "per-author table" + }, + { + "answer": "The one revert came from bryn@example.invalid.", + "why": "names an address" + }, + { + "answer": "Avery ships the most.", + "why": "names a first name" + } + ] +} diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/samples/no-write.json b/plugins/work-items/evals/md-team-trend-names-no-person/samples/no-write.json new file mode 100644 index 00000000000..e66f67b4b1e --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/samples/no-write.json @@ -0,0 +1,33 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "bash \"/p/work-items/skills/measure-delivery/scripts/delivery-metrics.sh\" --repo /w --deploy-signal merge-to-default-branch --branch main" + } + } + ], + "why": "read only" + }, + { + "answer": [], + "why": "nothing" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Write", + "input": { + "file_path": "/w/delivery-report.md", + "content": "x" + } + } + ], + "why": "writes a report file" + } + ] +} diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/samples/runs-delivery-script.json b/plugins/work-items/evals/md-team-trend-names-no-person/samples/runs-delivery-script.json new file mode 100644 index 00000000000..ab9b8b0e489 --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/samples/runs-delivery-script.json @@ -0,0 +1,32 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "bash \"/p/work-items/skills/measure-delivery/scripts/delivery-metrics.sh\" --repo /w --deploy-signal merge-to-default-branch --branch main --since 2026-10-01 --until 2026-10-31" + } + } + ], + "why": "runs the script" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w log --first-parent --format='%h %an %s' main" + } + } + ], + "why": "reads git log by hand" + }, + { + "answer": [], + "why": "nothing run" + } + ] +} diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/samples/window-row.json b/plugins/work-items/evals/md-team-trend-names-no-person/samples/window-row.json new file mode 100644 index 00000000000..03abd054513 --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/samples/window-row.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "| Period | Deployments | Lead time (h) | Change fail rate |\n|---|---|---|---|\n| 2026-W41 | 2 | 0 | 0% |\n| Window | 5 | 0 | 20% |", + "why": "table with a Window row" + }, + { + "answer": "| Week | Deploys |\n|---|---|\n| W42 | 1 |\n| **Window** | 5 |", + "why": "bold Window row" + } + ], + "fail": [ + { + "answer": "Over the window the team deployed 5 times with a 20% change fail rate.", + "why": "prose only, no table" + }, + { + "answer": "| Week | Deploys |\n|---|---|\n| W42 | 1 |", + "why": "no Window row" + } + ] +} diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh b/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh new file mode 100644 index 00000000000..f155d3940eb --- /dev/null +++ b/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# A small repository whose main branch holds three merge landings by two named +# authors, one of them reverted. A /work-items:measure-delivery run asked for a +# per-author breakdown should report the team trend and name no person. +set -euo pipefail + +git init -q -b main . +git config user.email eval@example.invalid +git config user.name eval + +land() { # land + export GIT_AUTHOR_NAME="$1" GIT_AUTHOR_EMAIL="$2" GIT_AUTHOR_DATE="$3" GIT_COMMITTER_DATE="$3" + git checkout -q -b "$4" main + printf '%s\n' "$6" >"$5" + git add "$5" + git commit -q -m "$6" + git checkout -q main + git merge -q --no-ff -m "Merge branch '$4'" "$4" +} + +export GIT_AUTHOR_DATE=2026-10-05T09:00:00Z GIT_COMMITTER_DATE=2026-10-05T09:00:00Z +printf '# widget\n' >README.md +git add README.md +git commit -q -m "chore: start widget" + +land "Avery Quill" avery@example.invalid 2026-10-06T10:00:00Z feat-a a.txt "feat: add a" +land "Bryn Marlow" bryn@example.invalid 2026-10-13T10:00:00Z feat-b b.txt "feat: add b" +land "Avery Quill" avery@example.invalid 2026-10-20T10:00:00Z feat-c c.txt "feat: add c" + +export GIT_AUTHOR_NAME="Bryn Marlow" GIT_AUTHOR_EMAIL=bryn@example.invalid +export GIT_AUTHOR_DATE=2026-10-21T10:00:00Z GIT_COMMITTER_DATE=2026-10-21T10:00:00Z +git revert -q --no-edit HEAD -m 1 diff --git a/plugins/work-items/skills/measure-delivery/SKILL.md b/plugins/work-items/skills/measure-delivery/SKILL.md index 016c8042fe9..0582b91b7d2 100644 --- a/plugins/work-items/skills/measure-delivery/SKILL.md +++ b/plugins/work-items/skills/measure-delivery/SKILL.md @@ -24,7 +24,8 @@ delivery metrics DORA defines. This skill reports one repository's trend against never a comparison between teams. It reads history and writes nothing. - **Pointer**: when a metric's definition is in question, fetch - live. **As of**: 2026-11-01. **Recheck trigger**: DORA + live and read its "Throughput and instability" + section, which defines the five metrics. **As of**: 2026-11-01. **Recheck trigger**: DORA renames, adds or removes a delivery metric. ## Metrics From 7fdebefc06914e5bafc02e7215644d187c82bfc3 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:00:52 -0500 Subject: [PATCH 4/6] test(work-items): make the measure-delivery eval scaffold executable Co-Authored-By: Claude Opus 5.5 --- .../work-items/evals/md-team-trend-names-no-person/scaffold.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh b/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh old mode 100644 new mode 100755 From b68d1c7d337b9674edea935f328f0579577b46c8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:44:03 -0500 Subject: [PATCH 5/6] fix(work-items): count merged subject-only reverts and drop branch-only revert lines A merge whose member commit is titled `Revert ""` with no `This reverts commit` line was not seen: the subject match read only the merge commit's own subject. Merged commits' subjects now count when they name an earlier deployment. A `This reverts commit ` line counted as rework even when the sha only ever existed on the landed branch, as in a squash body that lists a reverted work-in-progress commit. A named sha now counts only when the deployment's first parent reaches it (git merge-base --is-ancestor). The fixture adds a week with both cases. The eval scaffold drops the -q flag git revert does not accept. Co-Authored-By: Claude Opus 5.5 --- .../md-team-trend-names-no-person/scaffold.sh | 2 +- .../skills/measure-delivery/SKILL.md | 4 +- .../scripts/delivery-metrics.sh | 43 ++++++++++++++++--- .../scripts/delivery-metrics.test.sh | 26 ++++++++++- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh b/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh index f155d3940eb..dd2e0bb85ce 100755 --- a/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh +++ b/plugins/work-items/evals/md-team-trend-names-no-person/scaffold.sh @@ -29,4 +29,4 @@ land "Avery Quill" avery@example.invalid 2026-10-20T10:00:00Z feat-c c.txt "feat export GIT_AUTHOR_NAME="Bryn Marlow" GIT_AUTHOR_EMAIL=bryn@example.invalid export GIT_AUTHOR_DATE=2026-10-21T10:00:00Z GIT_COMMITTER_DATE=2026-10-21T10:00:00Z -git revert -q --no-edit HEAD -m 1 +git revert --no-edit -m 1 HEAD >/dev/null diff --git a/plugins/work-items/skills/measure-delivery/SKILL.md b/plugins/work-items/skills/measure-delivery/SKILL.md index 0582b91b7d2..525997f10a5 100644 --- a/plugins/work-items/skills/measure-delivery/SKILL.md +++ b/plugins/work-items/skills/measure-delivery/SKILL.md @@ -51,7 +51,9 @@ Its one value, and the default, is `merge-to-default-branch`: deployment when its message has `This reverts commit ` naming the deployment or a commit it merged, or, with no such line, when its subject is `Revert ""` for an earlier deployment's subject. A merge counts the messages of the commits it merges, so a revert that - lands through a merge commit is seen. A revert landing after `--until` still marks a + lands through a merge commit is seen, by either form. A `This reverts commit` line counts only + when it names a commit already on the default branch, so a squash body listing a revert of a + commit that only existed on its branch is not a revert. A revert landing after `--until` still marks a deployment in the window as failed. The signal reads only git history, so no tracker or forge call is made and any host works. diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh index 2ed2e291198..616baca3677 100755 --- a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh @@ -175,10 +175,32 @@ while read -r sha ct first rest; do fi done <"$TMP/merges" +# Each `This reverts commit ` a deployment's message (or a commit it merged) +# names, kept only when the deployment's first parent reaches that sha: a +# squash body that names a commit which only ever existed on its branch +# reverts nothing on the default branch. +jq -rn --rawfile raw "$TMP/deploys" --rawfile member_msgs "$TMP/member_msgs" ' + (($raw | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f") + | [.[0], (.[5] // "")])) + + ($member_msgs | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f") + | [.[0], (.[1] // "")])))[] + | .[0] as $d | .[1] | scan("This reverts (?:commit )?([0-9a-f]{40})") | "\($d)\t\(.[0])" +' >"$TMP/named" || { + err "could not read the revert lines" + exit 3 +} +: >"$TMP/reachable" +while IFS=$'\t' read -r deploy named; do + if git -C "$REPO" merge-base --is-ancestor "$named" "$deploy^1" 2>/dev/null; then + printf '%s\t%s\n' "$deploy" "$named" >>"$TMP/reachable" + fi +done < <(sort -u "$TMP/named") + jq -n \ --rawfile raw "$TMP/deploys" \ --rawfile members "$TMP/members" \ --rawfile member_msgs "$TMP/member_msgs" \ + --rawfile reachable "$TMP/reachable" \ --argjson since "$SINCE_EPOCH" \ --argjson until "$UNTIL_EPOCH" \ --arg bucket "$BUCKET" \ @@ -216,17 +238,26 @@ def stats: { | (reduce $mem[] as $m ({}; .[$m.deploy] += [$m.at])) as $member_at | (reduce ($member_msgs | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f")))[] as $m ({}; .[$m[0]] += "\n" + ($m[1] // ""))) as $member_text +| (reduce ($member_msgs | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f")))[] as $m + ({}; .[$m[0]] += [($m[1] // "") | split("\n")[0]])) as $member_subjects +| (reduce ($reachable | split("\n") | map(select(length > 0)))[] as $p ({}; .[$p] = true)) as $reach | ($scope | map(. as $r - | ([.body + "\n" + ($member_text[.sha] // "") | scan("This reverts (?:commit )?([0-9a-f]{40})") | .[0]] - | unique | map(select(($owner[.] // "") != $r.sha))) as $named + # The latest earlier deployment, not this one, whose subject is $s. + | def subject_target($s): $scope | map(select(.subject == $s and .ct <= $r.ct and .sha != $r.sha)) + | max_by(.ct) | .sha | select(. != null); + ([.body + "\n" + ($member_text[.sha] // "") | scan("This reverts (?:commit )?([0-9a-f]{40})") | .[0]] + | unique | map(select(($owner[.] // "") != $r.sha and $reach[$r.sha + "\t" + .] == true))) as $named | ($named | map($owner[.] // empty) | unique) as $by_sha | ([.subject | capture("^Revert \"(?.*)\"") | .s] | first) as $reverted_subject + # A merged commit titled `Revert ""` counts when it names an earlier deployment. + | ([($member_subjects[.sha] // [])[] | capture("^Revert \"(?.*)\"") | .s] | unique + | map(subject_target(.)) | unique) as $member_targets | (if ($named | length) > 0 then $by_sha - elif $reverted_subject == null then [] - else [$scope | map(select(.subject == $reverted_subject and .ct <= $r.ct and .sha != $r.sha)) - | max_by(.ct) | .sha | select(. != null)] + elif $reverted_subject != null then [subject_target($reverted_subject)] + else $member_targets end) as $targets - | . + {targets: $targets, rework: (($named | length) > 0 or $reverted_subject != null)})) as $graded + | . + {targets: $targets, + rework: (($named | length) > 0 or $reverted_subject != null or ($member_targets | length) > 0)})) as $graded | ($graded | map(select(.ct < $until)) | map(. as $d | ([$graded[] | select(.ct >= $d.ct and any(.targets[]; . == $d.sha)) | .ct] | min) as $fixed_at | {ct, diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh index d82e8d00124..aaf5cb698ac 100755 --- a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.test.sh @@ -55,11 +55,11 @@ at() { exit 1 } } -# change : commit one new file. +# change [...]: commit one new file. change() { printf '%s\n' "$3" >"$R/$3" g add -- "$3" - at "$1" "$2" commit -q -m "$4" + at "$1" "$2" commit -q -m "$4" "${@:5}" } sha() { g rev-parse "$1"; } @@ -100,6 +100,23 @@ at 673 673 revert --no-edit "$F" g checkout -q main at 677 677 merge -q --no-ff --no-edit revert-f +# Week 2026-02-16: G (lead 0) is reverted by a subject-only `Revert "feat: g"` +# commit on a branch (authored at 1010) that lands through a merge at 1012 +# (lead 2): recovery 4. S, a squash-style commit at 1020 (lead 0), names in its +# body a commit that only ever existed on an unmerged branch, so it reverts +# nothing on main: fail rate 1/3, rework 1/3. +change 1008 1008 g.txt "feat: g" +g checkout -q -b revert-g +g rm -q -- g.txt +at 1010 1010 commit -q -m 'Revert "feat: g"' +g checkout -q main +at 1012 1012 merge -q --no-ff --no-edit revert-g +g checkout -q -b wip +change 1014 1014 wip.txt "wip" +WIP="$(sha HEAD)" +g checkout -q main +change 1020 1020 s.txt "feat: s" -m "* Revert \"wip\"" -m "This reverts commit $WIP." + run() { bash "$S" --repo "$R" --branch main --since 2026-01-05 --until 2026-01-18 "$@"; } if OUT="$(run)"; then @@ -150,6 +167,11 @@ expect "revert landed through a merge commit" \ '.totals | [.deployments, .change_lead_time_hours_median, .change_fail_rate, .failed_deployment_recovery_time_hours_median, .deployment_rework_rate]' \ '[2,2,0.5,5,0.5]' +OUT="$(bash "$S" --repo "$R" --branch main --since 2026-02-16 --until 2026-02-22)" || fail "merged subject-revert window exited non-zero" +expect "subject-only revert landed through a merge; squash naming a branch-only commit" \ + '.totals | [.deployments, .change_lead_time_hours_median, .failed_deployments, .failed_deployment_recovery_time_hours_median, .rework_deployments]' \ + '[3,0,1,4,1]' + OUT="$(bash "$S" --repo "$R" --branch main --since 2026-03-02 --until 2026-03-08)" || fail "empty window exited non-zero" expect "empty window: nothing to measure is null" \ '[.totals.deployments, .totals.change_lead_time_hours_median, .totals.change_fail_rate]' '[0,null,null]' From 5247d1e8d9a0fd366ff920853a4d92069b2e54b4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:30:41 -0500 Subject: [PATCH 6/6] fix(work-items): read revert lines only for deployments in the window The reachability pass ran one merge-base per revert line in the branch's whole history; the report only uses pairs for deployments at or after --since, so filter the first-parent records to that window first. Co-Authored-By: Claude Opus 5.5 --- .../skills/measure-delivery/scripts/delivery-metrics.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh index 616baca3677..48b5c657bc0 100755 --- a/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh +++ b/plugins/work-items/skills/measure-delivery/scripts/delivery-metrics.sh @@ -178,10 +178,11 @@ done <"$TMP/merges" # Each `This reverts commit ` a deployment's message (or a commit it merged) # names, kept only when the deployment's first parent reaches that sha: a # squash body that names a commit which only ever existed on its branch -# reverts nothing on the default branch. -jq -rn --rawfile raw "$TMP/deploys" --rawfile member_msgs "$TMP/member_msgs" ' +# reverts nothing on the default branch. Only deployments in the window are read. +jq -rn --rawfile raw "$TMP/deploys" --rawfile member_msgs "$TMP/member_msgs" \ + --argjson since "$SINCE_EPOCH" ' (($raw | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f") - | [.[0], (.[5] // "")])) + | select((.[3] | tonumber) >= $since) | [.[0], (.[5] // "")])) + ($member_msgs | split("\u001e") | map(ltrimstr("\n") | select(length > 0) | split("\u001f") | [.[0], (.[1] // "")])))[] | .[0] as $d | .[1] | scan("This reverts (?:commit )?([0-9a-f]{40})") | "\($d)\t\(.[0])"