Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
bump: minor
---

### Added

- `limits.stranded-backoff-hours` sets, per cause (`no-run`, `no-report`, `failed-again`, `failed-mint`, `kill-switch`), the base hours of work-unblock's re-fire backoff for a stranded `agent-ready` issue, doubled once per re-fire already made and held at 24 hours. Every cause defaults to 3 hours, the base before this key existed.
8 changes: 8 additions & 0 deletions .github/actions/resolve-config/resolve.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -980,6 +980,14 @@ const limitsOf = (limits = {}) => ({
"fix-docs-per-run": limits["fix-docs-per-run"] ?? 2,
"admission-factor": limits["admission-factor"] ?? 0.8,
"admission-min-per-hour": limits["admission-min-per-hour"] ?? 1,
"stranded-backoff-hours": {
"no-run": 3,
"no-report": 3,
"failed-again": 3,
"failed-mint": 3,
"kill-switch": 3,
...limits["stranded-backoff-hours"],
},
// As set: rate-guard.mjs fills it from rate-limits.json beside it.
api: limits.api ?? {},
});
Expand Down
15 changes: 15 additions & 0 deletions .github/actions/resolve-config/resolve.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,8 @@ const LIMITS = {
"admission-factor": 0.8,
// The release-point floor the owner chose (2026-10-21 interview).
"admission-min-per-hour": 1,
// work-unblock's stranded re-fire backoff starts at 3 hours for every cause.
"stranded-backoff-hours": { "no-run": 3, "no-report": 3, "failed-again": 3, "failed-mint": 3, "kill-switch": 3 },
// Passed through as set; rate-guard.mjs fills it from the schema's defaults.
api: {},
};
Expand Down Expand Up @@ -376,6 +378,19 @@ test("limits: api passes through as set, and an unknown pool or field, or a shar
}
});

test("limits: stranded-backoff-hours overrides a cause's hours and keeps the rest; an unknown cause or hours past 24 is rejected", () => {
const config = edited((doc) => {
doc.limits = { "stranded-backoff-hours": { "no-run": 1 } };
});
assert.deepEqual(run(config).limits["stranded-backoff-hours"], { ...LIMITS["stranded-backoff-hours"], "no-run": 1 });
for (const bad of [{ "no-runs": 1 }, { "no-run": 0 }, { "no-run": 25 }, { "no-run": 1.5 }]) {
const rejected = edited((doc) => {
doc.limits = { "stranded-backoff-hours": bad };
});
assertRejected(() => run(rejected), "invalid-config");
}
});

test("limits: an unknown key, a value that is not an integer, or one out of its range is rejected", () => {
for (const limits of [{ "fallback-resets": 5 }, { "fallback-reset": 0 }, { "restarts-per-run": -1 }, { "reset-clamp-max": 1.5 }, { "limit-stops-per-head": "3" }, { "limit-stops-per-head": 11 }, { "limit-stops-per-issue": 11 }, { "restarts-per-run": 51 }, { "readmit-per-run": 0 }, { "readmit-per-run": 101 }, { "parent-rescan-per-run": 0 }, { "parent-rescan-per-run": 101 },{ "fallback-reset": 721 }, { "reset-clamp-max": 721 }, { "wip-cap": 0 }, { "wip-cap": 501 }, { "turn-cap-restarts": -1 }, { "turn-cap-restarts": 6 }, { "stop-retries": -1 }, { "stop-retries": 6 }, { "fix-ci-per-run": 0 }, { "fix-ci-per-run": 21 }, { "fix-ci-restarts": -1 }, { "fix-ci-restarts": 6 }, { "fix-docs-per-run": 0 }, { "fix-docs-per-run": 21 }, { "admission-window-hours": 24 }, { "admission-factor": 0 }, { "admission-factor": 1.1 }, { "admission-min-per-hour": -1 }, { "admission-min-per-hour": 101 }]) {
const config = edited((doc) => {
Expand Down
24 changes: 23 additions & 1 deletion .github/scripts/work-resolve-issue/limits.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,31 @@ export function readLimits(path) {
} catch {
limits = undefined;
}
return { ...resolveLimits(limits), ...resolveAdmission(limits) };
return { ...resolveLimits(limits), ...resolveAdmission(limits), "stranded-backoff-hours": resolveStrandedBackoff(limits?.["stranded-backoff-hours"]) };
}

// The `limits.stranded-backoff-hours` leaf: per cause work-unblock's sweep
// re-fires a stranded issue for, the base hours of its backoff, doubled once
// per re-fire already counted and held at STRANDED_BACKOFF_MAX.
export const STRANDED_BACKOFF_MAX = 24;
export const STRANDED_BACKOFF_DEFAULTS = Object.freeze({
"no-run": 3,
"no-report": 3,
"failed-again": 3,
"failed-mint": 3,
"kill-switch": 3,
});

// Every cause, each set whole number of hours from 1 to STRANDED_BACKOFF_MAX
// over its default.
export const resolveStrandedBackoff = (hours) =>
Object.fromEntries(
Object.entries(STRANDED_BACKOFF_DEFAULTS).map(([key, value]) => {
const set = hours?.[key];
return [key, Number.isInteger(set) && set >= 1 && set <= STRANDED_BACKOFF_MAX ? set : value];
}),
);

const positive = (value) => typeof value === "number" && Number.isFinite(value) && value > 0;

// The two `limits` leaves that pace new issue-lane admissions (check-wip-cap's
Expand Down
51 changes: 28 additions & 23 deletions .github/scripts/work-unblock/label-unblocked.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,11 @@
// An agent-ready issue left with nothing running is stranded: no hold, no open
// lane pull request or open pull request of a listed human's for it, no
// outcome marker since its newest agent-ready, and no work-resolve-issue run
// for that label, or only one that ended with no report, over STRANDED_HOURS
// ago. It is re-fired with the re-fire record below once per version of the
// lane workflow files, and after that on a backoff (STRANDED_HOURS doubling
// with each re-fire, at most BACKOFF_CAP_HOURS), recorded in one comment
// for that label, or only one that ended with no report, over its cause's
// limits.stranded-backoff-hours ago. It is re-fired with the re-fire record
// below once per version of the lane workflow files, and after that on a
// backoff (those hours doubling with each re-fire, at most
// BACKOFF_CAP_HOURS), recorded in one comment
// edited in place; it never gets needs-human. All share
// limits.restarts-per-run.
//
Expand Down Expand Up @@ -278,7 +279,7 @@ import { createGitHub, isTrustedClose as closedByHumanOrMerge, loadTrustedList,
import { writeOutputs } from "../../actions/check-kill-switch/check-kill-switch.mjs";
import { admissionLine, checkWip, searchRateLimit } from "../../actions/check-wip-cap/check-wip-cap.mjs";
import { clampReset, isLaneBranch, LANE_HOLDS, LANE_WORKFLOWS, parseOutcomeMarker, REFIRED, refireComment as refireRecord, refsHoldMarker } from "../work-resolve-issue/report-outcome.mjs";
import { readLimits, readWip } from "../work-resolve-issue/limits.mjs";
import { readLimits, readWip, resolveStrandedBackoff, STRANDED_BACKOFF_MAX } from "../work-resolve-issue/limits.mjs";
import { ADMIT_MARKER, admissionFacts, admissionParked, admissionSettled, ATTENDED, authorAdmitted, attendedHold, blockersOf, CONFIRMED, decide, deferralLapsed, loadRules, NEEDS_DECISION, openPullsFor, openSubIssues, READY, refusalText, removalStands, sayWhy, SIZE_SPLIT_PENDING, specOf, stillBlocking, subIssueWait, UNTRUSTED, untrustedHold } from "../intake-triage/admit.mjs";
import { clearedPastReport, retriageNote, doneReason, judgedSince, mergedRefsReader, splitLeftOpen, splitParentClosed } from "../intake-triage/admit.mjs";
import { budgetLatch } from "../../actions/log-api-budget/rate-guard.mjs";
Expand Down Expand Up @@ -369,8 +370,6 @@ const IN_FLIGHT = ["queued", "in_progress", "waiting", "pending", "requested"];
export const LOOKBACK_HOURS = 3;
// The sweep reads from this long before the last good sweep started.
export const SWEEP_MARGIN_MIN = 15;
// How long an agent-ready issue may sit with no run working on it.
export const STRANDED_HOURS = 3;
// The marker issue's label: a listed human closing an issue that carries it
// runs the sweep at once instead of waiting for the schedule (the hub reopens
// and closes it after a rate-limit breaker clears). That sweep re-fires at
Expand All @@ -385,8 +384,10 @@ export const SWEEP_LABEL = "work-unblock-sweep";
// last edit.
const INFRA = /^<!-- work-unblock infrastructure ([0-9a-f]{40}-[0-9a-f]{40}) -->/;
// The longest wait between backoff re-fires.
export const BACKOFF_CAP_HOURS = 24;
const backoffDue = (since, refires) => since + Math.min(STRANDED_HOURS * 2 ** refires, BACKOFF_CAP_HOURS) * HOUR_MS;
export const BACKOFF_CAP_HOURS = STRANDED_BACKOFF_MAX;
// When the next re-fire is due: `hours`, the cause's
// limits.stranded-backoff-hours, doubled per re-fire so far.
const backoffDue = (since, refires, hours) => since + Math.min(hours * 2 ** refires, BACKOFF_CAP_HOURS) * HOUR_MS;
const DIMENSIONS = ["size", "complexity", "uncertainty"];

// Whether the account that closed the issue (ACTOR_ID) and the one that
Expand Down Expand Up @@ -1189,11 +1190,13 @@ export const failedComment = ({ stops, run = "", onResumed = false }) =>
const FAILED_CAUSE = `The work-resolve-issue run that \`${LABEL}\` started failed or was cancelled before it reported, or it stopped or the token broker refused it because its workflow files differ from main's.`;
export const refireComment = (key, cause = FAILED_CAUSE) => refireRecord(key, cause);

// Why a stranded issue has nothing running, as the comments say it.
// Why a stranded issue has nothing running, as the comments say it, after
// `hours` with nothing running.
const hoursText = (hours) => `${hours} hour${hours === 1 ? "" : "s"}`;
const strandedCause = {
"no-run": `No work-resolve-issue run started in the ${STRANDED_HOURS} hours since \`${LABEL}\` was last added.`,
"no-report": `The work-resolve-issue run that \`${LABEL}\` started ended over ${STRANDED_HOURS} hours ago with no report and no pull request.`,
"failed-again": `The work-resolve-issue run that \`${LABEL}\` started failed or was cancelled before it reported, over ${STRANDED_HOURS} hours ago.`,
"no-run": (hours) => `No work-resolve-issue run started in the ${hoursText(hours)} since \`${LABEL}\` was last added.`,
"no-report": (hours) => `The work-resolve-issue run that \`${LABEL}\` started ended over ${hoursText(hours)} ago with no report and no pull request.`,
"failed-again": (hours) => `The work-resolve-issue run that \`${LABEL}\` started failed or was cancelled before it reported, over ${hoursText(hours)} ago.`,
};

const MINT_CAUSE = `The work-resolve-issue run that \`${LABEL}\` started could not get its lane token, or the token broker's leave to run.`;
Expand Down Expand Up @@ -1657,30 +1660,32 @@ async function refireRefusal({ github, actions, jobLog, repo, number, issue, las
const refiredOn = (marker, key) => marker.outcome === "refired" && !marker.breaker && marker.key === key;
const relabeled = (marker) =>
events.some((event) => isLabel(event, "labeled") && event.actor?.login === bot && Date.parse(event.created_at) >= marker.at);
const hoursFor = resolveStrandedBackoff(limits["stranded-backoff-hours"]);
// A re-fire past the first on these workflow files waits longer each time
// (backoffDue from `since`), and its record is the one backoff comment,
// edited in place, never a new comment or a hold.
const backoff = (key, since, cause) => {
// (backoffDue from `since`, starting at `hours`), and its record is the one
// backoff comment, edited in place, never a new comment or a hold.
const backoff = (key, since, cause, hours) => {
const first = Math.min(...markers.filter((marker) => refiredOn(marker, key)).map((marker) => marker.at));
// Each breaker record's own re-add, the App's first after it (dated by
// when it was posted), is that trip's, not this cause's; the backoff
// comment's edits never move it.
const adds = events.filter((event) => isLabel(event, "labeled") && event.actor?.login === bot);
const trips = new Set(markers.filter((marker) => marker.breaker).map((marker) => adds.find((event) => Date.parse(event.created_at) >= marker.at)));
const refires = adds.filter((event) => Date.parse(event.created_at) >= first && !trips.has(event)).length;
const due = backoffDue(since, refires);
const due = backoffDue(since, refires, hours);
if (now < due) {
return { reason: `re-fired ${refires} time(s) on workflow files ${key}; it is re-fired again after ${new Date(due).toISOString()}` };
}
const own = comments.filter((comment) => comment.user?.login === bot && INFRA.test(String(comment.body ?? ""))).at(-1);
return { reason: null, resets: due, refire: key, recorded: true, infra: { id: own?.id, body: infraComment(key, cause, refires + 1) } };
};
// An issue with nothing running since `since`: once STRANDED_HOURS have
// An issue with nothing running since `since`: once its cause's hours have
// passed with no open lane PR and no open pull request of a listed human's
// for it, re-fired, once per version of the lane workflow files and then on
// a backoff.
const stranded = async (since, why, cause) => {
if (now - since < STRANDED_HOURS * HOUR_MS) {
const hours = hoursFor[cause];
if (now - since < hours * HOUR_MS) {
return { reason: why };
}
const open = (await lanePulls({ github, repo, number, bot })).find((item) => item.state === "open");
Expand All @@ -1694,9 +1699,9 @@ async function refireRefusal({ github, actions, jobLog, repo, number, issue, las
const key = await workflowKey();
const fired = markers.filter((marker) => refiredOn(marker, key));
if (fired.some(relabeled)) {
return backoff(key, since, strandedCause[cause]);
return backoff(key, since, strandedCause[cause](hours), hours);
}
return { reason: null, resets: since, refire: key, recorded: fired.length > 0, cause: strandedCause[cause] };
return { reason: null, resets: since, refire: key, recorded: fired.length > 0, cause: strandedCause[cause](hours) };
};
const { mine: runs, untitled } = await laneRuns(actions, repo, number, addedAt);
if (runs.length === 0) {
Expand Down Expand Up @@ -1733,7 +1738,7 @@ async function refireRefusal({ github, actions, jobLog, repo, number, issue, las
// kill switch off, so it re-fires now; a mint trip ran the model and
// waits like any failed mint.
const clamp = stalled === "breaker" && stage === "check" ? 0 : limits["reset-clamp-min"];
const due = over >= 0 ? backoffDue(ended, over) : ended + clamp * HOUR_MS;
const due = over >= 0 ? backoffDue(ended, over, hoursFor["kill-switch"]) : ended + clamp * HOUR_MS;
if (now < due) {
const why = stalled === "breaker" ? `the broker refused its lane run for the lanes kill switch (trip ${trips + 1})` : "its lane run did no work";
return { reason: `${why}; it is re-fired after ${new Date(due).toISOString()}` };
Expand All @@ -1751,7 +1756,7 @@ async function refireRefusal({ github, actions, jobLog, repo, number, issue, las
// attempt, and the cap and restarts-per-run bound it.
const fired = markers.filter((marker) => refiredOn(marker, key)).length;
if (stalled === "mint-failed" && !recorded && fired >= limits["limit-stops-per-issue"]) {
return backoff(key, ended, MINT_CAUSE);
return backoff(key, ended, MINT_CAUSE, hoursFor["failed-mint"]);
}
const trip = stalled === "breaker" ? { run: refused.id, ended: Date.parse(refused.updated_at) || ended } : undefined;
return { reason: null, resets: due, refire: key, recorded, stall: stalled, trip };
Expand Down
34 changes: 34 additions & 0 deletions .github/scripts/work-unblock/label-unblocked.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4318,6 +4318,40 @@ test("the backoff between re-fires stops growing at 24 hours", async () => {
assert.deepEqual(writes(calls), [`PATCH /repos/${REPO}/issues/comments/52 ${JSON.stringify({ body: infraComment(KEY, NO_RUN_CAUSE, 7) })}`, ...restarted(30)]);
});

// limits.stranded-backoff-hours sets each cause's wait: a shorter no-run
// re-fires a label with no run after 1 hour, doubling from there, while a run
// that ended with no report keeps the 3-hour default.
test("a cause's stranded-backoff-hours shortens its wait and backoff; other causes keep 3 hours", async () => {
const limits = withLimits({ "stranded-backoff-hours": { "no-run": 1 } });
const oneHourCause = "No work-resolve-issue run started in the 1 hour since `agent-ready` was last added.";
// NOW is 2026-10-16T12:00:00Z: a label 1 h 0 min 1 s old with no run is re-fired.
const quick = failedRun({ runs: [], events: [added("2026-10-16T10:59:59Z", OWNER)] });
assert.equal(await sweep({ github: quick.github, actions: quick.actions }, limits), 0);
assert.deepEqual(writes(quick.calls), refiredFor(30, oneHourCause));
// The same label under the default waits.
const plain = failedRun({ runs: [], events: [added("2026-10-16T10:59:59Z", OWNER)] });
assert.equal(await sweep({ github: plain.github, actions: plain.actions }), 0);
assert.deepEqual(writes(plain.calls), []);
// A green run with no report that ended 2 hours ago still waits 3 hours.
const report = failedRun({ runs: [laneRun({ at: "2026-10-16T10:00:00Z", conclusion: "success" })], events: [added("2026-10-16T09:59:57Z", OWNER)] });
assert.equal(await sweep({ github: report.github, actions: report.actions }, limits), 0);
assert.deepEqual(writes(report.calls), []);
// After one re-fire the backoff doubles from 1 hour: 2 hours after the App's re-add.
const events = [
added("2026-10-16T06:00:00Z", OWNER),
{ event: "unlabeled", label: { name: "agent-ready" }, actor: LANES_BOT, created_at: "2026-10-16T08:00:00Z" },
added("2026-10-16T08:00:01Z"),
];
const comments = [note(), refireNote(KEY, "2026-10-16T07:59:59Z")];
const early = failedRun({ runs: [], comments, events });
assert.equal(await sweep({ github: early.github, actions: early.actions, now: Date.parse("2026-10-16T10:00:00Z") }, limits), 0);
assert.deepEqual(writes(early.calls), []);
const due = failedRun({ runs: [], comments, events });
assert.equal(await sweep({ github: due.github, actions: due.actions, now: Date.parse("2026-10-16T10:00:02Z") }, limits), 0);
const body = infraComment(KEY, oneHourCause, 2);
assert.deepEqual(writes(due.calls), [`POST /repos/${REPO}/issues/30/comments ${JSON.stringify({ body })}`, ...restarted(30)]);
});

// A listed human's open pull request for the issue is work running on it, as
// an open lane PR is: the issue is not stranded.
test("a stranded issue with a listed human's open closing PR is left alone; an unlisted author's PR is not running work", async () => {
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/work-unblock.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ name: work-unblock
# cancelled after admit and before resolve-issue started (pick-slot's timeout); a
# run a newer agent-ready run replaced is not. It does that once per issue
# per version of the lane's two workflow files on main, recorded in a comment,
# and after that on a backoff (three hours, doubling each time, at most 24),
# and after that on a backoff (its cause's limits.stranded-backoff-hours,
# doubling each time, at most 24),
# recorded in one comment it edits; it never adds needs-human for it. An open
# pull request of a listed human's for the issue counts as work running on it.
# When that run opened a lanes-App PR instead, and the PR merged with `Refs:`
Expand Down
Loading
Loading