Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1729,7 +1729,11 @@ latest upload, since one sender sends a session's uploads in order. The type is

- `pulls` are the PRs it worked on, attributed as `quest usage` attributes
them; a session with none yet, such as planning before its PR opens, keeps
its `branch`, and the next record names the PR.
its `branch`, and the next record names the PR. The sender remembers a sent
session while its latest branch, or a sub-agent's, heads no PR, and resends
it, with its sub-agents, once one opens: with the next session it sends from
that repository, since only then does it read the repository's PRs. It
forgets the session once the harness prunes its transcript.
- `role` is classifying when a PR's `## Review` record names it as the
[risk classifier](#risk-tiers), reviewing when it names it as a reviewer or
Codex marks it a review, else the one the first quest skill it ran gives: planning
Expand Down
79 changes: 0 additions & 79 deletions quest/m0/benchmark/resend-sessions.md

This file was deleted.

4 changes: 3 additions & 1 deletion src/cli/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1252,7 +1252,9 @@ export function main(argv: string[], out: Output): number | Promise<number> {
`${tokensPath()} pairs with that endpoint as a bearer token, one upload per endpoint and repository, ` +
`then again every ${SEND_INTERVAL / 1000} seconds until stopped, at low priority. A record is built as quest usage ` +
"builds it, from token-max's sessions on this host since the earliest queued session's first instant and " +
"the PRs updated since then, and replaces its earlier copy, so a resend never doubles. An endpoint that " +
"the PRs updated since then, and replaces its earlier copy, so a resend never doubles. A sent session whose " +
"latest branch, or a sub-agent's, heads no PR yet is sent again, with its sub-agents, once a pass for its " +
"repository reads a PR opened on that branch, until its transcript is gone. An endpoint that " +
"refuses the token (401 or 403) is sent nothing more until the token changes; any other failure keeps the " +
"sessions queued, retried after a minute and then twice as long each time up to six hours, and the next " +
"stop hook says so once. Before a record is built, a cheap model, " +
Expand Down
193 changes: 174 additions & 19 deletions src/cli/send.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Sending each session's record, and the breakages its PRs name, to the Kixel Code endpoint its checkout names. The stop hook only queues the session;
// one sender per user, run by its service manager, rebuilds each queued session's record, with its sub-agents', and
// posts them, keeping what failed for the next pass. A record replaces its earlier copy, so a resend never doubles.
// posts them, keeping what failed for the next pass, and queues a sent session again once a PR opens on a branch its
// record could not name a PR for. A record replaces its earlier copy, so a resend never doubles.
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { homedir, setPriority } from "node:os";
Expand All @@ -16,7 +17,19 @@ import { readRemote, remoteRepository } from "./github";
import { eachLine } from "./messages";
import { questRoot } from "./rules";
import { type Labeled, type SplitDependencies, type Steps, labelSession, readSteps, splitOf } from "./split";
import { type Pull, type Session, fixBreakages, fold, key, readPull, sessionRecords, updatedPulls } from "./usage";
import {
type Pull,
type Reached,
type Session,
fixBreakages,
fold,
heading,
key,
readPull,
sessionRecords,
unheaded,
updatedPulls,
} from "./usage";
import { type LoggedWait } from "./waits";

/** The checkout's file naming its record endpoint, as `{"records": "https://..."}`. */
Expand All @@ -35,6 +48,11 @@ export interface Pending {
origin: string;
prefix: string;
endpoint: string;
/**
* On a resend, the instant, epoch ms, its earlier upload read PRs from, so the resend reads at least as far back and
* names every PR that upload did.
*/
read?: number;
}

/** Where the sender keeps its queue and what it has said. */
Expand Down Expand Up @@ -184,26 +202,18 @@ export function queue(stopped: Stopped, env: NodeJS.ProcessEnv = process.env): Q
`quest: ${tokensPath(env)} holds no token for ${endpoint}, so this host sends it no session records.`,
);
if (existsSync(refusal(env, endpoint, token))) return { notices, queued: false };
const dir = join(stateDirectory(env), "pending");
const start = transcriptStart(stopped.transcript);
if (start === null) throw new Error(`${stopped.transcript} records no instant, so its session cannot be read`);
const queued = randomUUID();
const pending: Pending = {
const dir = enqueue(env, {
harness: stopped.harness,
session: stopped.session,
start,
queued,
queued: randomUUID(),
root,
origin: repository,
prefix: run(["rev-parse", "--show-prefix"]).stdout.trim(),
endpoint,
};
mkdirSync(dir, { recursive: true, mode: 0o700 });
// Each stop gets a file of its own, never rewritten, so a sender removes only the stops it read. Written whole,
// then renamed, so the sender never reads half a file.
const path = join(dir, `${queued}.json`);
writeFileSync(`${path}.new`, JSON.stringify(pending));
renameSync(`${path}.new`, path);
});
const sender = senderRuns(env);
if (!sender.runs)
// Said once per outage: the PID of the sender that stopped names it, or none for a host that never ran one.
Expand All @@ -214,6 +224,19 @@ export function queue(stopped: Stopped, env: NodeJS.ProcessEnv = process.env): Q
return { notices, queued: true };
}

/**
* Write `pending` to the queue, returning the queue's directory. Each stop gets a file of its own, never rewritten, so
* a sender removes only the stops it read. Written whole, then renamed, so the sender never reads half a file.
*/
function enqueue(env: NodeJS.ProcessEnv, pending: Pending): string {
const dir = join(stateDirectory(env), "pending");
mkdirSync(dir, { recursive: true, mode: 0o700 });
const path = join(dir, `${pending.queued}.json`);
writeFileSync(`${path}.new`, JSON.stringify(pending));
renameSync(`${path}.new`, path);
return dir;
}

/** The sender's last failure when the person has not seen it yet, marking it seen. */
function unseenFailure(env: NodeJS.ProcessEnv): string | null {
const path = join(stateDirectory(env), "failure");
Expand Down Expand Up @@ -368,13 +391,15 @@ async function sendOnce(deps: SendDependencies, dir: string): Promise<Sent[]> {
const [owner, repo] = origin.split("/") as [string, string];
const repository = await baseRepository(client, { owner, repo }, prefix);
// PRs are read from the last upload's read, if earlier, so a fix that merged after its sessions were sent,
// as through the merge queue, still reaches the store with the next session.
// as through the merge queue, still reaches the store with the next session. A resend reads from its earlier
// upload's read too, if earlier, so it drops no PR that upload named.
const read = deps.now() - CLOCK_SKEW;
const pulls: Pull[] = await updatedPulls(
client,
repository,
Math.min(since, readCursor(deps.env, id) ?? since),
const from = Math.min(
since,
readCursor(deps.env, id) ?? since,
...group.map((q) => q.pending.read ?? Number.POSITIVE_INFINITY),
);
const pulls: Pull[] = await updatedPulls(client, repository, from);
const emit = new Set<Session>();
const missing: typeof group = [];
for (const q of group) {
Expand All @@ -383,9 +408,11 @@ async function sendOnce(deps: SendDependencies, dir: string): Promise<Sent[]> {
else missing.push(q);
}
// A session token-max no longer lists, as when its transcript was deleted, can never be built: it is
// dropped and said, so it does not hold back the rest of its group forever.
// dropped and said, so it does not hold back the rest of its group forever. One remembered to name a PR
// later is dropped silently, since its first upload sent everything but that PR.
for (const q of missing) {
settle(q);
if (forget(deps.env, id, q.pending)) continue;
notes.push(
`token-max lists no ${q.pending.harness} session ${q.pending.session} on this host, so its record was dropped`,
);
Expand Down Expand Up @@ -494,6 +521,9 @@ async function sendOnce(deps: SendDependencies, dir: string): Promise<Sent[]> {
if (!response.ok)
throw new Error(`${endpoint} answered ${response.status}: ${(await response.text()).slice(0, 500)}`);
clearBackoff(deps.env, id);
remember(deps.env, id, { group, emit, byId, from, waiting: unheaded([...emit], exported, pulls) });
resendDue(deps.env, id, pulls, group);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
// Only once every resend these PRs call for is queued, so a pass that fails before then reads them again.
writeCursor(deps.env, id, read);
for (const q of group) settle(q);
sent.push({
Expand Down Expand Up @@ -669,6 +699,131 @@ function clearBackoff(env: NodeJS.ProcessEnv, id: string): void {
rmSync(backoffPath(env, id), { force: true });
}

/**
* A sent session whose tree's record could not name a PR one of its branches may yet head: the latest branch of it or
* a sub-agent sent with it headed no PR. It is queued again once one does, so that PR's cost and time count it.
*/
interface Remembered {
/** Its stop as last queued, with where that upload read PRs from, queued again under a new ID to resend it. */
pending: Pending;
/** Its transcript's files: once none is left, token-max lists it no more, so it is forgotten. */
files: string[];
/** Each latest branch in its tree that headed no PR when it was last sent. */
waiting: Reached[];
}

/** Group `id`'s remembered sessions, a file each, so a pass reads only its own group's. */
const rememberedDir = (env: NodeJS.ProcessEnv, id: string) => join(stateDirectory(env), "remembered", digest(id));
const rememberedPath = (env: NodeJS.ProcessEnv, id: string, pending: Pending) =>
join(rememberedDir(env, id), `${digest(`${pending.harness}\0${pending.session}`)}.json`);

/**
* Decide afresh, for each session of `group` the upload sent, whether to remember it: when the latest branch of any
* session in the tree it was sent with, it and its sub-agents, headed no PR. The whole tree counts, since
* `buildRecords` passes PRs between parent and sub-agent, so a PR opening on a sub-agent's branch changes its parent's
* record too.
*/
function remember(
env: NodeJS.ProcessEnv,
id: string,
sent: {
group: { pending: Pending }[];
emit: Set<Session>;
byId: Map<string, Session>;
/** Where the upload's PR read began. */
from: number;
waiting: Map<Session, Reached>;
},
): void {
const children = new Map<string, Session[]>();
for (const s of sent.emit) {
if (s.parent === null) continue;
const parent = `${s.harness}\0${s.parent}`;
children.set(parent, [...(children.get(parent) ?? []), s]);
}
// A session that stopped twice is queued twice; its latest stop stands for it.
const roots = new Map(sent.group.map((q) => [`${q.pending.harness}\0${q.pending.session}`, q.pending]));
for (const [name, pending] of roots) {
const root = sent.byId.get(name);
if (root === undefined || !sent.emit.has(root)) continue;
const tree = new Set([root]);
for (const s of tree) for (const child of children.get(`${s.harness}\0${s.id}`) ?? []) tree.add(child);
const waiting = [
...new Map(
[...tree].flatMap((s) => {
const reached = sent.waiting.get(s);
return reached ? [[JSON.stringify(reached), reached] as const] : [];
}),
).values(),
];
const path = rememberedPath(env, id, pending);
if (waiting.length === 0) {
rmSync(path, { force: true });
continue;
}
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
writeFileSync(
`${path}.new`,
JSON.stringify({
pending: { ...pending, read: sent.from },
files: root.files,
waiting,
} satisfies Remembered),
);
renameSync(`${path}.new`, path);
}
}

/** Forget `pending`'s session, returning whether it was remembered. */
function forget(env: NodeJS.ProcessEnv, id: string, pending: Pending): boolean {
const path = rememberedPath(env, id, pending);
if (!existsSync(path)) return false;
rmSync(path, { force: true });
return true;
}

/**
* Queue again each session group `id` remembers once a PR among `pulls` heads one of its waiting branches, so the
* group's next pass rebuilds its tree, and forget each whose transcript is gone. The group's sessions in `sent` were
* just rebuilt, so they are left as `remember` decided.
*/
function resendDue(env: NodeJS.ProcessEnv, id: string, pulls: Pull[], sent: { pending: Pending }[]): void {
const dir = rememberedDir(env, id);
if (!existsSync(dir)) return;
const fresh = new Set(sent.map((q) => rememberedPath(env, id, q.pending)));
const heads = heading(pulls);
for (const name of readdirSync(dir)) {
const path = join(dir, name);
if (!name.endsWith(".json") || fresh.has(path)) continue;
const kept = readRemembered(path);
// An unreadable file loses only the resend of a record already sent once, so it never blocks the group.
if (kept === null || !kept.files.some((file) => existsSync(file))) rmSync(path, { force: true });
else if (kept.waiting.some(heads)) enqueue(env, { ...kept.pending, queued: randomUUID() });
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}

function readRemembered(path: string): Remembered | null {
try {
const value = JSON.parse(readFileSync(path, "utf8")) as Partial<Remembered> | null;
return typeof value?.pending === "object" &&
value.pending !== null &&
Array.isArray(value.files) &&
value.files.every((file) => typeof file === "string") &&
Array.isArray(value.waiting) &&
value.waiting.every(
(each) =>
typeof each?.name === "string" &&
typeof each.first === "number" &&
Array.isArray(each.repositories) &&
each.repositories.every((r) => typeof r === "string"),
)
? (value as Remembered)
: null;
} catch {
return null;
}
}

/** Whether a checkout still holds `dir` or the nearest of its parents that exists, as `questRoot` finds it. */
function insideCheckout(dir: string): boolean {
for (let at = dir; ; at = dirname(at)) {
Expand Down
Loading
Loading