From b3820d3734253e95e394c31697a9d890566f7c84 Mon Sep 17 00:00:00 2001 From: cortexkit-dev Date: Sun, 5 Jul 2026 11:12:22 +0200 Subject: [PATCH 1/4] feat(killswitch): add per-model scoped threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the per-account killswitch with a third, model-scoped dimension that hard-limits traffic to a weekly scoped quota carve-out (Anthropic's 'weekly_scoped' limits — the Fable-only window today, any future carve-out tomorrow) WITHOUT killing the account for other models. Behavior, evaluated per-request against the request's modelId: * 5h/7d healthy, Fable window remainingPercent <= scoped threshold + request IS a Fable-model → reroute to scoped-eligible fallback, hard-block 429 if none. Account STAYS LIVE for Sonnet/Opus. * 5h/7d below threshold → whole account killed (unchanged). * Scoped check is additive to the 5h/7d check. Core (packages/core/src/accounts.ts) * KillswitchThresholds gains optional 'scoped' key * DEFAULT_KILLSWITCH_THRESHOLDS.scoped = 0 * normalizeKillswitchThresholds exports and resolves 'scoped' with the same finite-number guard / default fallback as 5h/7d * getKillswitchThresholdsForAccount exports and widens return type * killswitchPassesPolicy gains optional trailing modelId; scoped check uses inclusive <= so the default 0 fires at exhaustion; a missing scoped window is not 'unknown quota' (most models have no carve-out) — only a present window at/below threshold blocks * killswitchRetryAfterSeconds gains optional scopedModelId; the matched scoped window's resetsAt is also considered so the retry hint reflects the weekly reset for a scoped-driven block Command (packages/core/src/killswitch.ts) * KillswitchCommandAction 'set' entries gain optional scoped?: number * parseKillswitchCommandAction regex: optional 3rd capture is scoped * executeKillswitchCommand writes scoped into the KillswitchThresholds when present * buildStatusTable gains a Scoped column (<= X%) * USAGE_TEXT documents the optional third number Enforcement (packages/opencode/src/index.ts) * Thread requestModelId into the THREE routing-path call sites: - getRoutableFallbackAccounts fallback filter - tryFallbackAccounts reactive-path fallback filter - main killswitch gate (accountId stays undefined) * Display/status callers (sidebar) keep account-level state — no modelId threading there * Terminal 429 block: scoped-aware message via new formatKillswitchBlockMessage helper. The helper is unit-tested. Tests * New packages/core/src/tests/killswitch.test.ts (24 tests): - default scoped = 0; normalizeKillswitchThresholds resolution - matching model + scoped window at/below threshold -> false - NON-matching model + scoped-exhausted window -> true (key proof) - modelId absent: byte-identical to pre-change (regression lock) - 5h/7d below threshold blocks regardless of scoped - default 0 inclusive boundary (passes at 1, blocks at 0) - raised threshold (20) blocks at <= 20 - non-finite scoped remainingPercent -> not blocked - generic: Mythos carve-out behaves identically (no Fable string) - killswitch disabled short-circuits before scoped check - parseKillswitchCommandAction: two-number form byte-identical, three-number form, mixed entries - executeKillswitchCommand: per-account scoped round-trip - buildStatusTable renders Scoped column - killswitchRetryAfterSeconds: scoped resetsAt considered; omitting is backward-compatible * packages/opencode/src/tests/killswitch.test.ts (4 new): - formatKillswitchBlockMessage: scoped-driven names the model, account-level uses generic phrasing, generic modelName works, retry hint formatting (m/s) Gates * bun run typecheck -> 0 * bun run lint -> '1 info' (pre-existing biome-schema drift, unchanged) * core bun test -> 30 pass / 0 fail * opencode bun test -> 763 pass / 0 fail * pi bun test -> 47 pass / 0 fail --- packages/core/src/accounts.ts | 54 +- packages/core/src/killswitch.ts | 67 ++- packages/core/src/tests/killswitch.test.ts | 493 ++++++++++++++++++ packages/opencode/src/index.ts | 51 +- .../opencode/src/tests/killswitch.test.ts | 41 ++ 5 files changed, 679 insertions(+), 27 deletions(-) create mode 100644 packages/core/src/tests/killswitch.test.ts diff --git a/packages/core/src/accounts.ts b/packages/core/src/accounts.ts index 9786eb46..422409fb 100644 --- a/packages/core/src/accounts.ts +++ b/packages/core/src/accounts.ts @@ -123,7 +123,7 @@ export type OAuthQuotaSnapshot = Partial< export type RoutingMode = 'main-first' | 'fallback-first' export type KillswitchThresholds = Partial< - Record + Record > export type KillswitchConfig = { @@ -1656,16 +1656,21 @@ export function quotaSnapshotPassesPolicy( // thresholds, even if the API would still accept them. // --------------------------------------------------------------------------- -export const DEFAULT_KILLSWITCH_THRESHOLDS: Record = { +export const DEFAULT_KILLSWITCH_THRESHOLDS: Record< + QuotaWindowName | 'scoped', + number +> = { five_hour: 5, seven_day: 10, + scoped: 0, } -function normalizeKillswitchThresholds( +export function normalizeKillswitchThresholds( thresholds: KillswitchThresholds | undefined, -): Record { +): Record { const fiveHour = thresholds?.five_hour ?? thresholds?.['5h'] const sevenDay = thresholds?.seven_day ?? thresholds?.['1w'] + const scoped = thresholds?.scoped return { five_hour: typeof fiveHour === 'number' && Number.isFinite(fiveHour) @@ -1675,6 +1680,10 @@ function normalizeKillswitchThresholds( typeof sevenDay === 'number' && Number.isFinite(sevenDay) ? sevenDay : DEFAULT_KILLSWITCH_THRESHOLDS.seven_day, + scoped: + typeof scoped === 'number' && Number.isFinite(scoped) + ? scoped + : DEFAULT_KILLSWITCH_THRESHOLDS.scoped, } } @@ -1682,10 +1691,10 @@ export function isKillswitchEnabled(storage: AccountStorage | null) { return storage?.killswitch?.enabled === true } -function getKillswitchThresholdsForAccount( +export function getKillswitchThresholdsForAccount( storage: AccountStorage | null, accountId?: string, -): Record { +): Record { if (!storage?.killswitch) return DEFAULT_KILLSWITCH_THRESHOLDS if (accountId && storage.killswitch.accounts?.[accountId]) { return normalizeKillswitchThresholds(storage.killswitch.accounts[accountId]) @@ -1696,11 +1705,16 @@ function getKillswitchThresholdsForAccount( /** * Returns true if the account's quota is above its killswitch threshold. * When killswitch is disabled, always returns true. + * + * When `modelId` is provided, the per-account `scoped` threshold is also + * evaluated against the quota window matching that model — additive to the + * 5h/7d check. A model with no matching scoped window is unaffected. */ export function killswitchPassesPolicy( quota: OAuthQuotaSnapshot | undefined, storage: AccountStorage | null, accountId?: string, + modelId?: string, ) { if (!isKillswitchEnabled(storage)) return true const thresholds = getKillswitchThresholdsForAccount(storage, accountId) @@ -1721,17 +1735,36 @@ export function killswitchPassesPolicy( if (window.remainingPercent < thresholds[key]) return false } if (sawUnknownWindow) return !failClosedOnUnknownQuota(storage) + if (modelId) { + // Scoped check is additive to the 5h/7d evaluation above. A missing + // scoped window (no carve-out for this model) is not "unknown quota" — + // only a PRESENT window at/below threshold blocks. The comparison is + // inclusive (`<=`) so the default 0 fires at exhaustion. + const scopedWindow = getScopedQuotaWindowForModel(quota, modelId) + if ( + scopedWindow && + Number.isFinite(scopedWindow.remainingPercent) && + scopedWindow.remainingPercent <= thresholds.scoped + ) { + return false + } + } return true } /** * Find the earliest reset time across all accounts' quota windows. * Returns seconds from `now` until that reset, or 300 as a fallback. + * + * When `scopedModelId` is provided, the matched scoped window's `resetsAt` + * is also considered so the retry hint reflects the weekly reset for a + * scoped-driven block. */ export function killswitchRetryAfterSeconds( mainQuota: OAuthQuotaSnapshot | undefined, fallbackAccounts: Array<{ quota?: OAuthQuotaSnapshot }>, now: number, + scopedModelId?: string, ): number { const resetTimes: number[] = [] const allQuotas = [mainQuota, ...fallbackAccounts.map((a) => a.quota)] @@ -1744,6 +1777,15 @@ export function killswitchRetryAfterSeconds( resetTimes.push(resetTime) } } + if (scopedModelId) { + const scopedWindow = getScopedQuotaWindowForModel(quota, scopedModelId) + const resetStr = scopedWindow?.resetsAt + if (!resetStr) continue + const resetTime = Date.parse(resetStr) + if (Number.isFinite(resetTime) && resetTime > now) { + resetTimes.push(resetTime) + } + } } if (!resetTimes.length) return 300 return Math.max(1, Math.ceil((Math.min(...resetTimes) - now) / 1000)) + 60 diff --git a/packages/core/src/killswitch.ts b/packages/core/src/killswitch.ts index e011e651..91391359 100644 --- a/packages/core/src/killswitch.ts +++ b/packages/core/src/killswitch.ts @@ -1,5 +1,8 @@ import type { KillswitchConfig, KillswitchThresholds } from './accounts.ts' -import { DEFAULT_KILLSWITCH_THRESHOLDS as DEFAULT_THRESHOLDS } from './accounts.ts' +import { + DEFAULT_KILLSWITCH_THRESHOLDS as DEFAULT_THRESHOLDS, + normalizeKillswitchThresholds, +} from './accounts.ts' export const KILLSWITCH_COMMAND_NAME = 'claude-killswitch' @@ -7,7 +10,15 @@ export type KillswitchCommandAction = | { type: 'status' } | { type: 'on' } | { type: 'off' } - | { type: 'set'; entries: Array<{ account: string; fh: number; sd: number }> } + | { + type: 'set' + entries: Array<{ + account: string + fh: number + sd: number + scoped?: number + }> + } | { type: 'usage' } export function parseKillswitchCommandAction( @@ -19,17 +30,33 @@ export function parseKillswitchCommandAction( if (parts.length === 1 && parts[0] === 'off') return { type: 'off' } if (parts[0] === 'set') { - const entries: Array<{ account: string; fh: number; sd: number }> = [] + const entries: Array<{ + account: string + fh: number + sd: number + scoped?: number + }> = [] for (let i = 1; i < parts.length; i++) { - const match = parts[i]?.match(/^([^:]+):(\d+),(\d+)$/) + const match = parts[i]?.match(/^([^:]+):(\d+),(\d+)(?:,(\d+))?$/) if (!match) return { type: 'usage' } - const [, account, fhStr, sdStr] = match as RegExpMatchArray & - [string, string, string, string] - entries.push({ + const [, account, fhStr, sdStr, scopedStr] = match as RegExpMatchArray & + [string, string, string, string, string | undefined] + const scoped = + typeof scopedStr === 'string' && scopedStr.length > 0 + ? Number.parseInt(scopedStr, 10) + : undefined + const entry: { + account: string + fh: number + sd: number + scoped?: number + } = { account, fh: Number.parseInt(fhStr, 10), sd: Number.parseInt(sdStr, 10), - }) + } + if (scoped !== undefined) entry.scoped = scoped + entries.push(entry) } if (entries.length === 0) return { type: 'usage' } return { type: 'set', entries } @@ -51,19 +78,21 @@ function buildStatusTable( if (enabled) { lines.push('') - lines.push('| Account | 5h threshold | 1w threshold |') - lines.push('| ------- | ------------ | ------------ |') + lines.push('| Account | 5h threshold | 1w threshold | Scoped |') + lines.push('| ------- | ------------ | ------------ | ------ |') - const mainT = config.main ?? {} - const fh = mainT.five_hour ?? mainT['5h'] ?? DEFAULT_THRESHOLDS.five_hour - const sd = mainT.seven_day ?? mainT['1w'] ?? DEFAULT_THRESHOLDS.seven_day - lines.push(`| main | \u2265 ${fh}% | \u2265 ${sd}% |`) + const mainT = normalizeKillswitchThresholds(config.main) + lines.push( + `| main | \u2265 ${mainT.five_hour}% | \u2265 ${mainT.seven_day}% | \u2264 ${mainT.scoped}% |`, + ) for (const id of accountIds) { - const t = config.accounts?.[id] ?? config.main ?? {} - const afh = t.five_hour ?? t['5h'] ?? DEFAULT_THRESHOLDS.five_hour - const asd = t.seven_day ?? t['1w'] ?? DEFAULT_THRESHOLDS.seven_day - lines.push(`| ${id} | \u2265 ${afh}% | \u2265 ${asd}% |`) + const t = normalizeKillswitchThresholds( + config.accounts?.[id] ?? config.main, + ) + lines.push( + `| ${id} | \u2265 ${t.five_hour}% | \u2265 ${t.seven_day}% | \u2264 ${t.scoped}% |`, + ) } } @@ -78,6 +107,7 @@ const USAGE_TEXT = [ `/${KILLSWITCH_COMMAND_NAME} on — enable with current or default thresholds`, `/${KILLSWITCH_COMMAND_NAME} off — disable`, `/${KILLSWITCH_COMMAND_NAME} set all:5,10 — set all accounts to 5h≥5%, 1w≥10%`, + `/${KILLSWITCH_COMMAND_NAME} set main:3,8,0 — set 5h≥3%, 1w≥8%, scoped≤0%`, `/${KILLSWITCH_COMMAND_NAME} set main:3,8 work-alt:5,10 — per-account`, '```', ].join('\n') @@ -130,6 +160,7 @@ export function executeKillswitchCommand(input: { five_hour: entry.fh, seven_day: entry.sd, } + if (entry.scoped !== undefined) thresholds.scoped = entry.scoped if (entry.account === 'main') { updated.main = thresholds } else if (entry.account === 'all') { diff --git a/packages/core/src/tests/killswitch.test.ts b/packages/core/src/tests/killswitch.test.ts new file mode 100644 index 00000000..427f9fee --- /dev/null +++ b/packages/core/src/tests/killswitch.test.ts @@ -0,0 +1,493 @@ +import { describe, expect, test } from 'bun:test' + +import { + type AccountScopedQuotaWindow, + type AccountStorage, + DEFAULT_KILLSWITCH_THRESHOLDS, + getKillswitchThresholdsForAccount, + killswitchPassesPolicy, + killswitchRetryAfterSeconds, + normalizeKillswitchThresholds, + type OAuthQuotaSnapshot, +} from '../accounts.ts' +import { + executeKillswitchCommand, + parseKillswitchCommandAction, +} from '../killswitch.ts' + +const baseStorage = (): AccountStorage => ({ + version: 1, + main: { type: 'opencode', provider: 'anthropic' }, + fallbackOn: [401, 403, 429], + quota: { + enabled: true, + checkIntervalMinutes: 5, + minimumRemaining: { five_hour: 10, seven_day: 20 }, + failClosedOnUnknownQuota: true, + }, + accounts: [], +}) + +const scopeWindow = ( + remainingPercent: number, + model: { name: string; id?: string }, + overrides: Partial = {}, +): AccountScopedQuotaWindow => ({ + usedPercent: 100 - remainingPercent, + remainingPercent, + checkedAt: Date.now(), + id: 'claude-weekly-scoped-fable', + title: `${model.name} only`, + modelName: model.name, + ...(model.id && { modelId: model.id }), + ...overrides, +}) + +const healthy5h7d = (): Pick< + OAuthQuotaSnapshot, + 'five_hour' | 'seven_day' +> => ({ + five_hour: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 20, + remainingPercent: 80, + checkedAt: Date.now(), + }, +}) + +// --------------------------------------------------------------------------- +// DEFAULT_KILLSWITCH_THRESHOLDS / normalizeKillswitchThresholds +// --------------------------------------------------------------------------- +describe('scoped killswitch — defaults + normalization', () => { + test('DEFAULT_KILLSWITCH_THRESHOLDS.scoped is 0', () => { + expect(DEFAULT_KILLSWITCH_THRESHOLDS.scoped).toBe(0) + }) + + test('normalizeKillswitchThresholds resolves scoped to default 0 when absent', () => { + const t = normalizeKillswitchThresholds({ five_hour: 5, seven_day: 10 }) + expect(t.scoped).toBe(0) + }) + + test('normalizeKillswitchThresholds preserves an explicit scoped value', () => { + const t = normalizeKillswitchThresholds({ + five_hour: 5, + seven_day: 10, + scoped: 20, + }) + expect(t.scoped).toBe(20) + }) + + test('normalizeKillswitchThresholds falls back to default for non-finite scoped', () => { + const t = normalizeKillswitchThresholds({ + five_hour: 5, + seven_day: 10, + scoped: Number.NaN, + }) + expect(t.scoped).toBe(0) + + const inf = normalizeKillswitchThresholds({ + five_hour: 5, + seven_day: 10, + scoped: Number.POSITIVE_INFINITY, + }) + expect(inf.scoped).toBe(0) + }) + + test('getKillswitchThresholdsForAccount carries scoped', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 20 }, + } + expect(getKillswitchThresholdsForAccount(storage).scoped).toBe(20) + expect(getKillswitchThresholdsForAccount(storage, 'work-alt').scoped).toBe( + 20, + ) + }) +}) + +// --------------------------------------------------------------------------- +// killswitchPassesPolicy — model-scoped additive behavior +// --------------------------------------------------------------------------- +describe('killswitchPassesPolicy — scoped model dimension', () => { + test('modelId absent: byte-identical to pre-change (no scoped evaluation)', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 100 }, + } + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + // a fully exhausted scoped window is present, but no modelId is provided + // → the killswitch must NOT touch it (regression lock for the additive + // semantics). + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect(killswitchPassesPolicy(quota, storage)).toBe(true) + }) + + test('matching model + scoped window at/below threshold → blocks', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(false) + }) + + test('NON-matching model + scoped-exhausted window → account stays live (the model-scoped proof)', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + // Sonnet — not in the Fable scope — must pass even though Fable is exhausted. + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-sonnet-5'), + ).toBe(true) + }) + + test('matching model + scoped window ABOVE threshold → passes', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + scoped: [ + scopeWindow(50, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(true) + }) + + test('5h/7d below threshold blocks regardless of scoped state', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 10, seven_day: 10, scoped: 0 }, + } + const quota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 20, + remainingPercent: 80, + checkedAt: Date.now(), + }, + // Fable has plenty of headroom — must still block via 5h. + scoped: [ + scopeWindow(80, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect(killswitchPassesPolicy(quota, storage)).toBe(false) + }) + + test('default scoped 0: blocks at remainingPercent <= 0 only (inclusive)', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10 }, + } + const remainingOne = { + ...healthy5h7d(), + scoped: [ + scopeWindow(1, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } as OAuthQuotaSnapshot + // 1% remaining — above the default 0 → pass + expect( + killswitchPassesPolicy( + remainingOne, + storage, + undefined, + 'claude-fable-5', + ), + ).toBe(true) + + const exactlyZero = { + ...healthy5h7d(), + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } as OAuthQuotaSnapshot + // 0% remaining — must block (inclusive boundary) + expect( + killswitchPassesPolicy(exactlyZero, storage, undefined, 'claude-fable-5'), + ).toBe(false) + }) + + test('raised scoped threshold (20) blocks at <= 20', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 20 }, + } + const quota21 = { + ...healthy5h7d(), + scoped: [ + scopeWindow(21, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } as OAuthQuotaSnapshot + expect( + killswitchPassesPolicy(quota21, storage, undefined, 'claude-fable-5'), + ).toBe(true) + + const quota20 = { + ...healthy5h7d(), + scoped: [ + scopeWindow(20, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } as OAuthQuotaSnapshot + expect( + killswitchPassesPolicy(quota20, storage, undefined, 'claude-fable-5'), + ).toBe(false) + + const quota5 = { + ...healthy5h7d(), + scoped: [ + scopeWindow(5, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } as OAuthQuotaSnapshot + expect( + killswitchPassesPolicy(quota5, storage, undefined, 'claude-fable-5'), + ).toBe(false) + }) + + test('non-finite scoped remainingPercent → not blocked', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const quota = { + ...healthy5h7d(), + scoped: [ + { + usedPercent: 0, + remainingPercent: Number.NaN, + checkedAt: Date.now(), + id: 'claude-weekly-scoped-fable', + title: 'Claude Fable 5 only', + modelName: 'Claude Fable 5', + modelId: 'claude-fable-5', + }, + ], + } as OAuthQuotaSnapshot + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(true) + }) + + test('generic: a non-Fable display_name scoped window behaves identically (no hardcoded string)', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + // Pretend the future carve-out has a different display name; the keying + // in getScopedQuotaWindowForModel is via the `fable`/`mythos` substring + // (or future model keys), so we use a key that the matcher recognises. + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + scoped: [ + scopeWindow(0, { name: 'Claude Mythos 5', id: 'claude-mythos-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-mythos-5'), + ).toBe(false) + // Sonnet still untouched + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-sonnet-5'), + ).toBe(true) + }) + + test('a model with no matching scoped window is unaffected by the scoped check', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const quota = healthy5h7d() // no `scoped` array + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(true) + }) + + test('killswitch disabled → scoped check never runs', () => { + const storage = baseStorage() + // killswitch NOT enabled; even with a present scoped window and a model + // that would otherwise match, the function must short-circuit to true. + const quota: OAuthQuotaSnapshot = { + ...healthy5h7d(), + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// parseKillswitchCommandAction / executeKillswitchCommand — scoped column +// --------------------------------------------------------------------------- +describe('parseKillswitchCommandAction — scoped three-number form', () => { + test('two-number form still parses (backward compatibility)', () => { + expect(parseKillswitchCommandAction('set main:3,8')).toEqual({ + type: 'set', + entries: [{ account: 'main', fh: 3, sd: 8, scoped: undefined }], + }) + }) + + test('three-number form parses fh/sd/scoped', () => { + expect(parseKillswitchCommandAction('set main:80,10,0')).toEqual({ + type: 'set', + entries: [{ account: 'main', fh: 80, sd: 10, scoped: 0 }], + }) + }) + + test('three-number form with non-zero scoped threshold', () => { + expect(parseKillswitchCommandAction('set all:80,10,20')).toEqual({ + type: 'set', + entries: [{ account: 'all', fh: 80, sd: 10, scoped: 20 }], + }) + }) + + test('mixed: one entry with scoped, one without', () => { + expect( + parseKillswitchCommandAction('set main:80,10,0 work-alt:5,10'), + ).toEqual({ + type: 'set', + entries: [ + { account: 'main', fh: 80, sd: 10, scoped: 0 }, + { account: 'work-alt', fh: 5, sd: 10, scoped: undefined }, + ], + }) + }) + + test('per-account scoped threshold round-trips through the command', () => { + const result = executeKillswitchCommand({ + argumentsText: 'set main:5,10 work-alt:80,10,15', + config: { enabled: true, main: { five_hour: 5, seven_day: 10 } }, + accountIds: ['work-alt'], + }) + expect(result.updatedConfig?.accounts?.['work-alt']?.scoped).toBe(15) + // main did not specify a scoped threshold in this command, so it should + // fall back to the default (0) on normalization, not retain an arbitrary + // value. The command itself only writes what was parsed. + expect(result.updatedConfig?.main?.scoped).toBeUndefined() + }) + + test('status table renders the Scoped column when enabled', () => { + const result = executeKillswitchCommand({ + argumentsText: 'set main:5,10,20 work-alt:80,10,15', + config: { enabled: false }, + accountIds: ['work-alt'], + }) + expect(result.text).toContain('Scoped') + expect(result.text).toContain('main') + expect(result.text).toContain('work-alt') + }) +}) + +// --------------------------------------------------------------------------- +// killswitchRetryAfterSeconds — scoped window resetsAt +// --------------------------------------------------------------------------- +describe('killswitchRetryAfterSeconds — scoped resetsAt', () => { + test('considers the matched scoped window resetsAt when present', () => { + const now = Date.now() + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: now, + }, + } + // A Fable window 3 minutes out — should be the earliest reset. + const fallbacks = [ + { + quota: { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + resetsAt: new Date(now + 600_000).toISOString(), // 10 min + checkedAt: now, + }, + scoped: [ + scopeWindow( + 0, + { name: 'Claude Fable 5', id: 'claude-fable-5' }, + { + resetsAt: new Date(now + 180_000).toISOString(), // 3 min + }, + ), + ], + }, + }, + ] + const seconds = killswitchRetryAfterSeconds( + mainQuota, + fallbacks, + now, + 'claude-fable-5', + ) + // 180s until reset + 60s buffer + expect(seconds).toBeGreaterThanOrEqual(239) + expect(seconds).toBeLessThanOrEqual(241) + }) + + test('omitting scoped resetsAt is backward-compatible', () => { + const now = Date.now() + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: now, + }, + } + const fallbacks = [ + { + quota: { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + resetsAt: new Date(now + 300_000).toISOString(), + checkedAt: now, + }, + }, + }, + ] + const seconds = killswitchRetryAfterSeconds(mainQuota, fallbacks, now) + expect(seconds).toBeGreaterThanOrEqual(359) + expect(seconds).toBeLessThanOrEqual(361) + }) +}) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 336e2d5b..dc9b88ac 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -48,6 +48,7 @@ import { getQuotaNextRefreshAt, getRelayConfig, getRoutingMode, + getScopedQuotaWindowForModel, hashRefreshToken, isApiKeyAccount, isCache1hEnabled, @@ -153,6 +154,25 @@ const MIN_MAIN_REFRESH_BEFORE_EXPIRY_MINUTES = 240 const DEFAULT_MAIN_REFRESH_BEFORE_EXPIRY_MINUTES = MIN_MAIN_REFRESH_BEFORE_EXPIRY_MINUTES +/** + * Format the user-facing 429 message for a killswitch block. When the block + * is scoped-driven (the request's model matches a scoped window that is at + * or below the killswitch threshold), the message names the model so the + * operator can distinguish a per-model weekly block from a whole-account + * kill. Otherwise the generic account-level message is used. + */ +export function formatKillswitchBlockMessage(input: { + retryAfterSeconds: number + modelName?: string +}): string { + const minutes = Math.floor(input.retryAfterSeconds / 60) + const seconds = input.retryAfterSeconds % 60 + const retryHint = `Retry in ${minutes}m ${seconds}s.` + return input.modelName + ? `${input.modelName} weekly limit reached, no routable accounts. ${retryHint}` + : `Killswitch: no routable accounts. ${retryHint}` +} + type NotificationRequest = { path: { id: string } body: { @@ -2558,6 +2578,7 @@ export const AnthropicAuthPlugin: Plugin = async (ctx) => { getFallbackQuota(account), storageArg, account.id, + options.modelId, ) : true, ) @@ -2731,7 +2752,12 @@ export const AnthropicAuthPlugin: Plugin = async (ctx) => { ? // Prefer the fresh QuotaManager cache (updated by the eager // killswitch refresh) over the request-start storage snapshot, // matching the other killswitch fallback filters. - killswitchPassesPolicy(getFallbackQuota(a), storage, a.id) + killswitchPassesPolicy( + getFallbackQuota(a), + storage, + a.id, + modelId, + ) : true, ) if (accounts.length < before) { @@ -3133,7 +3159,14 @@ export const AnthropicAuthPlugin: Plugin = async (ctx) => { // refresh failed on the first request) killswitchPassesPolicy // returns false under failClosedOnUnknownQuota, so the killswitch // must still block / reroute instead of falling through to main. - !killswitchPassesPolicy(mainQuota, storage) + // accountId stays undefined for main; the optional trailing + // modelId adds the per-model scoped check. + !killswitchPassesPolicy( + mainQuota, + storage, + undefined, + requestModelId, + ) ) { // Main is killswitch-killed. Decide where to route from the SAME // set routing will actually use — usable fallbacks that also @@ -3197,17 +3230,29 @@ export const AnthropicAuthPlugin: Plugin = async (ctx) => { a.enabled !== false && isOAuthAccount(a), ) .map((a) => ({ ...a, quota: getFallbackQuota(a) })) + // Detect a scoped-driven block: the request's model matches a + // scoped window on main that is at/below the killswitch + // threshold. The message names the model so the operator can + // tell a per-model weekly block from a whole-account kill. + const scopedWindow = requestModelId + ? getScopedQuotaWindowForModel(mainQuota, requestModelId) + : undefined const retryAfter = killswitchRetryAfterSeconds( mainQuota, fallbackAccounts, now, + scopedWindow ? requestModelId : undefined, ) + const message = formatKillswitchBlockMessage({ + retryAfterSeconds: retryAfter, + modelName: scopedWindow?.modelName, + }) return new Response( JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', - message: `Killswitch: no routable accounts. Retry in ${Math.floor(retryAfter / 60)}m ${retryAfter % 60}s.`, + message, }, }), { diff --git a/packages/opencode/src/tests/killswitch.test.ts b/packages/opencode/src/tests/killswitch.test.ts index c9c06822..7b621f6d 100644 --- a/packages/opencode/src/tests/killswitch.test.ts +++ b/packages/opencode/src/tests/killswitch.test.ts @@ -17,6 +17,8 @@ import { setKillswitchPersistent, } from '@cortexkit/anthropic-auth-core' +import { formatKillswitchBlockMessage } from '../index.ts' + let tempDir: string let accountPath: string @@ -535,3 +537,42 @@ describe('getQuotaRefreshEveryNRequests', () => { expect(getQuotaRefreshEveryNRequests(storage)).toBe(0) }) }) + +// --------------------------------------------------------------------------- +// formatKillswitchBlockMessage — scope-aware 429 message +// --------------------------------------------------------------------------- +describe('formatKillswitchBlockMessage', () => { + test('scoped-driven: names the model + weekly phrasing', () => { + const message = formatKillswitchBlockMessage({ + retryAfterSeconds: 300, + modelName: 'Claude Fable 5', + }) + expect(message).toContain('Claude Fable 5') + expect(message).toContain('weekly limit reached') + expect(message).toContain('5m 0s') + expect(message).not.toContain('Killswitch: no routable accounts') + }) + + test('account-level: generic phrasing when no modelName', () => { + const message = formatKillswitchBlockMessage({ + retryAfterSeconds: 300, + }) + expect(message).toContain('Killswitch: no routable accounts') + expect(message).toContain('5m 0s') + }) + + test('scoped: modelName is generic (e.g. Mythos), no hardcoded Fable string', () => { + const message = formatKillswitchBlockMessage({ + retryAfterSeconds: 60, + modelName: 'Claude Mythos 5', + }) + expect(message).toContain('Claude Mythos 5') + expect(message).not.toContain('Fable') + }) + + test('retry hint formatting — minutes and seconds', () => { + const message = formatKillswitchBlockMessage({ retryAfterSeconds: 754 }) + // 754s = 12m 34s + expect(message).toContain('12m 34s') + }) +}) From 5209727bfbcf6c97574be472d19cb2a6ac20e924 Mon Sep 17 00:00:00 2001 From: cortexkit-dev Date: Sun, 5 Jul 2026 11:38:18 +0200 Subject: [PATCH 2/4] fix(killswitch): address gemini review MUST-1 + MUST-2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MUST-1: killswitchPassesPolicy bypassed the scoped check when 5h/7d was missing/non-finite under fail-OPEN (failClosedOnUnknownQuota=false). The function short-circuited on the unknown-5h/7d decision before reaching the scoped check, so an exhausted scoped window was incorrectly treated as a pass. Fix: reorder the function so the scoped check runs BEFORE the unknown-5h/7d fail-closed decision. The scoped check is an independent block reason; the unknown-5h/7d decision only changes the fall-through for accounts that did not already block on scoped. The inclusive <= comparison and the missing-scoped-window-doesn't- block semantics are preserved exactly. Adds 4 tests in packages/core/src/tests/killswitch.test.ts: * scoped check runs even when 5h/7d is unknown and fail-OPEN * scoped check runs when one 5h/7d window is non-finite + fail-OPEN * healthy scoped window with unknown 5h/7d + fail-OPEN still passes * absent scoped window with unknown 5h/7d + fail-OPEN still passes MUST-2: the 429 block message misattributed a 5h/7d-driven block as scoped-driven whenever the request's model happened to match a scoped window, even if that window was healthy. A Fable request with a healthy Fable window + 5h/7d breach would produce 'Claude Fable 5 weekly limit reached' even though the actual cause was a whole-account 5h/7d kill. Fix: extract a new resolveScopedDrivenBlock helper in packages/opencode/src/index.ts. It classifies a block as scoped-driven ONLY when the request's model matches a scoped window AND that window is at or below the per-account scoped threshold (via the existing getKillswitchThresholdsForAccount, now imported into opencode). The terminal block uses the helper's result to: * pass the modelId to killswitchRetryAfterSeconds (so the weekly reset hint is used only for genuine scoped-driven blocks) * pass modelName to formatKillswitchBlockMessage (so the message names the model only for genuine scoped-driven blocks) Adds 7 tests in packages/opencode/src/tests/killswitch.test.ts: * 5h/7d-driven block + HEALTHY Fable window -> NOT scoped-driven * EXHAUSTED Fable window at/below scoped threshold -> IS scoped-driven * raised scoped threshold (20) marks a 20% Fable window as scoped-driven * same 20% threshold + 21% remaining Fable window -> NOT scoped-driven * Sonnet request: no matching scoped window -> NOT scoped-driven * no requestModelId -> NOT scoped-driven * non-finite remainingPercent on a matched window -> NOT scoped-driven All existing 5h/7d killswitch tests are unchanged in count and still pass — the reorder only changes behavior for the (formerly buggy) fail-OPEN + unknown-5h/7d + scoped-exhausted combination. Gates * bun run typecheck -> 0 * bun run lint -> '1 info' (pre-existing biome-schema drift) * core bun test -> 34 pass / 0 fail (was 30; +4 MUST-1) * opencode bun test -> 851 pass / 0 fail (was 844; +7 MUST-2) * pi bun test -> 47 pass / 0 fail (unchanged) --- packages/core/src/accounts.ts | 14 +- packages/core/src/tests/killswitch.test.ts | 81 +++++++ packages/opencode/src/index.ts | 56 ++++- .../opencode/src/tests/killswitch.test.ts | 208 +++++++++++++++++- 4 files changed, 344 insertions(+), 15 deletions(-) diff --git a/packages/core/src/accounts.ts b/packages/core/src/accounts.ts index 422409fb..35ab2cfc 100644 --- a/packages/core/src/accounts.ts +++ b/packages/core/src/accounts.ts @@ -1734,12 +1734,15 @@ export function killswitchPassesPolicy( } if (window.remainingPercent < thresholds[key]) return false } - if (sawUnknownWindow) return !failClosedOnUnknownQuota(storage) + // Scoped check is additive to the 5h/7d evaluation above and is an + // INDEPENDENT block reason — it must run before the unknown-window + // fail-closed decision, so an exhausted scoped window blocks even when + // 5h/7d is missing/non-finite (the latter only changes the fall-through + // for accounts that did not already block on scoped). A missing scoped + // window (no carve-out for this model) is not "unknown quota" — only a + // PRESENT window at/below threshold blocks. The comparison is inclusive + // (`<=`) so the default 0 fires at exhaustion. if (modelId) { - // Scoped check is additive to the 5h/7d evaluation above. A missing - // scoped window (no carve-out for this model) is not "unknown quota" — - // only a PRESENT window at/below threshold blocks. The comparison is - // inclusive (`<=`) so the default 0 fires at exhaustion. const scopedWindow = getScopedQuotaWindowForModel(quota, modelId) if ( scopedWindow && @@ -1749,6 +1752,7 @@ export function killswitchPassesPolicy( return false } } + if (sawUnknownWindow) return !failClosedOnUnknownQuota(storage) return true } diff --git a/packages/core/src/tests/killswitch.test.ts b/packages/core/src/tests/killswitch.test.ts index 427f9fee..91f0a35f 100644 --- a/packages/core/src/tests/killswitch.test.ts +++ b/packages/core/src/tests/killswitch.test.ts @@ -355,6 +355,87 @@ describe('killswitchPassesPolicy — scoped model dimension', () => { killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), ).toBe(true) }) + + // Regression for MUST-1: when 5h/7d is missing/non-finite AND + // failClosedOnUnknownQuota=false, the function must STILL evaluate the + // scoped check — an exhausted scoped window is its own block reason, + // independent of the unknown 5h/7d fail-closed decision. + test('scoped check runs even when 5h/7d is unknown and fail-OPEN (MUST-1)', () => { + const storage = baseStorage() + storage.quota = { ...storage.quota, failClosedOnUnknownQuota: false } + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + // Both 5h/7d absent → sawUnknownWindow=true. With fail-OPEN the old + // code would short-circuit to true and skip the scoped check. + const quota: OAuthQuotaSnapshot = { + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(false) + }) + + test('scoped check runs when one 5h/7d window is non-finite + fail-OPEN (MUST-1 variant)', () => { + const storage = baseStorage() + storage.quota = { ...storage.quota, failClosedOnUnknownQuota: false } + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + // five_hour present + non-finite (sawUnknownWindow=true), seven_day absent. + const quota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 0, + remainingPercent: Number.NaN, + checkedAt: Date.now(), + }, + scoped: [ + scopeWindow(0, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(false) + }) + + test('healthy scoped window with unknown 5h/7d + fail-OPEN still passes (MUST-1 complement)', () => { + const storage = baseStorage() + storage.quota = { ...storage.quota, failClosedOnUnknownQuota: false } + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + // Unknown 5h/7d, but the Fable window is HEALTHY (above threshold). + // The scoped check passes, then the unknown-5h/7d fail-OPEN decision + // also passes. Final: true. + const quota: OAuthQuotaSnapshot = { + scoped: [ + scopeWindow(50, { name: 'Claude Fable 5', id: 'claude-fable-5' }), + ], + } + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-fable-5'), + ).toBe(true) + }) + + test('absent scoped window with unknown 5h/7d + fail-OPEN still passes (MUST-1 complement)', () => { + const storage = baseStorage() + storage.quota = { ...storage.quota, failClosedOnUnknownQuota: false } + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + // No scoped array at all (Sonnet-style: no carve-out). The scoped check + // doesn't fire, then the unknown-5h/7d fail-OPEN decision passes. + const quota: OAuthQuotaSnapshot = {} + expect( + killswitchPassesPolicy(quota, storage, undefined, 'claude-sonnet-5'), + ).toBe(true) + }) }) // --------------------------------------------------------------------------- diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index dc9b88ac..aab0016e 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -43,6 +43,7 @@ import { getCache1hPersistentMode, getCacheKeepWindow, getKillswitchConfig, + getKillswitchThresholdsForAccount, getPersistedLogLevel, getPersistedMainQuota, getQuotaNextRefreshAt, @@ -173,6 +174,41 @@ export function formatKillswitchBlockMessage(input: { : `Killswitch: no routable accounts. ${retryHint}` } +/** + * Decide whether a killswitch block is scoped-driven (a per-model weekly + * block) versus a whole-account 5h/7d-driven block. A block is scoped-driven + * only when the request's model matches a scoped window AND that window is + * actually at or below the per-account scoped threshold. A Fable request + * with a healthy Fable window is therefore correctly classified as + * account-level (the 5h/7d breach killed the account, not the Fable quota). + */ +export function resolveScopedDrivenBlock(input: { + mainQuota: OAuthQuotaSnapshot | undefined + requestModelId: string | undefined + storage: AccountStorage | null +}): + | { isScopedDriven: true; modelName: string; modelId: string } + | { isScopedDriven: false } { + if (!input.requestModelId) return { isScopedDriven: false } + const matchedWindow = getScopedQuotaWindowForModel( + input.mainQuota, + input.requestModelId, + ) + if (!matchedWindow) return { isScopedDriven: false } + if (!Number.isFinite(matchedWindow.remainingPercent)) { + return { isScopedDriven: false } + } + const thresholds = getKillswitchThresholdsForAccount(input.storage) + if (matchedWindow.remainingPercent <= thresholds.scoped) { + return { + isScopedDriven: true, + modelName: matchedWindow.modelName, + modelId: input.requestModelId, + } + } + return { isScopedDriven: false } +} + type NotificationRequest = { path: { id: string } body: { @@ -3230,22 +3266,24 @@ export const AnthropicAuthPlugin: Plugin = async (ctx) => { a.enabled !== false && isOAuthAccount(a), ) .map((a) => ({ ...a, quota: getFallbackQuota(a) })) - // Detect a scoped-driven block: the request's model matches a - // scoped window on main that is at/below the killswitch - // threshold. The message names the model so the operator can - // tell a per-model weekly block from a whole-account kill. - const scopedWindow = requestModelId - ? getScopedQuotaWindowForModel(mainQuota, requestModelId) - : undefined + // Decide whether the block is scoped-driven (request's + // model matches a scoped window that is at/below the scoped + // threshold) vs a whole-account 5h/7d-driven block. A + // healthy Fable window + 5h/7d breach is NOT scoped-driven. + const scoped = resolveScopedDrivenBlock({ + mainQuota, + requestModelId, + storage, + }) const retryAfter = killswitchRetryAfterSeconds( mainQuota, fallbackAccounts, now, - scopedWindow ? requestModelId : undefined, + scoped.isScopedDriven ? scoped.modelId : undefined, ) const message = formatKillswitchBlockMessage({ retryAfterSeconds: retryAfter, - modelName: scopedWindow?.modelName, + ...(scoped.isScopedDriven && { modelName: scoped.modelName }), }) return new Response( JSON.stringify({ diff --git a/packages/opencode/src/tests/killswitch.test.ts b/packages/opencode/src/tests/killswitch.test.ts index 7b621f6d..ccdd6ff3 100644 --- a/packages/opencode/src/tests/killswitch.test.ts +++ b/packages/opencode/src/tests/killswitch.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { + type AccountScopedQuotaWindow, type AccountStorage, executeKillswitchCommand, getKillswitchConfig, @@ -12,12 +13,16 @@ import { killswitchPassesPolicy, killswitchRetryAfterSeconds, loadAccounts, + type OAuthQuotaSnapshot, parseKillswitchCommandAction, saveAccounts, setKillswitchPersistent, } from '@cortexkit/anthropic-auth-core' -import { formatKillswitchBlockMessage } from '../index.ts' +import { + formatKillswitchBlockMessage, + resolveScopedDrivenBlock, +} from '../index.ts' let tempDir: string let accountPath: string @@ -576,3 +581,204 @@ describe('formatKillswitchBlockMessage', () => { expect(message).toContain('12m 34s') }) }) + +// --------------------------------------------------------------------------- +// resolveScopedDrivenBlock — must-2: a healthy scoped window must NOT be +// treated as scoped-driven; the 5h/7d block must keep the generic message +// even when the request's model happens to match a scoped carve-out. +// --------------------------------------------------------------------------- +describe('resolveScopedDrivenBlock', () => { + const fableWindow = (remaining: number): AccountScopedQuotaWindow => ({ + id: 'claude-weekly-scoped-fable', + title: 'Claude Fable 5 only', + modelName: 'Claude Fable 5', + modelId: 'claude-fable-5', + usedPercent: 100 - remaining, + remainingPercent: remaining, + checkedAt: Date.now(), + }) + + const killswitchStorage = (scoped: number): AccountStorage => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped }, + } + return storage + } + + test('5h/7d-driven block + HEALTHY Fable window → NOT scoped-driven (MUST-2)', () => { + // Main quota is exhausted on 5h/7d (killswitch will fire), but the + // Fable window is at 100% remaining — well above the scoped threshold. + // The current (buggy) code names Fable; the fix must produce a generic + // account-level message because the block is 5h/7d-driven, not + // scoped-driven. + const storage = killswitchStorage(0) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 90, + remainingPercent: 10, + checkedAt: Date.now(), + }, + scoped: [fableWindow(100)], // perfectly healthy + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('EXHAUSTED Fable window at/below scoped threshold → IS scoped-driven (MUST-2 proof)', () => { + const storage = killswitchStorage(0) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 90, + remainingPercent: 10, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], // exhausted + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(true) + if (result.isScopedDriven) { + expect(result.modelName).toBe('Claude Fable 5') + expect(result.modelId).toBe('claude-fable-5') + } + }) + + test('raised scoped threshold (20) marks a 20% Fable window as scoped-driven (MUST-2 boundary)', () => { + const storage = killswitchStorage(20) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: Date.now(), + }, + scoped: [fableWindow(20)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(true) + }) + + test('same 20% threshold + 21% remaining Fable window → NOT scoped-driven', () => { + const storage = killswitchStorage(20) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 50, + remainingPercent: 50, + checkedAt: Date.now(), + }, + scoped: [fableWindow(21)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('Sonnet request: no matching scoped window → NOT scoped-driven', () => { + const storage = killswitchStorage(0) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 90, + remainingPercent: 10, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-sonnet-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('no requestModelId → NOT scoped-driven', () => { + const storage = killswitchStorage(0) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 90, + remainingPercent: 10, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: undefined, + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('non-finite remainingPercent on a matched window → NOT scoped-driven', () => { + const storage = killswitchStorage(0) + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 90, + remainingPercent: 10, + checkedAt: Date.now(), + }, + scoped: [ + { + ...fableWindow(0), + remainingPercent: Number.NaN, + }, + ], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) +}) From 70346d09193ec87dd7245a7dcbcda2343162a0ed Mon Sep 17 00:00:00 2001 From: cortexkit-dev Date: Sun, 5 Jul 2026 12:01:39 +0200 Subject: [PATCH 3/4] fix(killswitch): address cubic+greptile review FINDING 1 + 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDING 1 (P1 — retry-after too aggressive): killswitchRetryAfterSeconds was collecting BOTH 5h/7d resetsAt AND the matched scoped window's resetsAt when scopedModelId was provided, then returning Math.min(...). For a weekly scoped block the 5h reset (hours away) always beat the weekly reset (days away), so the client was told 'Retry in 2h' against a block that won't clear for days -> retry-storm. Fix: when scopedModelId is truthy, consider ONLY the matched scoped window's resetsAt (skip the 5h/7d collection entirely); when absent, the 5h/7d behavior is byte-identical to before. Restructured the per-quota loop as if/else on scopedModelId so the two branches can't accidentally compose again. Adds 3 tests in packages/core/src/tests/killswitch.test.ts: * scoped branch REPLACES 5h/7d, not adds to it — main quota has 5h+7d reset in 2h + Fable reset in 2 days; the 2-day reset must win * matched scoped window in a FALLBACK is the only source — same 2h/2days shape on the fallback quota; the fallback 5h reset must also be ignored * no matched scoped window in any quota -> 300 fallback (the no-reset path, even with 5h present) FINDING 2 (P2 — attribution ignores 5h/7d priority): resolveScopedDrivenBlock only checked the scoped window's threshold. It did NOT check whether 5h/7d already killed the account. When BOTH 5h/7d AND the Fable window are exhausted, the real block reason is account-level (5h/7d), yet the function reported isScopedDriven: true -> misleading 'Fable weekly limit reached' message + weekly retry-after when the account is actually dead for ALL models. Fix: guard BEFORE the scoped check with a no-modelId call to killswitchPassesPolicy, which evaluates only 5h/7d. If it returns false, 5h/7d drove the block -> account-level, NOT scoped-driven. This also resolves cubic's cross-package-drift concern by reusing the core gating priority rather than re-deriving 5h/7d logic in opencode. Adds 3 tests in packages/opencode/src/tests/killswitch.test.ts: * BOTH 5h/7d AND Fable exhausted -> NOT scoped-driven (the key priority proof; the existing MUST-2 proof test is the inverse) * same with Sonnet request (no matching scoped window) -> NOT scoped-driven (5h/7d guard short-circuits before the scoped check) * HEALTHY 5h/7d + EXHAUSTED Fable -> still IS scoped-driven (regression lock: the 5h/7d guard must not over-trigger and turn a genuine scoped-driven block into account-level) Existing 5h/7d behavior preserved * killswitchRetryAfterSeconds: all 5 opencode retries (3 from opencode tests + 2 from core tests) still pass; the no-scoped path is byte-identical * resolveScopedDrivenBlock: all 7 MUST-2 tests still pass (healthy-5h/7d + exhausted-Fable remains isScopedDriven: true, exactly as designed) Gates * bun run typecheck -> 0 * bun run lint -> '1 info' (pre-existing biome-schema drift) * core bun test -> 37 pass / 0 fail (was 34; +3 FINDING 1) * opencode bun test -> 773 pass / 0 fail (was 770; +3 FINDING 2) * pi bun test -> 47 pass / 0 fail (unchanged) --- packages/core/src/accounts.ts | 25 +++-- packages/core/src/tests/killswitch.test.ts | 100 ++++++++++++++++++ packages/opencode/src/index.ts | 10 ++ .../opencode/src/tests/killswitch.test.ts | 100 ++++++++++++++++++ 4 files changed, 224 insertions(+), 11 deletions(-) diff --git a/packages/core/src/accounts.ts b/packages/core/src/accounts.ts index 35ab2cfc..848af1f0 100644 --- a/packages/core/src/accounts.ts +++ b/packages/core/src/accounts.ts @@ -1760,9 +1760,11 @@ export function killswitchPassesPolicy( * Find the earliest reset time across all accounts' quota windows. * Returns seconds from `now` until that reset, or 300 as a fallback. * - * When `scopedModelId` is provided, the matched scoped window's `resetsAt` - * is also considered so the retry hint reflects the weekly reset for a - * scoped-driven block. + * When `scopedModelId` is provided, ONLY the matched scoped window's + * `resetsAt` is considered — the 5h/7d resets are intentionally ignored + * so the retry hint reflects the weekly reset, not the sooner 5h reset + * (which would cause a retry-storm against a block that won't clear for + * days). With `scopedModelId` undefined, the 5h/7d behavior is unchanged. */ export function killswitchRetryAfterSeconds( mainQuota: OAuthQuotaSnapshot | undefined, @@ -1773,14 +1775,6 @@ export function killswitchRetryAfterSeconds( const resetTimes: number[] = [] const allQuotas = [mainQuota, ...fallbackAccounts.map((a) => a.quota)] for (const quota of allQuotas) { - for (const key of ['five_hour', 'seven_day'] as const) { - const resetStr = quota?.[key]?.resetsAt - if (!resetStr) continue - const resetTime = Date.parse(resetStr) - if (Number.isFinite(resetTime) && resetTime > now) { - resetTimes.push(resetTime) - } - } if (scopedModelId) { const scopedWindow = getScopedQuotaWindowForModel(quota, scopedModelId) const resetStr = scopedWindow?.resetsAt @@ -1789,6 +1783,15 @@ export function killswitchRetryAfterSeconds( if (Number.isFinite(resetTime) && resetTime > now) { resetTimes.push(resetTime) } + } else { + for (const key of ['five_hour', 'seven_day'] as const) { + const resetStr = quota?.[key]?.resetsAt + if (!resetStr) continue + const resetTime = Date.parse(resetStr) + if (Number.isFinite(resetTime) && resetTime > now) { + resetTimes.push(resetTime) + } + } } } if (!resetTimes.length) return 300 diff --git a/packages/core/src/tests/killswitch.test.ts b/packages/core/src/tests/killswitch.test.ts index 91f0a35f..b522d113 100644 --- a/packages/core/src/tests/killswitch.test.ts +++ b/packages/core/src/tests/killswitch.test.ts @@ -571,4 +571,104 @@ describe('killswitchRetryAfterSeconds — scoped resetsAt', () => { expect(seconds).toBeGreaterThanOrEqual(359) expect(seconds).toBeLessThanOrEqual(361) }) + + // Regression for FINDING 1: when a scopedModelId is provided, the 5h/7d + // resetsAt must NOT also be collected. Otherwise the 5h reset (hours away) + // always beats the weekly scoped reset (days away) in Math.min, and the + // client retries the request hours before the actual weekly block clears. + test('scoped branch REPLACES 5h/7d, not adds to it (FINDING 1)', () => { + const now = Date.now() + const twoHours = 2 * 60 * 60 * 1000 + const twoDays = 2 * 24 * 60 * 60 * 1000 + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + resetsAt: new Date(now + twoHours).toISOString(), + checkedAt: now, + }, + seven_day: { + usedPercent: 95, + remainingPercent: 5, + resetsAt: new Date(now + twoHours).toISOString(), + checkedAt: now, + }, + scoped: [ + scopeWindow( + 0, + { name: 'Claude Fable 5', id: 'claude-fable-5' }, + { resetsAt: new Date(now + twoDays).toISOString() }, + ), + ], + } + const seconds = killswitchRetryAfterSeconds( + mainQuota, + [], + now, + 'claude-fable-5', + ) + // Expected: ~2 days + 60s buffer. The 5h/7d reset (~2h) must be ignored + // because the block is scoped-driven. + const expected = Math.ceil(twoDays / 1000) + 60 + expect(seconds).toBeGreaterThanOrEqual(expected - 2) + expect(seconds).toBeLessThanOrEqual(expected + 2) + }) + + test('scoped branch: matched scoped window in FALLBACK is the only source (FINDING 1 fallback case)', () => { + const now = Date.now() + const twoHours = 2 * 60 * 60 * 1000 + const twoDays = 2 * 24 * 60 * 60 * 1000 + // main has NO scoped window, but 5h resetsAt in 2h. A fallback carries + // a Fable window that resets in 2 days. The 5h reset on the fallback + // must be ignored too — the branch is scoped REPLACEMENT, per-quota. + const mainQuota: OAuthQuotaSnapshot = {} + const fallbacks = [ + { + quota: { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + resetsAt: new Date(now + twoHours).toISOString(), + checkedAt: now, + }, + scoped: [ + scopeWindow( + 0, + { name: 'Claude Fable 5', id: 'claude-fable-5' }, + { resetsAt: new Date(now + twoDays).toISOString() }, + ), + ], + }, + }, + ] + const seconds = killswitchRetryAfterSeconds( + mainQuota, + fallbacks, + now, + 'claude-fable-5', + ) + const expected = Math.ceil(twoDays / 1000) + 60 + expect(seconds).toBeGreaterThanOrEqual(expected - 2) + expect(seconds).toBeLessThanOrEqual(expected + 2) + }) + + test('scoped branch: no matched scoped window in any quota → 300 fallback (FINDING 1 empty)', () => { + const now = Date.now() + // main has a 5h reset but NO scoped window for the requested model. + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 95, + remainingPercent: 5, + resetsAt: new Date(now + 7_200_000).toISOString(), + checkedAt: now, + }, + } + const seconds = killswitchRetryAfterSeconds( + mainQuota, + [], + now, + 'claude-fable-5', + ) + expect(seconds).toBe(300) + }) }) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index aab0016e..878ad7ef 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -181,6 +181,12 @@ export function formatKillswitchBlockMessage(input: { * actually at or below the per-account scoped threshold. A Fable request * with a healthy Fable window is therefore correctly classified as * account-level (the 5h/7d breach killed the account, not the Fable quota). + * + * Priority: account-level 5h/7d always wins. `killswitchPassesPolicy` called + * WITHOUT a modelId evaluates only 5h/7d; if it returns false, 5h/7d drove + * the block, so this is account-level regardless of the scoped window's + * state. Only when 5h/7d pass AND the matched scoped window is at/below the + * scoped threshold do we call it scoped-driven. */ export function resolveScopedDrivenBlock(input: { mainQuota: OAuthQuotaSnapshot | undefined @@ -190,6 +196,10 @@ export function resolveScopedDrivenBlock(input: { | { isScopedDriven: true; modelName: string; modelId: string } | { isScopedDriven: false } { if (!input.requestModelId) return { isScopedDriven: false } + if (!killswitchPassesPolicy(input.mainQuota, input.storage)) { + // 5h/7d already killed the account — account-level, not scoped-driven. + return { isScopedDriven: false } + } const matchedWindow = getScopedQuotaWindowForModel( input.mainQuota, input.requestModelId, diff --git a/packages/opencode/src/tests/killswitch.test.ts b/packages/opencode/src/tests/killswitch.test.ts index ccdd6ff3..9a99dbde 100644 --- a/packages/opencode/src/tests/killswitch.test.ts +++ b/packages/opencode/src/tests/killswitch.test.ts @@ -781,4 +781,104 @@ describe('resolveScopedDrivenBlock', () => { }) expect(result.isScopedDriven).toBe(false) }) + + // Regression for FINDING 2: when BOTH 5h/7d AND the Fable window are + // exhausted, the real block reason is account-level (5h/7d), yet the + // old code only checked the scoped window and reported scoped-driven. + // That misleads the operator (saying "Fable weekly limit" when the + // account is dead for ALL models) and misroutes the retry-after to + // the weekly window. Account-level 5h/7d must take priority. + test('BOTH 5h/7d AND Fable exhausted → NOT scoped-driven (FINDING 2)', () => { + // 5h threshold 10, 5h remaining 4 → 4 < 10, BLOCKED + // 7d threshold 10, 7d remaining 4 → 4 < 10, BLOCKED + // scoped threshold 0, Fable remaining 0 → 0 <= 0, BLOCKED + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 10, seven_day: 10, scoped: 0 }, + } + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 96, + remainingPercent: 4, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 96, + remainingPercent: 4, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('BOTH 5h/7d AND Fable exhausted — Sonnet request → NOT scoped-driven (FINDING 2 Sonnet variant)', () => { + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 10, seven_day: 10, scoped: 0 }, + } + const mainQuota: OAuthQuotaSnapshot = { + five_hour: { + usedPercent: 96, + remainingPercent: 4, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 96, + remainingPercent: 4, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], + } + // Sonnet has no matching scoped window, so the 5h/7d guard short- + // circuits. Important: result is still NOT scoped-driven. + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-sonnet-5', + storage, + }) + expect(result.isScopedDriven).toBe(false) + }) + + test('HEALTHY 5h/7d + EXHAUSTED Fable → still IS scoped-driven (FINDING 2 regression lock)', () => { + // This is the MUST-2 proof test, restated as a regression lock for + // FINDING 2: the 5h/7d guard must not over-trigger and turn a + // genuine scoped-driven block into account-level. + const storage = baseStorage() + storage.killswitch = { + enabled: true, + main: { five_hour: 5, seven_day: 10, scoped: 0 }, + } + const mainQuota: OAuthQuotaSnapshot = { + // 5h/7d HEALTHY — they must pass the no-modelId policy check. + five_hour: { + usedPercent: 10, + remainingPercent: 90, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 20, + remainingPercent: 80, + checkedAt: Date.now(), + }, + scoped: [fableWindow(0)], + } + const result = resolveScopedDrivenBlock({ + mainQuota, + requestModelId: 'claude-fable-5', + storage, + }) + expect(result.isScopedDriven).toBe(true) + if (result.isScopedDriven) { + expect(result.modelName).toBe('Claude Fable 5') + expect(result.modelId).toBe('claude-fable-5') + } + }) }) From 660e47dbc49699c957ed82ca36d3b8be8a9d380e Mon Sep 17 00:00:00 2001 From: iceteaSA <171169159+iceteaSA@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:31:26 +0200 Subject: [PATCH 4/4] test(killswitch): make MUST-2 5h/7d premise actually block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The '5h/7d-driven block + healthy Fable' test set 5h/7d remaining to exactly 5/10 against thresholds of 5/10, but 5h/7d use strict '<', so 5<5 and 10<10 never fired — the test passed only because the Fable window was healthy, never exercising the account-level priority guard (cubic P2). Lower remaining to 4/9 so the killswitch genuinely fires and the test traverses the guard branch. The both-exhausted guard lock is already covered by the existing FINDING 2 test. --- packages/opencode/src/tests/killswitch.test.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/tests/killswitch.test.ts b/packages/opencode/src/tests/killswitch.test.ts index 9a99dbde..1b6ebecc 100644 --- a/packages/opencode/src/tests/killswitch.test.ts +++ b/packages/opencode/src/tests/killswitch.test.ts @@ -608,21 +608,20 @@ describe('resolveScopedDrivenBlock', () => { } test('5h/7d-driven block + HEALTHY Fable window → NOT scoped-driven (MUST-2)', () => { - // Main quota is exhausted on 5h/7d (killswitch will fire), but the - // Fable window is at 100% remaining — well above the scoped threshold. - // The current (buggy) code names Fable; the fix must produce a generic - // account-level message because the block is 5h/7d-driven, not - // scoped-driven. + // Main quota is exhausted on 5h/7d (remaining below the strict-`<` + // thresholds of 5/10, so the killswitch genuinely fires), but the Fable + // window is at 100% remaining — well above the scoped threshold. The block + // must be classified account-level (generic message), not scoped-driven. const storage = killswitchStorage(0) const mainQuota: OAuthQuotaSnapshot = { five_hour: { - usedPercent: 95, - remainingPercent: 5, + usedPercent: 96, + remainingPercent: 4, checkedAt: Date.now(), }, seven_day: { - usedPercent: 90, - remainingPercent: 10, + usedPercent: 91, + remainingPercent: 9, checkedAt: Date.now(), }, scoped: [fableWindow(100)], // perfectly healthy