-
Notifications
You must be signed in to change notification settings - Fork 2
feat(web): per-backend spend on the fleet rail #334
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| import { describe, it, expect } from "vitest"; | ||
|
|
||
| import { costShare, fleetEconomics, UNKNOWN_PROVIDER } from "./fleet-economics"; | ||
| import type { SessionInfo, SessionUsage } from "../protocol/types"; | ||
|
|
||
| const usage = (over: Partial<SessionUsage> = {}): SessionUsage => | ||
| ({ | ||
| inputTokens: 0, | ||
| outputTokens: 0, | ||
| cacheReadTokens: 0, | ||
| cacheCreationTokens: 0, | ||
| totalCostUsd: 0, | ||
| numTurns: 0, | ||
| durationMs: 0, | ||
| ...over, | ||
| }) as SessionUsage; | ||
|
|
||
| const session = (id: string, over: Partial<SessionInfo> = {}): SessionInfo => | ||
| ({ id, name: id, status: "idle", ...over }) as SessionInfo; | ||
|
|
||
| describe("fleetEconomics", () => { | ||
| it("rolls spend up per backend and totals it", () => { | ||
| const econ = fleetEconomics([ | ||
| session("a", { providerId: "claude", usage: usage({ totalCostUsd: 1, inputTokens: 10 }) }), | ||
| session("b", { providerId: "claude", usage: usage({ totalCostUsd: 2, outputTokens: 5 }) }), | ||
| session("c", { providerId: "qwen", usage: usage({ totalCostUsd: 0.5 }) }), | ||
| ]); | ||
| expect(econ.byBackend.map((b) => b.providerId)).toEqual(["claude", "qwen"]); | ||
| expect(econ.byBackend[0]!.costUsd).toBe(3); | ||
| expect(econ.byBackend[0]!.sessions).toBe(2); | ||
| expect(econ.byBackend[0]!.inputTokens).toBe(10); | ||
| expect(econ.byBackend[0]!.outputTokens).toBe(5); | ||
| expect(econ.totalCostUsd).toBe(3.5); | ||
| expect(econ.sessions).toBe(3); | ||
| }); | ||
|
|
||
| it("orders by cost descending — the view answers 'what is burning money'", () => { | ||
| const econ = fleetEconomics([ | ||
| session("cheap", { providerId: "qwen", usage: usage({ totalCostUsd: 0.1 }) }), | ||
| session("dear", { providerId: "claude", usage: usage({ totalCostUsd: 9 }) }), | ||
| ]); | ||
| expect(econ.byBackend[0]!.providerId).toBe("claude"); | ||
| }); | ||
|
|
||
| it("breaks cost ties by name so the list does not reshuffle between renders", () => { | ||
| const econ = fleetEconomics([ | ||
| session("z", { providerId: "zeta", usage: usage({ totalCostUsd: 1 }) }), | ||
| session("a", { providerId: "alpha", usage: usage({ totalCostUsd: 1 }) }), | ||
| ]); | ||
| expect(econ.byBackend.map((b) => b.providerId)).toEqual(["alpha", "zeta"]); | ||
| }); | ||
|
|
||
| it("counts a session with no usage without inventing spend for it", () => { | ||
| // Ten idle sessions must read as ten sessions at $0, not as an empty fleet. | ||
| const econ = fleetEconomics([ | ||
| session("a", { providerId: "claude" }), | ||
| session("b", { providerId: "claude" }), | ||
| ]); | ||
| expect(econ.sessions).toBe(2); | ||
| expect(econ.byBackend[0]!.sessions).toBe(2); | ||
| expect(econ.totalCostUsd).toBe(0); | ||
| }); | ||
|
|
||
| it("reports a missing providerId as `unknown` rather than folding it into the default", () => { | ||
| // Attributing spend to a provider that may not have incurred it is worse | ||
| // than admitting the gap — the whole point is comparing backends. | ||
| const econ = fleetEconomics([session("a", { usage: usage({ totalCostUsd: 5 }) })]); | ||
| expect(econ.byBackend[0]!.providerId).toBe(UNKNOWN_PROVIDER); | ||
| expect(econ.byBackend[0]!.costUsd).toBe(5); | ||
| }); | ||
|
|
||
| it("counts only genuinely in-flight sessions as active", () => { | ||
| const econ = fleetEconomics([ | ||
| session("a", { providerId: "claude", status: "thinking" }), | ||
| session("b", { providerId: "claude", status: "tool_running" }), | ||
| session("c", { providerId: "claude", status: "waiting_approval" }), | ||
| session("d", { providerId: "claude", status: "idle" }), | ||
| session("e", { providerId: "claude", status: "error" }), | ||
| ]); | ||
| // waiting_approval is stopped, not working — it belongs in the attention | ||
| // queue, not the concurrency count. | ||
| expect(econ.active).toBe(2); | ||
| expect(econ.sessions).toBe(5); | ||
| }); | ||
|
|
||
| it("returns an empty rollup for an empty fleet", () => { | ||
| const econ = fleetEconomics([]); | ||
| expect(econ.byBackend).toEqual([]); | ||
| expect(econ.totalCostUsd).toBe(0); | ||
| expect(econ.active).toBe(0); | ||
| }); | ||
| }); | ||
|
|
||
| describe("costShare", () => { | ||
| it("reports each backend's share of total spend", () => { | ||
| const econ = fleetEconomics([ | ||
| session("a", { providerId: "claude", usage: usage({ totalCostUsd: 3 }) }), | ||
| session("b", { providerId: "qwen", usage: usage({ totalCostUsd: 1 }) }), | ||
| ]); | ||
| expect(costShare(econ, econ.byBackend[0]!)).toBeCloseTo(0.75); | ||
| expect(costShare(econ, econ.byBackend[1]!)).toBeCloseTo(0.25); | ||
| }); | ||
|
|
||
| it("is zero when nothing has been spent — never an even split", () => { | ||
| // Dividing by a zero total to show four backends at 25% would invent a | ||
| // fact from an absence. | ||
| const econ = fleetEconomics([ | ||
| session("a", { providerId: "claude" }), | ||
| session("b", { providerId: "qwen" }), | ||
| ]); | ||
| expect(costShare(econ, econ.byBackend[0]!)).toBe(0); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,122 @@ | ||||||||||||||||||
| /** | ||||||||||||||||||
| * Fleet economics — cost and tokens rolled up per backend | ||||||||||||||||||
| * (conductor-frontends-design §9, §7). | ||||||||||||||||||
| * | ||||||||||||||||||
| * §9 asks for the zoom ABOVE a single session: total spend across the fleet, | ||||||||||||||||||
| * broken down per backend, so a Claude row and a Gemini row are comparable at a | ||||||||||||||||||
| * glance. §7 calls that out as a metaharness differentiator rather than a | ||||||||||||||||||
| * nicety — no single-vendor tool needs a per-backend breakdown, because it only | ||||||||||||||||||
| * ever has one backend. | ||||||||||||||||||
| * | ||||||||||||||||||
| * Pure functions: the rollup is the decision worth testing, and it needs no | ||||||||||||||||||
| * reactive root. | ||||||||||||||||||
| */ | ||||||||||||||||||
|
|
||||||||||||||||||
| import type { SessionInfo } from "../protocol/types"; | ||||||||||||||||||
|
|
||||||||||||||||||
| export interface BackendSpend { | ||||||||||||||||||
| /** Provider id as the daemon reports it (`claude`, `gemini-cli`, `qwen`, …). */ | ||||||||||||||||||
| providerId: string; | ||||||||||||||||||
| sessions: number; | ||||||||||||||||||
| /** Sessions currently running a turn or a tool. */ | ||||||||||||||||||
| active: number; | ||||||||||||||||||
| inputTokens: number; | ||||||||||||||||||
| outputTokens: number; | ||||||||||||||||||
| costUsd: number; | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| export interface FleetEconomics { | ||||||||||||||||||
| byBackend: BackendSpend[]; | ||||||||||||||||||
| totalCostUsd: number; | ||||||||||||||||||
| totalInputTokens: number; | ||||||||||||||||||
| totalOutputTokens: number; | ||||||||||||||||||
| /** Sessions counted in this rollup. */ | ||||||||||||||||||
| sessions: number; | ||||||||||||||||||
| /** How many are working right now — §9's concurrent-agent count. */ | ||||||||||||||||||
| active: number; | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| /** Session statuses that mean a turn is actually in flight. */ | ||||||||||||||||||
| const BUSY = new Set(["thinking", "tool_running"]); | ||||||||||||||||||
|
|
||||||||||||||||||
| /** | ||||||||||||||||||
| * Bucket for a session whose backend the daemon did not report. | ||||||||||||||||||
| * | ||||||||||||||||||
| * Exported so consumers compare against this rather than re-typing the string: | ||||||||||||||||||
| * it is a sentinel that reaches the UI, and a caller that wants to style or | ||||||||||||||||||
| * filter it should not have to know the literal. | ||||||||||||||||||
| */ | ||||||||||||||||||
| export const UNKNOWN_PROVIDER = "unknown" as const; | ||||||||||||||||||
|
|
||||||||||||||||||
| /** | ||||||||||||||||||
| * Backend label for a session. | ||||||||||||||||||
| * | ||||||||||||||||||
| * A session with no `providerId` is reported as `unknown` rather than silently | ||||||||||||||||||
| * folded into the default backend. Attributing spend to a provider that may not | ||||||||||||||||||
| * have incurred it is worse than admitting the gap — the entire point of this | ||||||||||||||||||
| * view is comparing backends against each other. | ||||||||||||||||||
| */ | ||||||||||||||||||
| function backendOf(s: SessionInfo): string { | ||||||||||||||||||
| return s.providerId ?? UNKNOWN_PROVIDER; | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| /** | ||||||||||||||||||
| * Roll `sessions` up per backend. | ||||||||||||||||||
| * | ||||||||||||||||||
|
saucam marked this conversation as resolved.
|
||||||||||||||||||
| * Sessions with no usage still COUNT (they exist and occupy a slot) but | ||||||||||||||||||
| * contribute zero spend, so the session count and the cost stay independently | ||||||||||||||||||
| * true. A fleet of ten idle sessions reads as ten sessions at $0, not as an | ||||||||||||||||||
| * empty fleet. | ||||||||||||||||||
| * | ||||||||||||||||||
| * Ordered by cost descending — the question this view answers is "what is | ||||||||||||||||||
| * burning money", so the answer is on the first row. Backends that tie fall | ||||||||||||||||||
| * back to name order so the list does not reshuffle between renders. | ||||||||||||||||||
| */ | ||||||||||||||||||
| export function fleetEconomics(sessions: readonly SessionInfo[]): FleetEconomics { | ||||||||||||||||||
| const byId = new Map<string, BackendSpend>(); | ||||||||||||||||||
|
|
||||||||||||||||||
| for (const s of sessions) { | ||||||||||||||||||
| const id = backendOf(s); | ||||||||||||||||||
| const row = byId.get(id) ?? { | ||||||||||||||||||
| providerId: id, | ||||||||||||||||||
| sessions: 0, | ||||||||||||||||||
| active: 0, | ||||||||||||||||||
| inputTokens: 0, | ||||||||||||||||||
| outputTokens: 0, | ||||||||||||||||||
| costUsd: 0, | ||||||||||||||||||
| }; | ||||||||||||||||||
| row.sessions += 1; | ||||||||||||||||||
| if (BUSY.has(s.status)) row.active += 1; | ||||||||||||||||||
| const u = s.usage; | ||||||||||||||||||
| if (u) { | ||||||||||||||||||
| row.inputTokens += u.inputTokens; | ||||||||||||||||||
| row.outputTokens += u.outputTokens; | ||||||||||||||||||
| row.costUsd += u.totalCostUsd; | ||||||||||||||||||
| } | ||||||||||||||||||
| byId.set(id, row); | ||||||||||||||||||
| } | ||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider using Intl.NumberFormat for share percentages While the current implementation returns a 0-1 ratio which is correct for the data layer, consider adding a helper for UI formatting that uses Suggested fix:
Suggested change
|
||||||||||||||||||
|
|
||||||||||||||||||
| const byBackend = [...byId.values()].sort( | ||||||||||||||||||
| (a, b) => b.costUsd - a.costUsd || (a.providerId < b.providerId ? -1 : 1), | ||||||||||||||||||
| ); | ||||||||||||||||||
|
|
||||||||||||||||||
| return { | ||||||||||||||||||
| byBackend, | ||||||||||||||||||
| totalCostUsd: byBackend.reduce((n, b) => n + b.costUsd, 0), | ||||||||||||||||||
| totalInputTokens: byBackend.reduce((n, b) => n + b.inputTokens, 0), | ||||||||||||||||||
| totalOutputTokens: byBackend.reduce((n, b) => n + b.outputTokens, 0), | ||||||||||||||||||
| sessions: byBackend.reduce((n, b) => n + b.sessions, 0), | ||||||||||||||||||
| active: byBackend.reduce((n, b) => n + b.active, 0), | ||||||||||||||||||
| }; | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| /** | ||||||||||||||||||
| * Each backend's share of total spend, 0–1. | ||||||||||||||||||
| * | ||||||||||||||||||
| * Zero when nothing has been spent yet — NOT an even split. A fresh fleet has | ||||||||||||||||||
| * no shares to report, and dividing by a zero total to show four backends at | ||||||||||||||||||
| * 25% would be inventing a fact from an absence. | ||||||||||||||||||
| */ | ||||||||||||||||||
| export function costShare(econ: FleetEconomics, backend: BackendSpend): number { | ||||||||||||||||||
| return econ.totalCostUsd > 0 ? backend.costUsd / econ.totalCostUsd : 0; | ||||||||||||||||||
| } | ||||||||||||||||||
Uh oh!
There was an error while loading. Please reload this page.