Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 77 additions & 1 deletion src/cli/diagnose.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@ const healthy = {
lastStartedAtMs: NOW_MS - 30_000,
lastCompletedAtMs: NOW_MS - 29_000,
},
fleetConnect: {
state: 'connected',
attempts: 1,
lastAttemptAtMs: NOW_MS - 32_000,
lastDialedAtMs: NOW_MS - 31_000,
firstEventAtMs: NOW_MS - 30_500,
lastConnectedAtMs: NOW_MS - 30_000,
},
eventListener: { state: 'subscribed' },
},
}
Expand All @@ -82,6 +90,40 @@ describe('factory diagnose --deployed (#295)', () => {
expect(seen).toEqual([`${BASE}/healthz`])
expect(out.text()).toContain('dispatching')
expect(out.text()).toContain('readinessReconcile')
expect(out.text()).toContain('fleetConnect')
expect(out.text()).toContain('connected')
expect(out.text()).toContain(`lastAttemptAt : ${new Date(NOW_MS - 32_000).toISOString()}`)
expect(out.text()).toContain(`lastConnectedAt : ${new Date(NOW_MS - 30_000).toISOString()}`)
})

it('renders dialed as unconfirmed until a stream event is observed', async () => {
const out = buffer()
const code = await runFleetCli(['diagnose', '--deployed', BASE], {
stdout: out,
stderr: buffer(),
env: HERMETIC_ENV,
diagnoseFetch: stubFetch({
healthz: {
status: 200,
body: {
...healthy,
health: {
...healthy.health,
fleetConnect: {
...healthy.health.fleetConnect,
state: 'dialed',
firstEventAtMs: undefined,
lastConnectedAtMs: undefined,
},
},
},
},
}),
})

expect(code).toBe(0)
expect(out.text()).toContain('unconfirmed — the SDK accepted connect()')
expect(out.text()).toContain('healthy silent workspace may remain dialed')
})

// The 2026-08-19/20 outage: eight consecutive failures behind `ok: true`.
Expand Down Expand Up @@ -316,6 +358,41 @@ describe('factory diagnose --deployed (#295)', () => {
expect(report.evidence?.lastError).toContain('dispatch lifecycle is already terminal')
})

it('reads and renders fleet socket errors from authenticated evidence', async () => {
const out = buffer()
const code = await runFleetCli(['diagnose', '--deployed', BASE, '--token', 'op-token'], {
stdout: out,
stderr: buffer(),
env: HERMETIC_ENV,
diagnoseFetch: stubFetch({
healthz: {
status: 200,
body: {
...healthy,
health: {
...healthy.health,
fleetConnect: {
state: 'failed',
attempts: 2,
lastFailureAtMs: NOW_MS - 1_000,
},
},
},
},
evidence: {
status: 200,
body: {
phase: 'running',
fleetConnect: { lastError: 'FactoryAgentRegistrationError (AGENT_EXISTS)' },
},
},
}),
})

expect(code).toBe(0)
expect(out.text()).toContain('fleetConnect error : FactoryAgentRegistrationError (AGENT_EXISTS)')
})

it('still diagnoses when the evidence token is rejected', async () => {
const out = buffer()
const code = await runFleetCli(['diagnose', '--deployed', BASE, '--token', 'stale-token', '--json'], {
Expand Down Expand Up @@ -765,4 +842,3 @@ describe('factory diagnose --deployed (#295)', () => {
}
})
})

22 changes: 22 additions & 0 deletions src/cli/diagnose.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export interface DeployedEvidenceSummary {
reason?: string
phase?: string
lastError?: string
fleetConnectLastError?: string
consecutiveFailures?: number
}

Expand Down Expand Up @@ -445,11 +446,15 @@ async function readEvidence(
}
const body = asRecord(evidence.body)
const readiness = asRecord(body.readinessReconcile)
const fleetConnect = asRecord(body.fleetConnect)
return {
fetched: true,
httpStatus: evidence.status,
...(asText(body.phase) ? { phase: asText(body.phase) } : {}),
...(asText(readiness.lastError) ? { lastError: asText(readiness.lastError) } : {}),
...(asText(fleetConnect.lastError)
? { fleetConnectLastError: asText(fleetConnect.lastError) }
: {}),
...(asCount(readiness.consecutiveFailures) !== undefined
? { consecutiveFailures: asCount(readiness.consecutiveFailures) }
: {}),
Expand Down Expand Up @@ -523,6 +528,20 @@ export function renderDeployedDiagnosis(diagnosis: DeployedFactoryDiagnosis): st
)
}
}
const fleetConnect = health.fleetConnect
Comment thread
khaliqgant marked this conversation as resolved.
if (fleetConnect) {
lines.push(' fleetConnect:')
lines.push(` state : ${fleetConnect.state}`)
if (fleetConnect.state === 'dialed') {
lines.push(' confirmation : unconfirmed — the SDK accepted connect(), but no stream event has been observed; a healthy silent workspace may remain dialed')
}
lines.push(` attempts : ${fleetConnect.attempts ?? '—'}`)
lines.push(` lastAttemptAt : ${formatInstant(fleetConnect.lastAttemptAtMs)}`)
lines.push(` lastDialedAt : ${formatInstant(fleetConnect.lastDialedAtMs)}`)
lines.push(` firstEventAt : ${formatInstant(fleetConnect.firstEventAtMs)}`)
lines.push(` lastConnectedAt : ${formatInstant(fleetConnect.lastConnectedAtMs)}`)
lines.push(` lastFailureAt : ${formatInstant(fleetConnect.lastFailureAtMs)}`)
Comment thread
khaliqgant marked this conversation as resolved.
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
lines.push(` eventListener : ${health.eventListener?.state ?? 'unknown'}`)
} else if (diagnosis.unreadable) {
lines.push(' health block : none — this response carried no Factory health')
Expand All @@ -547,6 +566,9 @@ export function renderDeployedDiagnosis(diagnosis: DeployedFactoryDiagnosis): st
` evidence : ${evidence.fetched ? `read (HTTP ${evidence.httpStatus ?? 200})` : `not read — ${evidence.reason ?? 'unavailable'}`}`,
)
if (evidence.lastError) lines.push(` lastError : ${evidence.lastError}`)
if (evidence.fleetConnectLastError) {
lines.push(` fleetConnect error : ${evidence.fleetConnectLastError}`)
}
}

lines.push('')
Expand Down
25 changes: 24 additions & 1 deletion src/cli/fleet.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3238,6 +3238,13 @@ describe('fleet CLI runtime', () => {
retryAtMs: now + 59_500,
lastError: 'TimeoutError (FACTORY_FLEET_CONTROL_TIMEOUT)',
},
fleetConnect: {
state: 'failed',
attempts: 2,
lastAttemptAtMs: now - 2_000,
lastFailureAtMs: now - 400,
lastError: 'RelayEventStreamDisconnected',
},
}))
const output = buffer()
const factory = {
Expand All @@ -3257,6 +3264,11 @@ describe('fleet CLI runtime', () => {
failureThreshold: 2,
resetTimeoutMs: 60_000,
},
fleetConnect: {
state: 'connected' as const,
attempts: 1,
firstEventAtMs: now - 5_000,
},
})),
} as unknown as Factory

Expand All @@ -3281,6 +3293,11 @@ describe('fleet CLI runtime', () => {
consecutiveFailures: 2,
lastError: 'TimeoutError (FACTORY_FLEET_CONTROL_TIMEOUT)',
},
fleetConnect: {
state: 'failed',
attempts: 2,
lastError: 'RelayEventStreamDisconnected',
},
})
} finally {
await rm(root, { recursive: true, force: true })
Expand Down Expand Up @@ -3315,6 +3332,10 @@ describe('fleet CLI runtime', () => {
failureThreshold: 2,
resetTimeoutMs: 60_000,
},
fleetConnect: {
state: 'connected' as const,
attempts: 1,
},
})),
} as unknown as Factory

Expand All @@ -3327,7 +3348,9 @@ describe('fleet CLI runtime', () => {
})

expect(code).toBe(0)
expect(JSON.parse(output.text())).not.toHaveProperty('fleetControlPlane')
const status = JSON.parse(output.text())
expect(status).not.toHaveProperty('fleetControlPlane')
expect(status).not.toHaveProperty('fleetConnect')
} finally {
await rm(root, { recursive: true, force: true })
}
Expand Down
9 changes: 8 additions & 1 deletion src/cli/fleet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1264,9 +1264,11 @@ async function factoryStatusWithMountHealth(
heartbeatStaleMs: number,
versionInfo?: FactoryVersionInfo,
stateStoreStatus?: { backend: string },
): Promise<Omit<ReturnType<Factory['status']>, 'fleetControlPlane'> & {
): Promise<Omit<ReturnType<Factory['status']>, 'fleetControlPlane' | 'fleetConnect'> & {
/** Undefined when a live older daemon predates fleet control-plane reporting. */
fleetControlPlane?: ReturnType<Factory['status']>['fleetControlPlane']
/** Undefined when a live older daemon predates fleet socket reporting. */
fleetConnect?: ReturnType<Factory['status']>['fleetConnect']
stateStore?: { backend: string }
version?: string
installedAt?: string
Expand Down Expand Up @@ -1307,6 +1309,9 @@ async function factoryStatusWithMountHealth(
const fleetControlPlane = liveness.ok
? heartbeat?.fleetControlPlane
: observableStatus.fleetControlPlane
const fleetConnect = liveness.ok
? heartbeat?.fleetConnect
: observableStatus.fleetConnect
// Same rule as readinessReconcile (#303): a live daemon owns the batch, and
// a fresh local Factory instance holds no lifecycles. Falling back to that
// instance when a live daemon predates the field would publish its empty
Expand All @@ -1333,6 +1338,7 @@ async function factoryStatusWithMountHealth(
eventListener,
readinessReconcile,
fleetControlPlane,
fleetConnect,
...(dispatchCapacity ? { dispatchCapacity } : {}),
}
return {
Expand All @@ -1343,6 +1349,7 @@ async function factoryStatusWithMountHealth(
eventListener,
readinessReconcile,
fleetControlPlane,
fleetConnect,
...(dispatchCapacity ? { dispatchCapacity } : {}),
localMountDegraded: health.degraded,
...(health.reason ? { localMountDegradedReason: health.reason } : {}),
Expand Down
10 changes: 9 additions & 1 deletion src/fleet/control-plane-circuit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,15 @@ export function isFleetControlPlaneFailure(error: unknown): boolean {
}

/** Redacts arbitrary transport text before circuit state becomes observable. */
function describeControlPlaneError(error: unknown): string {
/**
* Reduce any cause to `Name (CODE)`.
*
* Exported so the fleet CONNECT status uses the identical redaction as the
* control-plane circuit: both values are published through `factory status` and
* the authenticated `/evidence`, and neither may carry a transport message,
* URL or credential.
*/
export function describeControlPlaneError(error: unknown): string {
if (!(error instanceof Error)) return 'unknown control-plane failure'
const code = (error as Error & { code?: unknown }).code
const safeCode = typeof code === 'string' && /^[A-Z0-9_]{1,80}$/u.test(code) ? ` (${code})` : ''
Expand Down
Loading