diff --git a/packages/cli/src/__tests__/lifecycle-telemetry.test.ts b/packages/cli/src/__tests__/lifecycle-telemetry.test.ts new file mode 100644 index 000000000..28dbae8d3 --- /dev/null +++ b/packages/cli/src/__tests__/lifecycle-telemetry.test.ts @@ -0,0 +1,231 @@ +/** + * lifecycle-telemetry.test.ts — Verifies trackSpawnConnected / + * trackSpawnDeleted emit the right PostHog events and persist the + * connect_count + last_connected_at metadata. + */ + +import type { SpawnRecord } from "../history"; + +import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"; +import { isNumber, isString } from "@openrouter/spawn-shared"; +// Import the real modules so we can spy on their exports without +// polluting the global module registry (mock.module contaminates +// other test files when running under --coverage). +import * as historyMod from "../history"; +import { trackSpawnConnected, trackSpawnDeleted } from "../shared/lifecycle-telemetry"; +import * as telemetryMod from "../shared/telemetry"; + +const savedMetadataCalls: Array<{ + entries: Record; + spawnId?: string; +}> = []; + +const capturedEvents: Array<{ + event: string; + properties: Record; +}> = []; + +// ── Helpers ───────────────────────────────────────────────────────────── + +function makeRecord(overrides: Partial = {}): SpawnRecord { + return { + id: "spawn-abc123", + agent: "claude", + cloud: "digitalocean", + timestamp: "2026-04-13T12:00:00.000Z", + connection: { + ip: "10.0.0.1", + user: "root", + cloud: "digitalocean", + metadata: {}, + }, + ...overrides, + }; +} + +// ── Tests ─────────────────────────────────────────────────────────────── + +describe("lifecycle-telemetry", () => { + let saveMetadataSpy: ReturnType; + let captureEventSpy: ReturnType; + + beforeEach(() => { + savedMetadataCalls.length = 0; + capturedEvents.length = 0; + + saveMetadataSpy = spyOn(historyMod, "saveMetadata").mockImplementation( + (entries: Record, spawnId?: string) => { + savedMetadataCalls.push({ + entries, + spawnId, + }); + }, + ); + captureEventSpy = spyOn(telemetryMod, "captureEvent").mockImplementation( + (event: string, properties: Record) => { + capturedEvents.push({ + event, + properties, + }); + }, + ); + }); + + afterEach(() => { + saveMetadataSpy.mockRestore(); + captureEventSpy.mockRestore(); + savedMetadataCalls.length = 0; + capturedEvents.length = 0; + }); + + describe("trackSpawnConnected", () => { + it("starts the connect count at 1 when metadata is empty", () => { + const record = makeRecord(); + const count = trackSpawnConnected(record); + + expect(count).toBe(1); + expect(savedMetadataCalls).toHaveLength(1); + expect(savedMetadataCalls[0].entries.connect_count).toBe("1"); + expect(savedMetadataCalls[0].spawnId).toBe("spawn-abc123"); + }); + + it("increments an existing connect count", () => { + const record = makeRecord({ + connection: { + ip: "10.0.0.1", + user: "root", + cloud: "digitalocean", + metadata: { + connect_count: "4", + }, + }, + }); + const count = trackSpawnConnected(record); + + expect(count).toBe(5); + expect(savedMetadataCalls[0].entries.connect_count).toBe("5"); + }); + + it("tolerates malformed connect_count by resetting to 1", () => { + const record = makeRecord({ + connection: { + ip: "10.0.0.1", + user: "root", + cloud: "digitalocean", + metadata: { + connect_count: "not-a-number", + }, + }, + }); + const count = trackSpawnConnected(record); + + // Malformed parses to 0, +1 = 1. Never throws. + expect(count).toBe(1); + }); + + it("updates last_connected_at to an ISO timestamp", () => { + trackSpawnConnected(makeRecord()); + + const ts = savedMetadataCalls[0].entries.last_connected_at; + expect(ts).toBeDefined(); + // ISO 8601 format YYYY-MM-DDTHH:MM:SS.sssZ + expect(ts).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + }); + + it("emits spawn_connected event with spawn metadata", () => { + const record = makeRecord(); + trackSpawnConnected(record); + + expect(capturedEvents).toHaveLength(1); + expect(capturedEvents[0].event).toBe("spawn_connected"); + expect(capturedEvents[0].properties.spawn_id).toBe("spawn-abc123"); + expect(capturedEvents[0].properties.agent).toBe("claude"); + expect(capturedEvents[0].properties.cloud).toBe("digitalocean"); + expect(capturedEvents[0].properties.connect_count).toBe(1); + }); + + it("is a no-op for records without an id or connection", () => { + const noId = makeRecord({ + id: undefined, + }); + expect(trackSpawnConnected(noId)).toBe(0); + expect(savedMetadataCalls).toHaveLength(0); + expect(capturedEvents).toHaveLength(0); + + const noConn = makeRecord({ + connection: undefined, + }); + expect(trackSpawnConnected(noConn)).toBe(0); + expect(savedMetadataCalls).toHaveLength(0); + expect(capturedEvents).toHaveLength(0); + }); + }); + + describe("trackSpawnDeleted", () => { + it("emits spawn_deleted with lifetime_hours computed from timestamp", () => { + // Record created 3 hours ago. With `new Date()` in the helper we can't + // easily mock the clock here, so we assert on a loose-but-correct + // range (3h +/- a minute). + const threeHoursAgo = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString(); + const record = makeRecord({ + timestamp: threeHoursAgo, + }); + + trackSpawnDeleted(record); + + expect(capturedEvents).toHaveLength(1); + expect(capturedEvents[0].event).toBe("spawn_deleted"); + const rawLifetime = capturedEvents[0].properties.lifetime_hours; + const lifetime = isNumber(rawLifetime) ? rawLifetime : 0; + expect(lifetime).toBeGreaterThanOrEqual(2.98); + expect(lifetime).toBeLessThanOrEqual(3.02); + }); + + it("reports the final connect count", () => { + const record = makeRecord({ + connection: { + ip: "10.0.0.1", + user: "root", + cloud: "digitalocean", + metadata: { + connect_count: "7", + }, + }, + }); + trackSpawnDeleted(record); + + expect(capturedEvents[0].properties.connect_count).toBe(7); + }); + + it("clamps negative lifetimes to 0 (corrupt clock / timestamp)", () => { + const futureTimestamp = new Date(Date.now() + 60 * 60 * 1000).toISOString(); + const record = makeRecord({ + timestamp: futureTimestamp, + }); + + trackSpawnDeleted(record); + + expect(capturedEvents[0].properties.lifetime_hours).toBe(0); + }); + + it("is a no-op for records without an id", () => { + trackSpawnDeleted( + makeRecord({ + id: undefined, + }), + ); + expect(capturedEvents).toHaveLength(0); + }); + + it("includes spawn_id, agent, cloud, and date on every event", () => { + trackSpawnDeleted(makeRecord()); + + const props = capturedEvents[0].properties; + expect(props.spawn_id).toBe("spawn-abc123"); + expect(props.agent).toBe("claude"); + expect(props.cloud).toBe("digitalocean"); + expect(isString(props.date)).toBe(true); + expect(props.date).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + }); + }); +}); diff --git a/packages/cli/src/__tests__/telemetry.test.ts b/packages/cli/src/__tests__/telemetry.test.ts index 3d8808718..9a6f98664 100644 --- a/packages/cli/src/__tests__/telemetry.test.ts +++ b/packages/cli/src/__tests__/telemetry.test.ts @@ -11,6 +11,7 @@ */ import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; +import { isString } from "@openrouter/spawn-shared"; import * as v from "valibot"; // ── Schemas for validating PostHog payloads ───────────────────────────────── @@ -408,9 +409,62 @@ describe("telemetry", () => { mod.captureWarning("should not send"); mod.captureError("test", new Error("should not send")); + mod.captureEvent("should_not_send", { + spawn_id: "abc", + }); await flushAndWait(); expect(fetchMock).not.toHaveBeenCalled(); }); }); + + describe("captureEvent", () => { + it("emits a batched event with the given name and properties", async () => { + const mod = await import("../shared/telemetry.js"); + mod.initTelemetry("1.2.3-test"); + await drainStaleEvents(); + + mod.captureEvent("funnel_started", { + fast_mode: true, + elapsed_ms: 0, + }); + await flushAndWait(); + + const body = getLastBatchBody(fetchMock); + expect(body).not.toBeNull(); + const evt = body?.batch[0]; + expect(evt?.event).toBe("funnel_started"); + expect(evt?.properties.fast_mode).toBe(true); + expect(evt?.properties.elapsed_ms).toBe(0); + expect(evt?.properties.spawn_version).toBe("1.2.3-test"); + }); + + it("scrubs string property values but leaves non-strings alone", async () => { + const mod = await import("../shared/telemetry.js"); + mod.initTelemetry("1.2.3-test"); + await drainStaleEvents(); + + mod.captureEvent("spawn_connected", { + spawn_id: "abc123", + note: "contact me at alice@example.com about sk-or-v1-1234567890abcdef", + connect_count: 5, + lifetime_hours: 3.5, + }); + await flushAndWait(); + + const body = getLastBatchBody(fetchMock); + const props = body?.batch[0]?.properties; + // Non-string values pass through untouched. + expect(props?.spawn_id).toBe("abc123"); + expect(props?.connect_count).toBe(5); + expect(props?.lifetime_hours).toBe(3.5); + // String values get scrubbed. + const rawNote = props?.note; + const note = isString(rawNote) ? rawNote : ""; + expect(note).toContain("[REDACTED_EMAIL]"); + expect(note).not.toContain("alice@example.com"); + expect(note).toContain("[REDACTED_KEY]"); + expect(note).not.toContain("sk-or-v1-1234567890abcdef"); + }); + }); }); diff --git a/packages/cli/src/commands/delete.ts b/packages/cli/src/commands/delete.ts index f55b527c6..5109fabcb 100644 --- a/packages/cli/src/commands/delete.ts +++ b/packages/cli/src/commands/delete.ts @@ -22,6 +22,7 @@ import { validateServerIdentifier, validateUsername, } from "../security.js"; +import { trackSpawnDeleted } from "../shared/lifecycle-telemetry.js"; import { getHistoryPath } from "../shared/paths.js"; import { asyncTryCatch, asyncTryCatchIf, isNetworkError, tryCatch } from "../shared/result.js"; import { ensureSpriteAuthenticated, ensureSpriteCli, destroyServer as spriteDestroyServer } from "../sprite/sprite.js"; @@ -259,6 +260,8 @@ export async function confirmAndDelete( if (success) { const detail = lastMessage ? `: ${lastMessage}` : ""; p.log.success(`Server "${label}" deleted${detail}`); + // Lifecycle telemetry: lifetime hours + final login count. + trackSpawnDeleted(record); } else { const detail = lastMessage ? `: ${lastMessage}` : ""; p.log.error(`Failed to delete "${label}"${detail}`); @@ -448,6 +451,8 @@ export async function cmdDelete( const ok = await execDeleteServer(record); if (ok) { p.log.success(`Server "${label}" deleted`); + // Lifecycle telemetry: headless path also fires the event. + trackSpawnDeleted(record); } } return; diff --git a/packages/cli/src/commands/list.ts b/packages/cli/src/commands/list.ts index 7d46d3d5b..9fd84eb44 100644 --- a/packages/cli/src/commands/list.ts +++ b/packages/cli/src/commands/list.ts @@ -15,6 +15,7 @@ import { updateRecordIp, } from "../history.js"; import { agentKeys, cloudKeys, loadManifest } from "../manifest.js"; +import { trackSpawnConnected } from "../shared/lifecycle-telemetry.js"; import { asyncTryCatch, tryCatch, unwrapOr } from "../shared/result.js"; import { cmdConnect, cmdEnterAgent, cmdOpenDashboard } from "./connect.js"; import { confirmAndDelete } from "./delete.js"; @@ -707,6 +708,10 @@ export async function handleRecordAction( } if (action === "reconnect") { + // Lifecycle telemetry: record the login BEFORE we hand off to SSH. + // cmdConnect spawns an interactive session and never returns under normal + // use, so calling trackSpawnConnected after would be unreachable code. + trackSpawnConnected(selected); const reconnectResult = await asyncTryCatch(() => cmdConnect(conn, selected.agent)); if (!reconnectResult.ok) { p.log.error(`Connection failed: ${getErrorMessage(reconnectResult.error)}`); diff --git a/packages/cli/src/shared/lifecycle-telemetry.ts b/packages/cli/src/shared/lifecycle-telemetry.ts new file mode 100644 index 000000000..7aa63544c --- /dev/null +++ b/packages/cli/src/shared/lifecycle-telemetry.ts @@ -0,0 +1,101 @@ +// shared/lifecycle-telemetry.ts — Track spawn-level lifecycle events: +// login count and total lifetime on delete. +// +// Why it's here and not in telemetry.ts: +// telemetry.ts is a low-level primitive (PostHog batching, scrubbing, +// session context). It deliberately has no knowledge of SpawnRecord, +// history, or any product concepts. Lifecycle helpers need both, so +// they live one layer up. +// +// Event shapes (all respect SPAWN_TELEMETRY=0 opt-out via captureEvent): +// +// spawn_connected { spawn_id, agent, cloud, connect_count, date } +// spawn_deleted { spawn_id, agent, cloud, lifetime_hours, connect_count, date } +// +// Persistence model: +// connect_count + last_connected_at are stored inside +// SpawnRecord.connection.metadata as strings (the existing schema is +// Record, so we serialize numbers as strings and parse +// on read). saveMetadata merges — no risk of clobbering other keys. + +import type { SpawnRecord } from "../history.js"; + +import { saveMetadata } from "../history.js"; +import { captureEvent } from "./telemetry.js"; + +/** Read the stored connect count for a spawn, defaulting to 0. */ +function readConnectCount(record: SpawnRecord): number { + const raw = record.connection?.metadata?.connect_count; + if (!raw) { + return 0; + } + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n >= 0 ? n : 0; +} + +/** Compute lifetime hours between spawn creation and now (or delete time). */ +function computeLifetimeHours(record: SpawnRecord, endIso?: string): number { + const start = Date.parse(record.timestamp); + const end = endIso ? Date.parse(endIso) : Date.now(); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { + return 0; + } + return Math.round(((end - start) / (1000 * 60 * 60)) * 100) / 100; +} + +/** + * Record a user reconnecting to an existing spawn. + * + * Increments the stored connect_count, updates last_connected_at, and fires + * a spawn_connected telemetry event. Returns the new count so callers can + * also display it if they want. + */ +export function trackSpawnConnected(record: SpawnRecord): number { + if (!record.id || !record.connection) { + return 0; + } + const newCount = readConnectCount(record) + 1; + const nowIso = new Date().toISOString(); + + saveMetadata( + { + connect_count: String(newCount), + last_connected_at: nowIso, + }, + record.id, + ); + + captureEvent("spawn_connected", { + spawn_id: record.id, + agent: record.agent, + cloud: record.cloud, + connect_count: newCount, + date: nowIso, + }); + + return newCount; +} + +/** + * Record a user deleting a spawn. + * + * Emits a spawn_deleted event with the total lifetime (hours) and final + * login count, so we can build a "typical spawn lives N hours, N logins" + * picture in aggregate. Call AFTER the cloud destroy succeeds — failed + * deletes should not fire this event. + */ +export function trackSpawnDeleted(record: SpawnRecord): void { + if (!record.id) { + return; + } + const nowIso = new Date().toISOString(); + + captureEvent("spawn_deleted", { + spawn_id: record.id, + agent: record.agent, + cloud: record.cloud, + lifetime_hours: computeLifetimeHours(record, nowIso), + connect_count: readConnectCount(record), + date: nowIso, + }); +} diff --git a/packages/cli/src/shared/orchestrate.ts b/packages/cli/src/shared/orchestrate.ts index 6d9436978..3fc0e8866 100644 --- a/packages/cli/src/shared/orchestrate.ts +++ b/packages/cli/src/shared/orchestrate.ts @@ -28,6 +28,7 @@ import { isWindows } from "./shell.js"; import { injectSpawnSkill } from "./spawn-skill.js"; import { sleep, startSshTunnel } from "./ssh.js"; import { ensureSshKeys, getSshKeyOpts } from "./ssh-keys.js"; +import { captureEvent, setTelemetryContext } from "./telemetry.js"; import { logDebug, logError, @@ -43,6 +44,25 @@ import { withRetry, } from "./ui.js"; +// ── Funnel telemetry ──────────────────────────────────────────────────────── +// +// Tracks onboarding pipeline drop-off. Events flow through the shared +// PostHog pipeline in shared/telemetry.ts and respect SPAWN_TELEMETRY=0 opt-out. +// No PII — only agent/cloud names and elapsed timing. The goal is to answer +// "where do users bail before reaching a running agent" at the fleet level. +let _funnelStart = 0; + +function funnelElapsedMs(): number { + return _funnelStart > 0 ? Date.now() - _funnelStart : 0; +} + +function trackFunnel(step: string, extra: Record = {}): void { + captureEvent(step, { + elapsed_ms: funnelElapsedMs(), + ...extra, + }); +} + /** Docker container name used by --beta docker deployments. */ export const DOCKER_CONTAINER_NAME = "spawn-agent"; /** Docker registry hosting spawn agent images. */ @@ -298,8 +318,16 @@ export async function runOrchestration( logInfo(`${agent.name} on ${cloud.cloudLabel}`); process.stderr.write("\n"); + // Funnel telemetry: mark the start of the onboarding pipeline and attach + // agent/cloud as context so every event carries them automatically. + _funnelStart = Date.now(); + setTelemetryContext("agent", agentName); + setTelemetryContext("cloud", cloud.cloudName); + trackFunnel("funnel_started"); + // 1. Authenticate with cloud provider await cloud.authenticate(); + trackFunnel("funnel_cloud_authed"); const betaFeatures = new Set((process.env.SPAWN_BETA ?? "").split(",").filter(Boolean)); const fastMode = process.env.SPAWN_FAST === "1" || betaFeatures.has("parallel"); @@ -370,12 +398,14 @@ export async function runOrchestration( recordSpawn(spawnId, agentName, cloud.cloudName, connection); await cloud.waitForReady(); } + trackFunnel("funnel_vm_ready"); // API key must succeed if (apiKeyResult.status === "rejected") { throw apiKeyResult.reason; } const apiKey = apiKeyResult.value; + trackFunnel("funnel_credentials_ready"); // Model ID const rawModelId = process.env.MODEL_ID || loadPreferredModel(agentName) || agent.modelDefault; @@ -414,6 +444,7 @@ export async function runOrchestration( } } } + trackFunnel("funnel_install_completed"); // Inject env + continue with shared post-install flow clearInterval(keepAlive); @@ -434,6 +465,7 @@ export async function runOrchestration( // 2. Get API key const resolveApiKey = options?.getApiKey ?? getOrPromptApiKey; const apiKey = await resolveApiKey(agentName, cloud.cloudName); + trackFunnel("funnel_credentials_ready"); // 3. Pre-provision hooks if (agent.preProvision) { @@ -473,6 +505,7 @@ export async function runOrchestration( logError(getErrorMessage(r.error)); await retryOrQuit("Server may still be starting. Keep waiting?"); } + trackFunnel("funnel_vm_ready"); // 7. Env config const envPairs = agent.envVars(apiKey); @@ -504,6 +537,7 @@ export async function runOrchestration( } } } + trackFunnel("funnel_install_completed"); // Inject env + continue with shared post-install flow await injectEnvVars(cloud, envContent); @@ -595,6 +629,7 @@ async function postInstall( logWarn("Agent configuration failed (continuing with defaults)"); } } + trackFunnel("funnel_configure_completed"); // GitHub CLI setup if (!enabledSteps || enabledSteps.has("github")) { @@ -715,6 +750,7 @@ async function postInstall( await retryOrQuit("Retry pre-launch setup?"); } } + trackFunnel("funnel_prelaunch_completed"); // Web dashboard access let tunnelHandle: SshTunnelHandle | undefined; @@ -809,6 +845,13 @@ async function postInstall( logInfo(`Agent setup complete — ${agent.name} is ready on ${cloud.cloudLabel}`); process.stderr.write("\n"); + // Final funnel event — pipeline completed all the way to handoff. + // Downstream analysis: (funnel_started count) - (funnel_handoff count) = + // total drop-off. Per-step counts reveal where the drop-off happens. + trackFunnel("funnel_handoff", { + headless: process.env.SPAWN_HEADLESS === "1", + }); + const launchCmd = agent.launchCmd(); saveLaunchCmd(launchCmd, spawnId); diff --git a/packages/cli/src/shared/telemetry.ts b/packages/cli/src/shared/telemetry.ts index 5965943ff..1c0c2abf0 100644 --- a/packages/cli/src/shared/telemetry.ts +++ b/packages/cli/src/shared/telemetry.ts @@ -1,7 +1,9 @@ -// shared/telemetry.ts — PostHog telemetry for errors, warnings, and crashes. +// shared/telemetry.ts — PostHog telemetry for errors, warnings, crashes, and +// low-volume product events (funnel steps, spawn lifecycle). // Default on. Disable with SPAWN_TELEMETRY=0. -// Strictly errors/warnings/crashes — no command tracking, no session events. +// Never sends command args, file paths, or user prompt content. +import { isString } from "@openrouter/spawn-shared"; import { asyncTryCatch } from "./result.js"; // Same PostHog project as feedback.ts @@ -177,6 +179,31 @@ export function captureWarning(message: string): void { }); } +/** + * Capture a generic telemetry event (funnel steps, lifecycle events, etc.). + * + * Respects SPAWN_TELEMETRY=0 — when opt-out is set this is a no-op. All string + * values in `properties` are passed through the same scrubber as errors and + * warnings, so paths, API keys, emails, and IPs are redacted before upload. + * + * Intended for low-volume, high-signal product events like: + * - funnel_* (onboarding pipeline drop-off tracking in orchestrate.ts) + * - spawn_connected / spawn_deleted (lifecycle events) + * + * NOT intended for command tracking, keystroke tracking, or anything that + * could incidentally capture user-typed prompts or file paths. + */ +export function captureEvent(event: string, properties: Record = {}): void { + if (!_enabled) { + return; + } + const scrubbed: Record = {}; + for (const [key, value] of Object.entries(properties)) { + scrubbed[key] = isString(value) ? scrub(value) : value; + } + pushEvent(event, scrubbed); +} + /** Map our error types to PostHog mechanism types. */ function mechanismType(type: string): string { switch (type) {