From 72c0cf070a93c33aeafa8723a76261c60f5f3952 Mon Sep 17 00:00:00 2001 From: Ahmed Abushagur Date: Wed, 15 Apr 2026 00:55:29 -0700 Subject: [PATCH] fix(telemetry): opt-in default + picker funnel events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs from the #3305 rollout: 1. Test pollution: orchestrate.test.ts imports runOrchestration directly and never calls initTelemetry, but _enabled defaulted to true in the module so captureEvent happily fired real events at PostHog tagged agent=testagent. The onboarding funnel filled up with CI fixture data. 2. Funnel started too late: funnel_* events fired inside runOrchestration, which is only called AFTER the interactive picker completes. Users who bail at the agent/cloud/setup-options/name prompts were invisible — yet that's exactly where real drop-off happens. Fix 1 — telemetry.ts: - Default _enabled = false. Nothing fires until initTelemetry is explicitly called. Production (index.ts) calls it; tests that need telemetry (telemetry.test.ts) call it with BUN_ENV/NODE_ENV cleared. - Belt-and-suspenders: initTelemetry now short-circuits when BUN_ENV === "test" || NODE_ENV === "test", so even if future code calls it from a test context, events stay local. Fix 2 — picker instrumentation: New events fired before runOrchestration in every entry path: spawn_launched { mode: interactive | agent_interactive | direct | headless } menu_shown / menu_selected / menu_cancelled (only when user has prior spawns) agent_picker_shown agent_selected { agent } — also sets telemetry context cloud_picker_shown cloud_selected { cloud } — also sets telemetry context preflight_passed setup_options_shown setup_options_selected { step_count } name_prompt_shown name_entered picker_completed Wired into: commands/interactive.ts cmdInteractive + cmdAgentInteractive commands/run.ts cmdRun (direct `spawn `) cmdRunHeadless (only spawn_launched) runOrchestration's existing funnel_* events continue to fire unchanged. The final funnel in PostHog: spawn_launched → agent_selected → cloud_selected → preflight_passed → setup_options_selected → name_entered → picker_completed → funnel_started → funnel_cloud_authed → funnel_credentials_ready → funnel_vm_ready → funnel_install_completed → funnel_configure_completed → funnel_prelaunch_completed → funnel_handoff Tests: - telemetry.test.ts: 2 new env-guard tests (BUN_ENV, NODE_ENV), plus updated beforeEach to clear both env vars so existing tests still exercise initTelemetry. - Full suite: 2131/2131 pass, biome 0 errors. Bumps 1.0.12 -> 1.0.13 (patch — auto-propagates under #3296 policy). --- packages/cli/package.json | 2 +- packages/cli/src/__tests__/telemetry.test.ts | 51 +++++++++++++++- packages/cli/src/commands/interactive.ts | 62 ++++++++++++++++++++ packages/cli/src/commands/run.ts | 34 +++++++++++ packages/cli/src/shared/telemetry.ts | 18 +++++- 5 files changed, 164 insertions(+), 3 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 2bf9a7897..0b4a8d719 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openrouter/spawn", - "version": "1.0.12", + "version": "1.0.13", "type": "module", "bin": { "spawn": "cli.js" diff --git a/packages/cli/src/__tests__/telemetry.test.ts b/packages/cli/src/__tests__/telemetry.test.ts index 9a6f98664..2ee7490e5 100644 --- a/packages/cli/src/__tests__/telemetry.test.ts +++ b/packages/cli/src/__tests__/telemetry.test.ts @@ -105,13 +105,21 @@ function getFirstExceptionEntry( describe("telemetry", () => { let originalFetch: typeof global.fetch; let originalTelemetry: string | undefined; + let originalBunEnv: string | undefined; + let originalNodeEnv: string | undefined; let fetchMock: ReturnType; beforeEach(() => { originalFetch = global.fetch; originalTelemetry = process.env.SPAWN_TELEMETRY; - // Enable telemetry + originalBunEnv = process.env.BUN_ENV; + originalNodeEnv = process.env.NODE_ENV; + // Enable telemetry — these tests need initTelemetry() to actually flip + // _enabled to true so they can assert on the sent payloads. Clearing + // BUN_ENV/NODE_ENV lets the test-env guard in initTelemetry pass. delete process.env.SPAWN_TELEMETRY; + delete process.env.BUN_ENV; + delete process.env.NODE_ENV; // Mock fetch to capture PostHog payloads fetchMock = mock(() => Promise.resolve(new Response("ok"))); global.fetch = fetchMock; @@ -124,6 +132,16 @@ describe("telemetry", () => { } else { delete process.env.SPAWN_TELEMETRY; } + if (originalBunEnv !== undefined) { + process.env.BUN_ENV = originalBunEnv; + } else { + delete process.env.BUN_ENV; + } + if (originalNodeEnv !== undefined) { + process.env.NODE_ENV = originalNodeEnv; + } else { + delete process.env.NODE_ENV; + } }); /** Flush telemetry and wait for async send. */ @@ -416,6 +434,37 @@ describe("telemetry", () => { expect(fetchMock).not.toHaveBeenCalled(); }); + + it("does not send events when BUN_ENV=test (CI guard)", async () => { + process.env.BUN_ENV = "test"; + + const mod = await import("../shared/telemetry.js"); + mod.initTelemetry("0.0.0-test"); + await drainStaleEvents(); + + mod.captureEvent("funnel_started", { + agent: "claude", + }); + mod.captureError("test", new Error("ci")); + await flushAndWait(); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("does not send events when NODE_ENV=test (CI guard)", async () => { + process.env.NODE_ENV = "test"; + + const mod = await import("../shared/telemetry.js"); + mod.initTelemetry("0.0.0-test"); + await drainStaleEvents(); + + mod.captureEvent("funnel_started", { + agent: "claude", + }); + await flushAndWait(); + + expect(fetchMock).not.toHaveBeenCalled(); + }); }); describe("captureEvent", () => { diff --git a/packages/cli/src/commands/interactive.ts b/packages/cli/src/commands/interactive.ts index 92b9b6a82..489e4928b 100644 --- a/packages/cli/src/commands/interactive.ts +++ b/packages/cli/src/commands/interactive.ts @@ -8,6 +8,7 @@ import { getAgentOptionalSteps } from "../shared/agents.js"; import { hasSavedOpenRouterKey } from "../shared/oauth.js"; import { asyncTryCatch, tryCatch, unwrapOr } from "../shared/result.js"; import { maybeShowStarPrompt } from "../shared/star-prompt.js"; +import { captureEvent, setTelemetryContext } from "../shared/telemetry.js"; import { validateModelId } from "../shared/ui.js"; import { cmdLink } from "./link.js"; import { activeServerPicker } from "./list.js"; @@ -278,10 +279,19 @@ export { getAndValidateCloudChoices, promptSetupOptions, promptSpawnName, select export async function cmdInteractive(): Promise { p.intro(pc.inverse(` spawn v${VERSION} `)); + // Funnel entry — fires BEFORE any prompt so we catch users who bail at + // the very first screen. See also: funnel_* events in orchestrate.ts. + captureEvent("spawn_launched", { + mode: "interactive", + }); + // If the user has existing spawns, offer a top-level menu so they can // reconnect without knowing about `spawn list` or `spawn last`. const activeServers = getActiveServers(); if (activeServers.length > 0) { + captureEvent("menu_shown", { + active_servers: activeServers.length, + }); const topChoice = await p.select({ message: "What would you like to do?", options: [ @@ -296,8 +306,12 @@ export async function cmdInteractive(): Promise { ], }); if (p.isCancel(topChoice)) { + captureEvent("menu_cancelled"); handleCancel(); } + captureEvent("menu_selected", { + choice: String(topChoice), + }); if (topChoice === "connect") { const manifestResult = await asyncTryCatch(() => loadManifestWithSpinner()); const manifest = manifestResult.ok ? manifestResult.data : null; @@ -307,10 +321,20 @@ export async function cmdInteractive(): Promise { } const manifest = await loadManifestWithSpinner(); + captureEvent("agent_picker_shown"); const agentChoice = await selectAgent(manifest); + captureEvent("agent_selected", { + agent: agentChoice, + }); + setTelemetryContext("agent", agentChoice); const { clouds, hintOverrides } = getAndValidateCloudChoices(manifest, agentChoice); + captureEvent("cloud_picker_shown"); const cloudChoice = await selectCloud(manifest, clouds, hintOverrides); + captureEvent("cloud_selected", { + cloud: cloudChoice, + }); + setTelemetryContext("cloud", cloudChoice); // Handle "Link Existing Server" — redirect to spawn link with the agent pre-selected if (cloudChoice === "link-existing") { @@ -324,27 +348,37 @@ export async function cmdInteractive(): Promise { } await preflightCredentialCheck(manifest, cloudChoice); + captureEvent("preflight_passed"); // Skip setup prompt if steps already set via --steps or --config if (!process.env.SPAWN_ENABLED_STEPS) { + captureEvent("setup_options_shown"); const enabledSteps = await promptSetupOptions(agentChoice); if (enabledSteps) { process.env.SPAWN_ENABLED_STEPS = [ ...enabledSteps, ].join(","); + captureEvent("setup_options_selected", { + step_count: enabledSteps.size, + }); } } // Skills picker (--beta skills) await maybePromptSkills(manifest, agentChoice); + captureEvent("name_prompt_shown"); const spawnName = await promptSpawnName(); + // promptSpawnName cancels via handleCancel() on its own path if the user + // bails; if we reach this line the name was entered successfully. + captureEvent("name_entered"); const agentName = manifest.agents[agentChoice].name; const cloudName = manifest.clouds[cloudChoice].name; p.log.step(`Launching ${pc.bold(agentName)} on ${pc.bold(cloudName)}`); p.log.info(`Next time, run directly: ${pc.cyan(`spawn ${agentChoice} ${cloudChoice}`)}`); p.outro("Handing off to spawn script..."); + captureEvent("picker_completed"); const success = await execScript( cloudChoice, @@ -364,10 +398,19 @@ export async function cmdInteractive(): Promise { export async function cmdAgentInteractive(agent: string, prompt?: string, dryRun?: boolean): Promise { p.intro(pc.inverse(` spawn v${VERSION} `)); + // Same funnel entry as cmdInteractive — mode distinguishes the short-form + // (`spawn claude`) entry point from the full interactive picker. + captureEvent("spawn_launched", { + mode: "agent_interactive", + }); + const manifest = await loadManifestWithSpinner(); const resolvedAgent = resolveAgentKey(manifest, agent); if (!resolvedAgent) { + captureEvent("agent_invalid", { + raw: agent, + }); const agentMatch = findClosestKeyByNameOrKey(agent, agentKeys(manifest), (k) => manifest.agents[k].name); p.log.error(`Unknown agent: ${pc.bold(agent)}`); if (agentMatch) { @@ -377,8 +420,19 @@ export async function cmdAgentInteractive(agent: string, prompt?: string, dryRun process.exit(1); } + // Agent was pre-supplied on the command line — treat as implicitly selected. + captureEvent("agent_selected", { + agent: resolvedAgent, + }); + setTelemetryContext("agent", resolvedAgent); + const { clouds, hintOverrides } = getAndValidateCloudChoices(manifest, resolvedAgent); + captureEvent("cloud_picker_shown"); const cloudChoice = await selectCloud(manifest, clouds, hintOverrides); + captureEvent("cloud_selected", { + cloud: cloudChoice, + }); + setTelemetryContext("cloud", cloudChoice); // Handle "Link Existing Server" — redirect to spawn link with the agent pre-selected if (cloudChoice === "link-existing") { @@ -397,24 +451,32 @@ export async function cmdAgentInteractive(agent: string, prompt?: string, dryRun } await preflightCredentialCheck(manifest, cloudChoice); + captureEvent("preflight_passed"); // Skip setup prompt if steps already set via --steps or --config if (!process.env.SPAWN_ENABLED_STEPS) { + captureEvent("setup_options_shown"); const enabledSteps = await promptSetupOptions(resolvedAgent); if (enabledSteps) { process.env.SPAWN_ENABLED_STEPS = [ ...enabledSteps, ].join(","); + captureEvent("setup_options_selected", { + step_count: enabledSteps.size, + }); } } + captureEvent("name_prompt_shown"); const spawnName = await promptSpawnName(); + captureEvent("name_entered"); const agentName = manifest.agents[resolvedAgent].name; const cloudName = manifest.clouds[cloudChoice].name; p.log.step(`Launching ${pc.bold(agentName)} on ${pc.bold(cloudName)}`); p.log.info(`Next time, run directly: ${pc.cyan(`spawn ${resolvedAgent} ${cloudChoice}`)}`); p.outro("Handing off to spawn script..."); + captureEvent("picker_completed"); const success = await execScript( cloudChoice, diff --git a/packages/cli/src/commands/run.ts b/packages/cli/src/commands/run.ts index e05fd321b..ec5d3208d 100644 --- a/packages/cli/src/commands/run.ts +++ b/packages/cli/src/commands/run.ts @@ -20,6 +20,7 @@ import { import { asyncTryCatch, isFileError, tryCatch, tryCatchIf } from "../shared/result.js"; import { getLocalShell, isWindows } from "../shared/shell.js"; import { maybeShowStarPrompt } from "../shared/star-prompt.js"; +import { captureEvent, setTelemetryContext } from "../shared/telemetry.js"; import { logError, logInfo, logStep, prepareStdinForHandoff, toKebabCase } from "../shared/ui.js"; import { promptSetupOptions, promptSpawnName } from "./interactive.js"; import { handleRecordAction } from "./list.js"; @@ -987,6 +988,13 @@ function runBundleHeadless( export async function cmdRunHeadless(agent: string, cloud: string, opts: HeadlessOptions = {}): Promise { const { prompt, debug, outputFormat, spawnName } = opts; + // Funnel entry for headless runs. No picker to instrument — headless either + // validates and proceeds straight to runOrchestration, or it errors out. + // The orchestrate.ts funnel_* events cover the rest. + captureEvent("spawn_launched", { + mode: "headless", + }); + // Phase 1: Validate inputs (exit code 3) const validationResult = tryCatch(() => { validateIdentifier(agent, "Agent name"); @@ -1230,6 +1238,13 @@ export async function cmdRun( dryRun?: boolean, debug?: boolean, ): Promise { + // Funnel entry for the non-interactive `spawn ` path. + // mode distinguishes this from the interactive pickers so we can split the + // funnel by entry point in PostHog. + captureEvent("spawn_launched", { + mode: "direct", + }); + const manifest = await loadManifestWithSpinner(); ({ agent, cloud } = resolveAndLog(manifest, agent, cloud)); @@ -1237,24 +1252,42 @@ export async function cmdRun( ({ agent, cloud } = detectAndFixSwappedArgs(manifest, agent, cloud)); validateEntities(manifest, agent, cloud); + // Both arguments were pre-supplied — treat as implicit selection so the + // funnel has the same shape regardless of entry point. + captureEvent("agent_selected", { + agent, + }); + captureEvent("cloud_selected", { + cloud, + }); + setTelemetryContext("agent", agent); + setTelemetryContext("cloud", cloud); + if (dryRun) { showDryRunPreview(manifest, agent, cloud, prompt); return; } await preflightCredentialCheck(manifest, cloud); + captureEvent("preflight_passed"); // Skip setup prompt if steps already set via --steps or --config if (!process.env.SPAWN_ENABLED_STEPS) { + captureEvent("setup_options_shown"); const enabledSteps = await promptSetupOptions(agent); if (enabledSteps) { process.env.SPAWN_ENABLED_STEPS = [ ...enabledSteps, ].join(","); + captureEvent("setup_options_selected", { + step_count: enabledSteps.size, + }); } } + captureEvent("name_prompt_shown"); const spawnName = await promptSpawnName(); + captureEvent("name_entered"); // If a name was given, check whether an active instance with that name already // exists for this agent + cloud combination. When it does, route the user into @@ -1276,6 +1309,7 @@ export async function cmdRun( const cloudName = manifest.clouds[cloud].name; const suffix = prompt ? " with prompt..." : "..."; p.log.step(`Launching ${pc.bold(agentName)} on ${pc.bold(cloudName)}${suffix}`); + captureEvent("picker_completed"); const success = await execScript( cloud, diff --git a/packages/cli/src/shared/telemetry.ts b/packages/cli/src/shared/telemetry.ts index 1c0c2abf0..3380447d1 100644 --- a/packages/cli/src/shared/telemetry.ts +++ b/packages/cli/src/shared/telemetry.ts @@ -123,7 +123,14 @@ interface TelemetryEvent { // ── State ─────────────────────────────────────────────────────────────────── -let _enabled = true; +// Telemetry is OPT-IN: nothing fires until initTelemetry() is called. This +// matters for tests that import modules which call captureEvent — without +// this default, every `bun test` run of orchestrate.test.ts fired real +// PostHog events tagged agent=testagent, because the test imports +// runOrchestration directly (bypassing index.ts's initTelemetry call) but +// runOrchestration calls captureEvent unconditionally. Defaulting _enabled +// to false means no events escape until a real process explicitly opts in. +let _enabled = false; let _sessionId = ""; let _context: Record = {}; const _events: TelemetryEvent[] = []; @@ -133,6 +140,15 @@ let _flushScheduled = false; /** Initialize telemetry. Call once at startup. */ export function initTelemetry(version: string): void { + // Never send telemetry from test environments. bun:test sets BUN_ENV=test, + // Node test runners set NODE_ENV=test. Without this guard, every CI run of + // orchestrate.test.ts fires real PostHog events tagged agent=testagent, + // polluting the onboarding funnel with fixture data. (See #3305 follow-up.) + if (process.env.NODE_ENV === "test" || process.env.BUN_ENV === "test") { + _enabled = false; + return; + } + _enabled = process.env.SPAWN_TELEMETRY !== "0"; if (!_enabled) { return;