diff --git a/.changeset/usage-telemetry.md b/.changeset/usage-telemetry.md new file mode 100644 index 000000000..c0fc7d076 --- /dev/null +++ b/.changeset/usage-telemetry.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Collect usage telemetry (command name, flag names, duration, outcome, a random machine identifier — and your workspace and app IDs when a project is linked; never arguments, option values, paths, or personal data). The first run only shows a disclosure notice and sends nothing (CI environments send from the first run), and `--verbose` prints every event before it is sent. Control it with the new `clerk telemetry status|disable|enable` subcommand, or the `CLERK_TELEMETRY_DISABLED` / `DO_NOT_TRACK` environment variables (any non-false value opts out). diff --git a/README.md b/README.md index ab88d248d..0df83f278 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Commands: impersonate|imp [options] [user] Impersonate a Clerk user env Manage environment variables config Manage instance configuration + telemetry Control CLI usage telemetry (status, disable, enable) enable Enable Clerk features on the linked instance disable Disable Clerk features on the linked instance api [options] [endpoint] [filter] Make authenticated requests to the Clerk API @@ -56,3 +57,17 @@ Commands: help [command] Display help for command bird Play Clerk Bird, a Flappy Bird game in your terminal ``` + +## Telemetry + +The Clerk CLI collects usage telemetry: command name, flag names, duration, outcome, +environment signals (OS, install method, terminal), a random machine identifier — and +your workspace and app IDs when a project is linked. It never collects command +arguments, option values, file paths, or personal data. The first run only shows a +disclosure notice and sends nothing (CI environments send from the first run), and +`clerk --verbose` prints every event before it is sent. +See https://clerk.com/docs/telemetry for details. + +Opt out with `clerk telemetry disable`, or by setting `CLERK_TELEMETRY_DISABLED=1` +(the standard `DO_NOT_TRACK=1` also works). `clerk telemetry status` shows the +effective state and why. diff --git a/packages/cli-core/src/cli-program.test.ts b/packages/cli-core/src/cli-program.test.ts index b74608de1..d8b5eb20b 100644 --- a/packages/cli-core/src/cli-program.test.ts +++ b/packages/cli-core/src/cli-program.test.ts @@ -1,6 +1,14 @@ -import { test, expect, describe } from "bun:test"; -import { createProgram, formatApiBody, outputJsonError } from "./cli-program.ts"; -import { ApiError } from "./lib/errors.ts"; +import { test, expect, describe, beforeEach, afterEach } from "bun:test"; +import { createProgram, formatApiBody, outputJsonError, reportError } from "./cli-program.ts"; +import { + ApiError, + CliError, + ERROR_CODE, + EXIT_CODE, + PlapiError, + UserAbortError, +} from "./lib/errors.ts"; +import { telemetryResultForError } from "./lib/telemetry.ts"; import { useCaptureLog } from "./test/lib/stubs.ts"; test("registers users as a top-level command", () => { @@ -367,3 +375,210 @@ describe("outputJsonError", () => { expect(parse().error).not.toHaveProperty("examples"); }); }); + +// `reportError` is the whole error-dispatch cascade of `runProgram`'s catch, +// minus the `process.exit` call. Exercising it directly is the only way to +// cover every branch — going through `runProgram` needs a `process.exit` spy +// and can only reach one branch per invocation. +describe("reportError", () => { + const captured = useCaptureLog(); + + // `getMode()` falls back to TTY detection, which reports agent mode under the + // test runner. Drive the env var rather than `setMode()` so the override is + // restorable — `mode.ts` exposes no way to clear a forced mode. + const originalMode = process.env.CLERK_MODE; + const asHuman = () => { + process.env.CLERK_MODE = "human"; + }; + const asAgent = () => { + process.env.CLERK_MODE = "agent"; + }; + + beforeEach(asHuman); + afterEach(() => { + if (originalMode === undefined) { + delete process.env.CLERK_MODE; + } else { + process.env.CLERK_MODE = originalMode; + } + }); + + const json = () => JSON.parse(captured.err.trim()) as { error: Record }; + + /** Matches what `@inquirer/prompts` throws on Ctrl+C. */ + const promptExitError = () => { + const error = new Error("User force closed the prompt with SIGINT"); + error.name = "ExitPromptError"; + return error; + }; + + describe("aborts", () => { + test("UserAbortError exits clean and prints nothing", () => { + expect(reportError(new UserAbortError(), false)).toBe(EXIT_CODE.SUCCESS); + expect(captured.err).toBe(""); + expect(captured.out).toBe(""); + }); + + test("a force-closed prompt exits clean and prints nothing", () => { + expect(reportError(promptExitError(), false)).toBe(EXIT_CODE.SUCCESS); + expect(captured.err).toBe(""); + expect(captured.out).toBe(""); + }); + + test("an ExitPromptError with a different message is not treated as an abort", () => { + const error = new Error("something else"); + error.name = "ExitPromptError"; + expect(reportError(error, false)).toBe(EXIT_CODE.GENERAL); + expect(captured.err).toContain("something else"); + }); + }); + + describe("CliError", () => { + test("returns the error's own exit code", () => { + const error = new CliError("bad flag", { + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }); + expect(reportError(error, false)).toBe(EXIT_CODE.USAGE); + }); + + test("human mode prints message, examples, and docs link", () => { + const error = new CliError("Not linked", { + code: ERROR_CODE.NOT_LINKED, + docsUrl: "https://example.com/docs/link", + examples: [{ command: "clerk link", description: "Link this directory" }], + }); + reportError(error, false); + expect(captured.err).toContain("Not linked"); + expect(captured.err).toContain("clerk link"); + expect(captured.err).toContain("For more information, see: https://example.com/docs/link"); + }); + + test("agent mode emits a JSON payload carrying code, docs URL, and examples", () => { + asAgent(); + const error = new CliError("Not linked", { + code: ERROR_CODE.NOT_LINKED, + docsUrl: "https://example.com/docs/link", + examples: [{ command: "clerk link", description: "Link this directory" }], + }); + expect(reportError(error, false)).toBe(EXIT_CODE.GENERAL); + expect(json().error).toMatchObject({ + code: ERROR_CODE.NOT_LINKED, + message: "Not linked", + docsUrl: "https://example.com/docs/link", + examples: [{ command: "clerk link", description: "Link this directory" }], + }); + }); + + test("agent mode falls back to human rendering when the error has no code", () => { + asAgent(); + reportError(new CliError("uncoded failure"), false); + expect(captured.err).toContain("uncoded failure"); + expect(() => json()).toThrow(); + }); + }); + + describe("ApiError", () => { + const body = JSON.stringify({ + errors: [ + { code: "form_param_missing", message: "Missing param", meta: { param_name: "email" } }, + ], + clerk_trace_id: "trace_123", + }); + + // Only the prefix + status wiring is asserted here; the detail string is + // `formatStructuredError`'s job and is covered by the `formatApiBody` block. + test("human mode prefixes with the status", () => { + reportError(new ApiError(400, body), false); + expect(captured.err).toContain("Request failed (400): Missing param"); + }); + + test("human mode prefers the error's context over the default prefix", () => { + const error = new ApiError(400, body); + error.context = "Platform API request failed"; + reportError(error, false); + expect(captured.err).toContain("Platform API request failed (400):"); + expect(captured.err).not.toContain("Request failed (400):"); + }); + + test("verbose adds the request URL and trace id", () => { + reportError(new PlapiError(400, body, "https://api.clerk.com/v1/apps"), true); + expect(captured.err).toContain("URL: https://api.clerk.com/v1/apps"); + expect(captured.err).toContain("Trace: trace_123"); + }); + + test("agent mode emits the code and a structured errors array", () => { + asAgent(); + reportError(new ApiError(400, body), false); + expect(json().error).toMatchObject({ + code: "form_param_missing", + message: "Request failed (400): Missing param\n Parameter: email", + errors: [ + { code: "form_param_missing", message: "Missing param", meta: { param_name: "email" } }, + ], + }); + }); + + test("agent mode falls back to api_error and omits errors when there is no code or meta", () => { + asAgent(); + reportError(new ApiError(502, "upstream exploded"), false); + expect(json().error.code).toBe("api_error"); + expect(json().error).not.toHaveProperty("errors"); + }); + }); + + describe("unexpected failures", () => { + test("a plain Error prints its message in human mode", () => { + expect(reportError(new Error("socket hang up"), false)).toBe(EXIT_CODE.GENERAL); + expect(captured.err).toContain("socket hang up"); + }); + + test("a plain Error becomes unexpected_error in agent mode", () => { + asAgent(); + reportError(new Error("socket hang up"), false); + expect(json().error).toMatchObject({ + code: "unexpected_error", + message: "socket hang up", + }); + }); + + test("a non-Error throw gets a generic message in human mode", () => { + expect(reportError("just a string", false)).toBe(EXIT_CODE.GENERAL); + expect(captured.err).toContain("An unexpected error occurred"); + }); + + test("a non-Error throw gets a generic message in agent mode", () => { + asAgent(); + expect(reportError({ nope: true }, false)).toBe(EXIT_CODE.GENERAL); + expect(json().error).toMatchObject({ + code: "unexpected_error", + message: "An unexpected error occurred", + }); + }); + }); + + // `telemetryResultForError` runs the same instanceof cascade to decide what + // exit code the event reports. If the two drift, telemetry records a code the + // user never saw, and nothing else in the suite would notice. + describe("agrees with telemetryResultForError on the exit code", () => { + const fixtures: [string, unknown][] = [ + ["UserAbortError", new UserAbortError()], + ["prompt exit", promptExitError()], + ["CliError (default code)", new CliError("boom")], + [ + "CliError (usage code)", + new CliError("bad flag", { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }), + ], + ["ApiError", new ApiError(404, "{}")], + ["PlapiError", new PlapiError(500, "{}", "https://api.clerk.com/v1/apps")], + ["plain Error", new Error("socket hang up")], + ["non-Error throw", "just a string"], + ]; + + for (const [name, error] of fixtures) { + test(name, () => { + expect(reportError(error, false)).toBe(telemetryResultForError(error).exitCode); + }); + } + }); +}); diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index 9ea89cb54..53f90b4e1 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -13,6 +13,7 @@ import { registerUsers } from "./commands/users/index.ts"; import { registerImpersonate } from "./commands/impersonate/index.ts"; import { registerEnv } from "./commands/env/index.ts"; import { registerConfig } from "./commands/config/index.ts"; +import { registerTelemetry } from "./commands/telemetry/index.ts"; import { registerToggles } from "./commands/toggles/index.ts"; import { registerApi } from "./commands/api/index.ts"; import { registerDoctor } from "./commands/doctor/index.ts"; @@ -45,6 +46,11 @@ import { isAgent } from "./mode.ts"; import { log } from "./lib/log.ts"; import { maybeNotifyUpdate, getCurrentVersion } from "./lib/update-check.ts"; import { registerExtras } from "@clerk/cli-extras"; +import { + finalizeAndSendTelemetry, + startCommandTelemetry, + telemetryResultForError, +} from "./lib/telemetry.ts"; /** * The root `clerk` program with its global options applied, so registrants @@ -66,6 +72,7 @@ const registrants: CommandRegistrant[] = [ registerImpersonate, registerEnv, registerConfig, + registerTelemetry, registerToggles, registerApi, registerDoctor, @@ -100,7 +107,9 @@ export function createProgram(): Program { ) .option("--verbose", "Show detailed output (enables debug messages)") as Program; - program.hook("preAction", async () => { + program.hook("preAction", async (_thisCommand, actionCommand) => { + // First so hook-time failures (e.g. invalid --mode) still produce an event. + startCommandTelemetry(actionCommand); // Reset log level at the start of each command invocation so a previous // --verbose doesn't leak into subsequent runs. setLogLevel("info"); @@ -231,78 +240,109 @@ export async function runProgram( try { const { argv, from } = await resolveArgv(args, options?.from); await program.parseAsync(argv, { from }); + // Some commands report failure via process.exitCode instead of throwing — + // read it back so telemetry doesn't record them as successes. + const softExitCode = Number(process.exitCode ?? EXIT_CODE.SUCCESS); + await finalizeAndSendTelemetry({ + outcome: softExitCode === EXIT_CODE.SUCCESS ? "success" : "error", + exitCode: softExitCode, + }); } catch (error) { - const verbose = program.opts().verbose ?? false; + // Started before rendering so the message is printed before we block on the + // send — a slow telemetry endpoint never delays the error output — then + // awaited once at the single exit below, so no branch can forget to flush. + // `reportError` is synchronous, so the send makes no progress until the + // await; the ordering is the point, not overlap. + const pendingTelemetry = finalizeAndSendTelemetry(telemetryResultForError(error)); + const exitCode = reportError(error, program.opts().verbose ?? false); + await pendingTelemetry; + process.exit(exitCode); + } +} - if (error instanceof UserAbortError || isPromptExitError(error)) { - process.exit(EXIT_CODE.SUCCESS); - } +/** + * Render the user-facing form of a thrown error and return the exit code it + * should produce. Deliberately does not exit: `runProgram` owns the single + * exit so the pending telemetry send is always awaited first. + * + * Must stay synchronous — `runProgram` relies on nothing awaiting between the + * telemetry send starting and the message reaching the terminal. + * + * The returned code must match `telemetryResultForError`'s `exitCode` for the + * same error, or telemetry records an exit code the user never saw. Both + * cascades are pinned together in `cli-program.test.ts`. + * + * Exported for testing; production callers go through `runProgram`. + */ +export function reportError(error: unknown, verbose: boolean): number { + if (error instanceof UserAbortError || isPromptExitError(error)) { + return EXIT_CODE.SUCCESS; + } - if (error instanceof CliError) { - if (isAgent() && error.code) { - outputJsonError(error.code, error.message, error.docsUrl, undefined, error.examples); - } else { - if (error.message) { - log.error(error.message); - } - if (error.examples?.length) { - log.info(`\n${formatExamplesBlock(error.examples)}`); - } - if (error.docsUrl) { - log.info(`\nFor more information, see: ${error.docsUrl}`); - } + if (error instanceof CliError) { + if (isAgent() && error.code) { + outputJsonError(error.code, error.message, error.docsUrl, undefined, error.examples); + } else { + if (error.message) { + log.error(error.message); } - process.exit(error.exitCode); - } - - if (error instanceof ApiError) { - const detail = formatApiBody(error, verbose); - const prefix = error.context ?? "Request failed"; - if (isAgent()) { - const apiErrors: ApiErrorEntry[] | undefined = - error.code || error.meta - ? [ - { - ...(error.code ? { code: error.code } : {}), - ...(error.message ? { message: error.message } : {}), - ...(error.meta ? { meta: error.meta } : {}), - }, - ] - : undefined; - outputJsonError( - error.code ?? "api_error", - `${prefix} (${error.status}): ${detail}`, - undefined, - apiErrors, - ); - } else { - log.error(`${prefix} (${error.status}): ${detail}`); - if (verbose && (error instanceof PlapiError || error instanceof FapiError) && error.url) { - log.error(` URL: ${error.url}`); - } - if (verbose && error.clerkTraceId) { - log.error(` Trace: ${error.clerkTraceId}`); - } + if (error.examples?.length) { + log.info(`\n${formatExamplesBlock(error.examples)}`); + } + if (error.docsUrl) { + log.info(`\nFor more information, see: ${error.docsUrl}`); } - process.exit(EXIT_CODE.GENERAL); } + return error.exitCode; + } - if (error instanceof Error) { - if (isAgent()) { - outputJsonError("unexpected_error", error.message); - } else { - log.error(error.message); + if (error instanceof ApiError) { + const detail = formatApiBody(error, verbose); + const prefix = error.context ?? "Request failed"; + if (isAgent()) { + const apiErrors: ApiErrorEntry[] | undefined = + error.code || error.meta + ? [ + { + ...(error.code ? { code: error.code } : {}), + ...(error.message ? { message: error.message } : {}), + ...(error.meta ? { meta: error.meta } : {}), + }, + ] + : undefined; + outputJsonError( + error.code ?? "api_error", + `${prefix} (${error.status}): ${detail}`, + undefined, + apiErrors, + ); + } else { + log.error(`${prefix} (${error.status}): ${detail}`); + if (verbose && (error instanceof PlapiError || error instanceof FapiError) && error.url) { + log.error(` URL: ${error.url}`); + } + if (verbose && error.clerkTraceId) { + log.error(` Trace: ${error.clerkTraceId}`); } - process.exit(EXIT_CODE.GENERAL); } + return EXIT_CODE.GENERAL; + } + if (error instanceof Error) { if (isAgent()) { - outputJsonError("unexpected_error", "An unexpected error occurred"); + outputJsonError("unexpected_error", error.message); } else { - log.error("An unexpected error occurred"); + log.error(error.message); } - process.exit(EXIT_CODE.GENERAL); + return EXIT_CODE.GENERAL; + } + + if (isAgent()) { + outputJsonError("unexpected_error", "An unexpected error occurred"); + } else { + log.error("An unexpected error occurred"); } + return EXIT_CODE.GENERAL; } interface ApiErrorEntry { diff --git a/packages/cli-core/src/commands/telemetry/README.md b/packages/cli-core/src/commands/telemetry/README.md new file mode 100644 index 000000000..676b9a41a --- /dev/null +++ b/packages/cli-core/src/commands/telemetry/README.md @@ -0,0 +1,26 @@ +# clerk telemetry + +Control CLI usage telemetry. + +## Usage + +```sh +clerk telemetry status # Show whether telemetry is enabled and why +clerk telemetry disable # Persist an opt-out for this machine +clerk telemetry enable # Remove the persisted opt-out +``` + +`status` prints the bare state (`enabled`/`disabled`) on stdout for scripts, with the +winning reason on stderr, in precedence order: the +`CLERK_TELEMETRY_DISABLED` / `DO_NOT_TRACK` environment variables, then the persisted +opt-out from `clerk telemetry disable`, then the automatic dev-build guard. In agent +mode it emits the status object as JSON on stdout. + +A run of `clerk telemetry disable` never sends a telemetry event itself — the opt-out +is re-checked after the command executes. + +## Clerk API endpoints + +None. These subcommands only read and write the local CLI config file +(`telemetryDisabled` flag). Telemetry events themselves are documented in the root +README's Telemetry section. diff --git a/packages/cli-core/src/commands/telemetry/index.ts b/packages/cli-core/src/commands/telemetry/index.ts new file mode 100644 index 000000000..48128a564 --- /dev/null +++ b/packages/cli-core/src/commands/telemetry/index.ts @@ -0,0 +1,79 @@ +import type { Program } from "../../cli-program.ts"; +import { setTelemetryDisabled } from "../../lib/config.ts"; +import { getTelemetryStatus, type TelemetryStatus } from "../../lib/telemetry.ts"; +import { log } from "../../lib/log.ts"; +import { isAgent } from "../../mode.ts"; + +function describeDisabledReason(status: Exclude): string { + switch (status.reason) { + case "env": + return `Disabled by the \`${status.envVar}\` environment variable.`; + case "config": + return "Disabled via `clerk telemetry disable`. Re-enable with `clerk telemetry enable`."; + case "dev-build": + return "Disabled automatically for dev builds (`0.0.0-dev`)."; + } +} + +export async function telemetryStatus(): Promise { + const status = await getTelemetryStatus(); + if (isAgent()) { + log.data(JSON.stringify(status)); + return; + } + log.data(status.enabled ? "enabled" : "disabled"); + if (status.enabled) { + log.info("Opt out with `clerk telemetry disable` (or `CLERK_TELEMETRY_DISABLED=1`)."); + } else { + log.info(describeDisabledReason(status)); + } +} + +export async function telemetryDisable(): Promise { + await setTelemetryDisabled(true); + log.success("Telemetry disabled. Nothing will be sent from this machine."); +} + +export async function telemetryEnable(): Promise { + await setTelemetryDisabled(false); + log.success("Telemetry enabled."); + const status = await getTelemetryStatus(); + if (status.enabled) return; + if (status.reason === "env") { + log.warn(`\`${status.envVar}\` is still set — telemetry stays disabled until it is unset.`); + return; + } + if (status.reason === "dev-build") { + log.warn("This is a dev build (`0.0.0-dev`) — telemetry stays disabled regardless."); + } +} + +export function registerTelemetry(program: Program): void { + const telemetry = program + .command("telemetry") + .description("Control CLI usage telemetry (status, disable, enable)"); + + telemetry + .command("status") + .description("Show whether telemetry is enabled and why") + .setExamples([ + { command: "clerk telemetry status", description: "Show the current telemetry state" }, + ]) + .action(telemetryStatus); + + telemetry + .command("disable") + .description("Disable telemetry for this machine (persisted)") + .setExamples([ + { command: "clerk telemetry disable", description: "Opt out of usage telemetry" }, + ]) + .action(telemetryDisable); + + telemetry + .command("enable") + .description("Re-enable telemetry for this machine") + .setExamples([ + { command: "clerk telemetry enable", description: "Opt back in to usage telemetry" }, + ]) + .action(telemetryEnable); +} diff --git a/packages/cli-core/src/commands/users/interactive/instance-context.test.ts b/packages/cli-core/src/commands/users/interactive/instance-context.test.ts index 96b620317..ef94a615a 100644 --- a/packages/cli-core/src/commands/users/interactive/instance-context.test.ts +++ b/packages/cli-core/src/commands/users/interactive/instance-context.test.ts @@ -15,6 +15,9 @@ mock.module("../../../lib/listage.ts", () => ({ })); mock.module("../../../lib/config.ts", () => ({ + // fetch.ts (imported process-wide) reads these from config.ts. + getTelemetryDisabled: async () => false, + getTelemetryNoticeShown: async () => true, resolveAppContext: (...args: unknown[]) => mockResolveAppContext(...args), resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), resolveFetchedApplicationInstance: ( diff --git a/packages/cli-core/src/commands/webhooks/listen.test.ts b/packages/cli-core/src/commands/webhooks/listen.test.ts index fc3ec55e7..5b2040f33 100644 --- a/packages/cli-core/src/commands/webhooks/listen.test.ts +++ b/packages/cli-core/src/commands/webhooks/listen.test.ts @@ -32,6 +32,9 @@ mock.module("./relay-client.ts", () => ({ RelayClient: FakeRelayClient })); const mockGetRelayEntry = mock(); const mockSetRelayEntry = mock(); mock.module("../../lib/config.ts", () => ({ + // fetch.ts (imported process-wide) reads these from config.ts. + getTelemetryDisabled: async () => false, + getTelemetryNoticeShown: async () => true, getRelayEntry: (...args: unknown[]) => mockGetRelayEntry(...args), setRelayEntry: (...args: unknown[]) => mockSetRelayEntry(...args), })); diff --git a/packages/cli-core/src/lib/config.test.ts b/packages/cli-core/src/lib/config.test.ts index c8deae64a..03d2f4eae 100644 --- a/packages/cli-core/src/lib/config.test.ts +++ b/packages/cli-core/src/lib/config.test.ts @@ -16,6 +16,12 @@ const { resolveInstanceId, resolveAppContext, resolveFetchedApplicationInstance, + ensureMachineUuid, + getTelemetryDisabled, + getTelemetryNoticeShown, + markTelemetryNoticeShown, + setTelemetryDisabled, + setEnvironment, _setConfigDir, } = await import("./config.ts"); type Profile = @@ -331,4 +337,65 @@ describe("config", () => { }); }); }); + + describe("telemetry config", () => { + test("ensureMachineUuid generates once and persists", async () => { + const first = await ensureMachineUuid(); + expect(first).toMatch(/^[0-9a-f-]{36}$/); + const second = await ensureMachineUuid(); + expect(second).toBe(first); + }); + + test("machineUuid survives readConfig round-trip with other fields", async () => { + const uuid = await ensureMachineUuid(); + await setEnvironment("production"); + const config = await readConfig(); + expect(config.machineUuid).toBe(uuid); + }); + + test("markTelemetryNoticeShown returns true exactly once", async () => { + expect(await markTelemetryNoticeShown()).toBe(true); + expect(await markTelemetryNoticeShown()).toBe(false); + }); + + test("getTelemetryNoticeShown peeks without mutating", async () => { + expect(await getTelemetryNoticeShown()).toBe(false); + expect(await getTelemetryNoticeShown()).toBe(false); + await markTelemetryNoticeShown(); + expect(await getTelemetryNoticeShown()).toBe(true); + }); + + test("setTelemetryDisabled(true) persists and getTelemetryDisabled reads it", async () => { + expect(await getTelemetryDisabled()).toBe(false); + await setTelemetryDisabled(true); + expect(await getTelemetryDisabled()).toBe(true); + const config = await readConfig(); + expect(config.telemetryDisabled).toBe(true); + }); + + test("setTelemetryDisabled(false) removes the flag entirely", async () => { + await setTelemetryDisabled(true); + await setTelemetryDisabled(false); + expect(await getTelemetryDisabled()).toBe(false); + const config = await readConfig(); + expect(config.telemetryDisabled).toBeUndefined(); + }); + + test("telemetryDisabled survives a round-trip with other config writes", async () => { + await setTelemetryDisabled(true); + await setEnvironment("production"); + expect(await getTelemetryDisabled()).toBe(true); + }); + + test("disabling telemetry sheds the machine identity", async () => { + const before = await ensureMachineUuid(); + await setTelemetryDisabled(true); + const config = await readConfig(); + expect(config.machineUuid).toBeUndefined(); + + await setTelemetryDisabled(false); + const after = await ensureMachineUuid(); + expect(after).not.toBe(before); + }); + }); }); diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index c0bccb29f..41943b990 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -55,6 +55,9 @@ interface ClerkConfig { auth?: Record; profiles: Record; relay?: Record; + machineUuid?: string; + telemetryNoticeShown?: boolean; + telemetryDisabled?: boolean; } function defaultConfig(): ClerkConfig { @@ -71,6 +74,10 @@ function migrateRawConfig(raw: Record): ClerkConfig { profiles: (raw.profiles as Record) ?? {}, }; + if (typeof raw.machineUuid === "string") config.machineUuid = raw.machineUuid; + if (raw.telemetryNoticeShown === true) config.telemetryNoticeShown = true; + if (raw.telemetryDisabled === true) config.telemetryDisabled = true; + if (raw.relay && typeof raw.relay === "object" && !Array.isArray(raw.relay)) { const relay: Record = {}; for (const [key, val] of Object.entries(raw.relay as Record)) { @@ -207,6 +214,49 @@ export async function setRelayEntry(key: string, entry: RelayEntry): Promise { + const config = await readConfig(); + if (config.machineUuid) return config.machineUuid; + config.machineUuid = crypto.randomUUID(); + await writeConfig(config); + return config.machineUuid; +} + +/** Read-only peek at the notice flag (the User-Agent gating needs it). */ +export async function getTelemetryNoticeShown(): Promise { + const config = await readConfig(); + return config.telemetryNoticeShown === true; +} + +/** Flip the one-time telemetry notice flag. Returns true only on the transition. */ +export async function markTelemetryNoticeShown(): Promise { + const config = await readConfig(); + if (config.telemetryNoticeShown) return false; + config.telemetryNoticeShown = true; + await writeConfig(config); + return true; +} + +/** Persisted telemetry opt-out, set via `clerk telemetry disable`. */ +export async function getTelemetryDisabled(): Promise { + const config = await readConfig(); + return config.telemetryDisabled === true; +} + +export async function setTelemetryDisabled(disabled: boolean): Promise { + const config = await readConfig(); + if (disabled) { + config.telemetryDisabled = true; + // Opting out is an identity boundary: shed the machine id so a later + // opt-in starts unlinkable from prior events. + delete config.machineUuid; + } else { + delete config.telemetryDisabled; + } + await writeConfig(config); +} + type ResolvedVia = "remote" | "git-common-dir" | "directory"; export async function resolveProfile(cwd: string): Promise< diff --git a/packages/cli-core/src/lib/constants.ts b/packages/cli-core/src/lib/constants.ts index e6dbc1b81..f88439df2 100644 --- a/packages/cli-core/src/lib/constants.ts +++ b/packages/cli-core/src/lib/constants.ts @@ -49,3 +49,9 @@ export const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour export const UPDATE_PACKAGE_NAME = "clerk"; export const UPDATE_CACHE_FILE = join(CLERK_CACHE_DIR, "update-check.json"); export const NPM_REGISTRY_URL = "https://registry.npmjs.org/"; + +// ── Telemetry ───────────────────────────────────────────────────────────── + +/** Event ingestion endpoint (telemetry-service worker → BigQuery). */ +export const DEFAULT_TELEMETRY_ENDPOINT = "https://clerk-telemetry.com/v1/event"; +export const TELEMETRY_TIMEOUT_MS = 1000; diff --git a/packages/cli-core/src/lib/env-signals.test.ts b/packages/cli-core/src/lib/env-signals.test.ts new file mode 100644 index 000000000..0f7b94d1b --- /dev/null +++ b/packages/cli-core/src/lib/env-signals.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "bun:test"; +import { + detectAiAgent, + detectInScreen, + detectInstallMethod, + detectInTmux, + detectTerminalProgram, +} from "./env-signals.ts"; + +describe("detectAiAgent", () => { + test.each([ + [{ ANTIGRAVITY_CLI_ALIAS: "1" }, "antigravity"], + [{ CLAUDECODE: "1" }, "claude_code"], + [{ CLINE_ACTIVE: "true" }, "cline"], + [{ CODEX_SANDBOX: "1" }, "codex_cli"], + [{ CODEX_THREAD_ID: "abc" }, "codex_cli"], + [{ CODEX_SANDBOX_NETWORK_DISABLED: "1" }, "codex_cli"], + [{ CODEX_CI: "1" }, "codex_cli"], + [{ CURSOR_AGENT: "1" }, "cursor"], + [{ GEMINI_CLI: "1" }, "gemini_cli"], + [{ OPENCODE: "1" }, "open_code"], + [{ OPENCLAW_SHELL: "1" }, "openclaw"], + ])("detects %o as %s", (env, expected) => { + expect(detectAiAgent(env)).toBe(expected); + }); + + test("returns empty string when nothing is set", () => { + expect(detectAiAgent({})).toBe(""); + }); + + test("ignores empty-string values", () => { + expect(detectAiAgent({ CLAUDECODE: "" })).toBe(""); + }); +}); + +describe("detectTerminalProgram", () => { + test("LC_TERMINAL wins and is slugged", () => { + expect(detectTerminalProgram({ LC_TERMINAL: "iTerm2", TERM_PROGRAM: "Apple_Terminal" })).toBe( + "iterm2", + ); + }); + + test("fallback values are slugged and capped, never verbatim", () => { + expect(detectTerminalProgram({ TERM_PROGRAM: "Apple_Terminal" })).toBe("apple_terminal"); + expect(detectTerminalProgram({ TERM_PROGRAM: "iTerm.app" })).toBe("iterm.app"); + expect(detectTerminalProgram({ TERM_PROGRAM: "/Users/x/secret stuff!" })).toBe( + "usersxsecretstuff", + ); + expect(detectTerminalProgram({ LC_TERMINAL: "x".repeat(100) })).toHaveLength(32); + }); + + test.each([ + [{ WARP_CLIENT_VERSION: "1" }, "warp"], + [{ WT_SESSION: "guid" }, "windows_terminal"], + [{ KITTY_WINDOW_ID: "1" }, "kitty"], + [{ ALACRITTY_WINDOW_ID: "1" }, "alacritty"], + [{ ALACRITTY_LOG: "/tmp/x" }, "alacritty"], + [{ WEZTERM_EXECUTABLE: "/bin/wezterm" }, "wezterm"], + [{ WEZTERM_PANE: "0" }, "wezterm"], + [{ GHOSTTY_RESOURCES_DIR: "/x" }, "ghostty"], + ])("detects %o as %s", (env, expected) => { + expect(detectTerminalProgram(env)).toBe(expected); + }); + + test("falls back to slugged TERM_PROGRAM, then empty string", () => { + expect(detectTerminalProgram({ TERM_PROGRAM: "vscode" })).toBe("vscode"); + expect(detectTerminalProgram({})).toBe(""); + }); +}); + +describe("detectInstallMethod", () => { + const noEnv = {}; + + test("CLERK_INSTALL_METHOD override wins for known values only", () => { + expect(detectInstallMethod({ CLERK_INSTALL_METHOD: "homebrew" }, "/anything")).toBe("homebrew"); + }); + + test("unknown CLERK_INSTALL_METHOD values are ignored, not forwarded", () => { + expect(detectInstallMethod({ CLERK_INSTALL_METHOD: "/tmp/evil" }, "/usr/local/bin/clerk")).toBe( + "unknown", + ); + // Falls through to normal detection instead of trusting the override. + expect( + detectInstallMethod( + { CLERK_INSTALL_METHOD: "scoop" }, + "/opt/homebrew/Cellar/clerk/bin/clerk", + ), + ).toBe("homebrew"); + }); + + test.each([ + ["/opt/homebrew/Cellar/clerk/1.0/bin/clerk", "homebrew"], + ["/home/linuxbrew/.linuxbrew/bin/clerk", "homebrew"], + ["/Users/x/.npm/_npx/abc123/node_modules/@clerk/cli-darwin-arm64/bin/clerk", "npx"], + ["/private/tmp/bunx-501-clerk@latest/node_modules/.bin/clerk", "bunx"], + ])("classifies execPath %s as %s", (execPath, expected) => { + expect(detectInstallMethod(noEnv, execPath)).toBe(expected); + }); + + test("windows-style homebrew-less path with backslashes and node_modules is npm_global", () => { + expect( + detectInstallMethod( + noEnv, + "C:\\Users\\x\\AppData\\Roaming\\npm\\node_modules\\@clerk\\cli-win32-x64\\bin\\clerk.exe", + ), + ).toBe("npm_global"); + }); + + test("npm_lifecycle_event means a package script", () => { + expect( + detectInstallMethod({ npm_lifecycle_event: "dev" }, "/repo/node_modules/.bin/clerk"), + ).toBe("npm_run"); + }); + + test("npm_command=exec means npx", () => { + expect(detectInstallMethod({ npm_command: "exec" }, "/somewhere/clerk")).toBe("npx"); + }); + + test("bun user agent without lifecycle event means bunx", () => { + expect( + detectInstallMethod( + { npm_config_user_agent: "bun/1.3.0 npm/? node/v24" }, + "/somewhere/clerk", + ), + ).toBe("bunx"); + }); + + test("bare node_modules path means npm_global", () => { + expect( + detectInstallMethod(noEnv, "/usr/local/lib/node_modules/@clerk/cli-linux-x64/bin/clerk"), + ).toBe("npm_global"); + }); + + test("anything else is unknown", () => { + expect(detectInstallMethod(noEnv, "/usr/local/bin/clerk")).toBe("unknown"); + }); +}); + +describe("tmux / screen", () => { + test("detects tmux via TMUX and screen via STY", () => { + expect(detectInTmux({ TMUX: "/tmp/tmux-1000/default" })).toBe(true); + expect(detectInTmux({})).toBe(false); + expect(detectInScreen({ STY: "1234.pts-0" })).toBe(true); + expect(detectInScreen({})).toBe(false); + }); +}); diff --git a/packages/cli-core/src/lib/env-signals.ts b/packages/cli-core/src/lib/env-signals.ts new file mode 100644 index 000000000..ae4c4d399 --- /dev/null +++ b/packages/cli-core/src/lib/env-signals.ts @@ -0,0 +1,111 @@ +/** + * Environment signals for CLI telemetry: which AI agent, terminal, and + * install method a run came from. The returned strings are analytics keys — + * renaming one breaks downstream queries. + * + * All functions take an injected env so tests never depend on the ambient + * environment (the dev machine may itself run inside an AI agent or tmux). + */ + +export type EnvLike = Record; + +// A privacy control must fail toward "off": any non-empty value counts as an +// opt-out unless it is explicitly "0" or "false" (case-insensitive). +const isOptOutEnv = (value?: string): boolean => { + if (!value) return false; + const normalized = value.toLowerCase(); + return normalized !== "0" && normalized !== "false"; +}; + +const OPT_OUT_ENV_VARS = ["CLERK_TELEMETRY_DISABLED", "DO_NOT_TRACK"] as const; + +export type OptOutEnvVar = (typeof OPT_OUT_ENV_VARS)[number]; + +export function optOutEnvVar(env: EnvLike): OptOutEnvVar | null { + for (const envVar of OPT_OUT_ENV_VARS) { + if (isOptOutEnv(env[envVar])) { + return envVar; + } + } + + return null; +} + +// Truthiness (not equality) is deliberate: harnesses use different marker +// values — gemini/opencode set "1", cline sets "true", openclaw sets a mode +// string like "tui-local". +export function detectAiAgent(env: EnvLike): string { + if (env.ANTIGRAVITY_CLI_ALIAS) return "antigravity"; + if (env.CLAUDECODE) return "claude_code"; + if (env.CLINE_ACTIVE) return "cline"; + if ( + env.CODEX_SANDBOX || + env.CODEX_THREAD_ID || + env.CODEX_SANDBOX_NETWORK_DISABLED || + env.CODEX_CI + ) { + return "codex_cli"; + } + if (env.CURSOR_AGENT) return "cursor"; + if (env.GEMINI_CLI) return "gemini_cli"; + if (env.OPENCODE) return "open_code"; + if (env.OPENCLAW_SHELL) return "openclaw"; + return ""; +} + +// LC_TERMINAL / TERM_PROGRAM carry arbitrary text; every other signal here is +// a closed enum. Slug + cap them so no unbounded string reaches the payload. +const TERMINAL_SLUG_MAX = 32; +function slugTerminal(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9._-]/g, "") + .slice(0, TERMINAL_SLUG_MAX); +} + +export function detectTerminalProgram(env: EnvLike): string { + if (env.LC_TERMINAL) return slugTerminal(env.LC_TERMINAL); + if (env.WARP_CLIENT_VERSION) return "warp"; + if (env.WT_SESSION) return "windows_terminal"; + if (env.KITTY_WINDOW_ID) return "kitty"; + if (env.ALACRITTY_WINDOW_ID || env.ALACRITTY_LOG) return "alacritty"; + if (env.WEZTERM_EXECUTABLE || env.WEZTERM_PANE) return "wezterm"; + if (env.GHOSTTY_RESOURCES_DIR) return "ghostty"; + return env.TERM_PROGRAM ? slugTerminal(env.TERM_PROGRAM) : ""; +} + +/** + * How the CLI binary was installed/invoked. The npm wrapper and package + * runners leave `npm_*` vars in the child env; direct binary installs are + * classified by executable path. + */ +const INSTALL_METHODS = new Set(["npm_global", "npm_run", "npx", "bunx", "homebrew"]); + +export function detectInstallMethod(env: EnvLike, execPath: string): string { + // The override is only honored for known values — it must not become an + // arbitrary-string channel into the payload. + if (env.CLERK_INSTALL_METHOD && INSTALL_METHODS.has(env.CLERK_INSTALL_METHOD)) { + return env.CLERK_INSTALL_METHOD; + } + + const path = execPath.toLowerCase().replaceAll("\\", "/"); + if (path.includes("/cellar/") || path.includes("/homebrew/") || path.includes("/linuxbrew/")) { + return "homebrew"; + } + if (path.includes("/_npx/")) return "npx"; + if (path.includes("bunx-")) return "bunx"; + + if (env.npm_lifecycle_event) return "npm_run"; + if (env.npm_command === "exec") return "npx"; + if (env.npm_config_user_agent?.startsWith("bun")) return "bunx"; + if (path.includes("/node_modules/")) return "npm_global"; + return "unknown"; +} + +export function detectInTmux(env: EnvLike): boolean { + return Boolean(env.TMUX); +} + +export function detectInScreen(env: EnvLike): boolean { + return Boolean(env.STY); +} diff --git a/packages/cli-core/src/lib/fetch.test.ts b/packages/cli-core/src/lib/fetch.test.ts index 39f031889..d967fc95f 100644 --- a/packages/cli-core/src/lib/fetch.test.ts +++ b/packages/cli-core/src/lib/fetch.test.ts @@ -1,5 +1,9 @@ -import { test, expect, describe, afterEach, mock } from "bun:test"; -import { loggedFetch } from "./fetch.ts"; +import { test, expect, describe, afterEach, beforeEach, mock } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { _resetUserAgentCache, loggedFetch } from "./fetch.ts"; +import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; const originalFetch = globalThis.fetch; @@ -42,3 +46,69 @@ describe("loggedFetch", () => { expect(init.headers.get("User-Agent")).toMatch(/^Clerk-CLI\//); }); }); + +describe("AIAgent segment honors the telemetry opt-out", () => { + let configDir: string; + let originalClaudecode: string | undefined; + let originalCi: string | undefined; + + async function sentUserAgent(): Promise { + globalThis.fetch = mock( + async () => new Response("ok", { status: 200 }), + ) as unknown as typeof fetch; + await loggedFetch("https://example.test/x", { tag: "test" }); + const [, init] = (globalThis.fetch as unknown as ReturnType).mock.calls[0]!; + return init.headers.get("User-Agent")!; + } + + beforeEach(async () => { + configDir = await mkdtemp(join(tmpdir(), "clerk-fetch-test-")); + _setConfigDir(configDir); + _resetUserAgentCache(); + originalClaudecode = process.env.CLAUDECODE; + originalCi = process.env.CI; + process.env.CLAUDECODE = "1"; + delete process.env.CI; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + }); + + afterEach(async () => { + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + _resetUserAgentCache(); + if (originalClaudecode === undefined) delete process.env.CLAUDECODE; + else process.env.CLAUDECODE = originalClaudecode; + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + await rm(configDir, { recursive: true, force: true }); + }); + + test("segment present once the disclosure notice has been shown", async () => { + await markTelemetryNoticeShown(); + expect(await sentUserAgent()).toContain("AIAgent/claude_code"); + }); + + test("segment omitted before the disclosure notice has been shown", async () => { + expect(await sentUserAgent()).not.toContain("AIAgent/"); + }); + + test("segment present in CI even before the notice (CI is exempt from the grace)", async () => { + process.env.CI = "1"; + expect(await sentUserAgent()).toContain("AIAgent/claude_code"); + }); + + test("segment omitted under an env opt-out", async () => { + await markTelemetryNoticeShown(); + process.env.DO_NOT_TRACK = "yes"; + expect(await sentUserAgent()).not.toContain("AIAgent/"); + }); + + test("segment omitted after clerk telemetry disable", async () => { + await markTelemetryNoticeShown(); + await setTelemetryDisabled(true); + expect(await sentUserAgent()).not.toContain("AIAgent/"); + }); +}); diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index ffebe505c..ceb686f15 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -11,10 +11,34 @@ import { log } from "./log.ts"; import { withNetworkAccess } from "./host-execution.ts"; import { buildUserAgent } from "./user-agent.ts"; +import { optOutEnvVar } from "./env-signals.ts"; +import { getTelemetryDisabled, getTelemetryNoticeShown } from "./config.ts"; -const USER_AGENT = buildUserAgent(); +let userAgentPromise: Promise | undefined; -export type LoggedFetchInit = RequestInit & { tag: string }; +/** Test-only: recompute the User-Agent on the next request. */ +export function _resetUserAgentCache(): void { + userAgentPromise = undefined; +} + +function resolveUserAgent(): Promise { + // The AIAgent segment exists purely for analytics classification, so it + // honors the telemetry opt-outs (env vars and `clerk telemetry disable`) + // and, like the events, stays absent until the disclosure notice has been + // shown — CI exempt, matching maybeShowTelemetryNotice. + userAgentPromise ??= (async () => { + const beforeDisclosure = + !process.env.CI && !(await getTelemetryNoticeShown().catch(() => false)); + const omitAgentSegment = + optOutEnvVar(process.env) !== null || + beforeDisclosure || + (await getTelemetryDisabled().catch(() => true)); + return buildUserAgent(process.env, { agentToken: !omitAgentSegment }); + })(); + return userAgentPromise; +} + +export type LoggedFetchInit = RequestInit & { tag: string; bestEffort?: boolean }; /** * Normalized response shape returned by the higher-level API request wrappers @@ -29,14 +53,14 @@ export interface ApiResponse { } export async function loggedFetch(url: URL | string, options: LoggedFetchInit): Promise { - const { tag, ...init } = options; + const { tag, bestEffort, ...init } = options; const method = init.method ?? "GET"; const urlStr = url.toString(); const headers = new Headers(init.headers); - if (!headers.has("user-agent")) headers.set("User-Agent", USER_AGENT); + if (!headers.has("user-agent")) headers.set("User-Agent", await resolveUserAgent()); log.debug(`${tag}: ${method} ${urlStr}`); const response = await withNetworkAccess( - { operation: "connect", target: urlStr, label: tag }, + { operation: "connect", target: urlStr, label: tag, bestEffort }, async () => fetch(url, { ...init, headers }), ); if (!response.ok) { diff --git a/packages/cli-core/src/lib/host-execution.ts b/packages/cli-core/src/lib/host-execution.ts index bdd7280f6..1c03261c0 100644 --- a/packages/cli-core/src/lib/host-execution.ts +++ b/packages/cli-core/src/lib/host-execution.ts @@ -19,6 +19,8 @@ export interface HostCapabilityDetails { operation?: HostOperation; target?: string; label?: string; + /** Optional call (e.g. telemetry): its failure never surfaces capability warnings. */ + bestEffort?: boolean; } export interface HostStateProbeFailure { @@ -165,7 +167,9 @@ export async function withHostCapability( try { return await fn(); } catch (error) { - observeHostCapabilityFailure(capability, error, details); + if (!details.bestEffort) { + observeHostCapabilityFailure(capability, error, details); + } throw error; } } diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts new file mode 100644 index 000000000..d28ae48e3 --- /dev/null +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -0,0 +1,348 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DEV_CLI_VERSION } from "./version.ts"; +import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import { + finalizeAndSendTelemetry, + getTelemetryStatus, + startCommandTelemetry, + telemetryEnabled, + telemetryResultForError, + type TelemetryCommand, +} from "./telemetry.ts"; +import { ApiError, CliError, EXIT_CODE, UserAbortError } from "./errors.ts"; +import { setLogLevel } from "./log.ts"; +import { useCaptureLog } from "../test/lib/stubs.ts"; + +// Isolate config I/O (machine uuid, notice flag) from the real user config dir. +let configDir: string; +beforeEach(async () => { + configDir = await mkdtemp(join(tmpdir(), "clerk-telemetry-test-")); + _setConfigDir(configDir); +}); +afterEach(async () => { + _setConfigDir(undefined); + await rm(configDir, { recursive: true, force: true }); +}); + +describe("telemetryEnabled", () => { + const REAL = "1.2.3"; + + test("enabled for release builds by default", () => { + expect(telemetryEnabled({}, REAL)).toBe(true); + }); + + // Any non-empty value except an explicit "0"/"false" opts out. + test.each([ + [{ CLERK_TELEMETRY_DISABLED: "1" }], + [{ CLERK_TELEMETRY_DISABLED: "true" }], + [{ CLERK_TELEMETRY_DISABLED: "yes" }], + [{ CLERK_TELEMETRY_DISABLED: "anything" }], + [{ DO_NOT_TRACK: "1" }], + [{ DO_NOT_TRACK: "TRUE" }], + [{ DO_NOT_TRACK: "on" }], + ])("opt-out env %o disables", (env) => { + expect(telemetryEnabled(env, REAL)).toBe(false); + }); + + test.each([ + [{ CLERK_TELEMETRY_DISABLED: "0" }], + [{ CLERK_TELEMETRY_DISABLED: "false" }], + [{ CLERK_TELEMETRY_DISABLED: "FALSE" }], + [{ CLERK_TELEMETRY_DISABLED: "" }], + [{ DO_NOT_TRACK: "0" }], + [{ DO_NOT_TRACK: "false" }], + ])("explicit-false env %o stays enabled", (env) => { + expect(telemetryEnabled(env, REAL)).toBe(true); + }); + + test("dev builds are disabled unless CLERK_TELEMETRY_URL is set", () => { + expect(telemetryEnabled({}, DEV_CLI_VERSION)).toBe(false); + expect(telemetryEnabled({ CLERK_TELEMETRY_URL: "http://localhost:9" }, DEV_CLI_VERSION)).toBe( + true, + ); + }); + + test("opt-out beats the URL escape hatch", () => { + expect( + telemetryEnabled({ CLERK_TELEMETRY_URL: "http://localhost:9", DO_NOT_TRACK: "1" }, REAL), + ).toBe(false); + }); +}); + +describe("getTelemetryStatus", () => { + const REAL = "1.2.3"; + + test("reports env opt-out first, naming the winning variable", async () => { + expect( + await getTelemetryStatus({ CLERK_TELEMETRY_DISABLED: "1", DO_NOT_TRACK: "1" }, REAL), + ).toEqual({ enabled: false, reason: "env", envVar: "CLERK_TELEMETRY_DISABLED" }); + expect(await getTelemetryStatus({ DO_NOT_TRACK: "yes" }, REAL)).toEqual({ + enabled: false, + reason: "env", + envVar: "DO_NOT_TRACK", + }); + }); + + test("reports the persisted config opt-out", async () => { + await setTelemetryDisabled(true); + expect(await getTelemetryStatus({}, REAL)).toEqual({ enabled: false, reason: "config" }); + }); + + test("persisted opt-out beats the URL escape hatch", async () => { + await setTelemetryDisabled(true); + const status = await getTelemetryStatus({ CLERK_TELEMETRY_URL: "http://localhost:9" }, REAL); + expect(status.enabled).toBe(false); + }); + + test("reports dev builds, and enabled otherwise", async () => { + expect(await getTelemetryStatus({}, DEV_CLI_VERSION)).toEqual({ + enabled: false, + reason: "dev-build", + }); + expect(await getTelemetryStatus({}, REAL)).toEqual({ enabled: true }); + }); +}); + +describe("telemetryResultForError", () => { + test("maps user aborts", () => { + expect(telemetryResultForError(new UserAbortError())).toEqual({ + outcome: "abort", + exitCode: EXIT_CODE.SUCCESS, + }); + }); + + test("maps CliError with code and exit code", () => { + const error = new CliError("nope", { code: "not_linked" }); + expect(telemetryResultForError(error)).toEqual({ + outcome: "error", + exitCode: error.exitCode, + errorCode: "not_linked", + }); + }); + + test("maps CliError without code", () => { + expect(telemetryResultForError(new CliError("nope")).errorCode).toBe("cli_error"); + }); + + test("maps ApiError (code is null for a non-JSON body → api_error fallback)", () => { + const error = new ApiError(500, "boom"); + expect(telemetryResultForError(error)).toEqual({ + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + errorCode: "api_error", + }); + }); + + test("maps unknown errors", () => { + expect(telemetryResultForError(new Error("x"))).toEqual({ + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + errorCode: "unexpected_error", + }); + }); +}); + +describe("finalizeAndSendTelemetry", () => { + const originalFetch = globalThis.fetch; + + // Isolate from ambient env: clear before each test too (not just after) so a + // pre-set CLERK_TELEMETRY_DISABLED/DO_NOT_TRACK/CLERK_TELEMETRY_URL in the + // shell can't change whether telemetry is enabled for the first test. + beforeEach(() => { + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + }); + afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + }); + + function fakeCommand(): TelemetryCommand { + return { name: () => "list", options: [], getOptionValueSource: () => undefined, parent: null }; + } + + test("no-op when telemetry is disabled (no fetch, no throw)", async () => { + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + // dev version + no CLERK_TELEMETRY_URL → disabled + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(0); + }); + + test("swallows network failures", async () => { + await markTelemetryNoticeShown(); // past the grace run — reach the send path + process.env.CLERK_TELEMETRY_URL = "https://unreachable.invalid/v1/event"; + globalThis.fetch = (async () => { + throw new Error("network down"); + }) as unknown as typeof fetch; + startCommandTelemetry(fakeCommand()); + // Must resolve, not reject. + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + }); + + test("the deadline bounds the whole telemetry job, not just the fetch", async () => { + await markTelemetryNoticeShown(); + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + globalThis.fetch = (() => new Promise(() => {})) as unknown as typeof fetch; // hangs forever + startCommandTelemetry(fakeCommand()); + const started = Date.now(); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }, 100); + expect(Date.now() - started).toBeLessThan(500); + }); + + test("no send when telemetry is disabled via persisted config", async () => { + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + await setTelemetryDisabled(true); + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(0); + }); + + describe("sandbox-looking failures", () => { + const captured = useCaptureLog(); + + test("permission-shaped telemetry failures never trigger the sandbox warning", async () => { + await markTelemetryNoticeShown(); + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + globalThis.fetch = (async () => { + throw new Error("EPERM: operation not permitted"); + }) as unknown as typeof fetch; + // Agent mode (no TTY in tests) — the mode where the sandbox hint fires. + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(captured.err).not.toContain("Host-only"); + }); + }); + + describe("verbose payload dump", () => { + const captured = useCaptureLog(); + + test("dumps the event payload at debug level before the POST", async () => { + await markTelemetryNoticeShown(); + globalThis.fetch = (async () => new Response("{}")) as unknown as typeof fetch; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + setLogLevel("debug"); + try { + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + } finally { + setLogLevel("info"); + } + expect(captured.err).toContain("telemetry: event {"); + expect(captured.err).toContain('"machine_uuid"'); + }); + }); + + describe("first-run notice (agent mode, non-CI)", () => { + const captured = useCaptureLog(); + let originalCi: string | undefined; + + beforeEach(() => { + originalCi = process.env.CI; + delete process.env.CI; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + }); + afterEach(() => { + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + }); + + test("agents get the notice and the no-send grace run too", async () => { + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + + // No CLERK_MODE and no TTY in tests → agent mode. + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(0); + expect(captured.err).toContain("Nothing has been sent during this run"); + + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(1); + }); + + test("CI sends from the first run with no notice", async () => { + process.env.CI = "true"; + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(1); + expect(captured.err).not.toContain("usage telemetry"); + }); + }); + + describe("first-run notice (human, non-CI)", () => { + const captured = useCaptureLog(); + let originalCi: string | undefined; + let originalMode: string | undefined; + + beforeEach(() => { + originalCi = process.env.CI; + originalMode = process.env.CLERK_MODE; + delete process.env.CI; + process.env.CLERK_MODE = "human"; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + }); + afterEach(() => { + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + }); + + test("the run that shows the notice sends nothing; the next run sends", async () => { + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(0); + expect(captured.err).toContain("Nothing has been sent during this run"); + + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(1); + }); + + test("no notice and no skip once it has already been shown", async () => { + await markTelemetryNoticeShown(); + let called = 0; + globalThis.fetch = (async () => { + called += 1; + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeCommand()); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(called).toBe(1); + expect(captured.err).not.toContain("usage telemetry"); + }); + }); +}); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts new file mode 100644 index 000000000..27b6d51f5 --- /dev/null +++ b/packages/cli-core/src/lib/telemetry.ts @@ -0,0 +1,250 @@ +/** + * Per-invocation usage telemetry. + * + * One CLI_COMMAND_EXECUTED event per command run, POSTed to the + * telemetry-service worker (BigQuery behind it). Opt out with + * `clerk telemetry disable` (persisted) or the CLERK_TELEMETRY_DISABLED / + * DO_NOT_TRACK env vars. Dev builds send nothing unless CLERK_TELEMETRY_URL + * overrides the endpoint (test escape hatch). + * + * Telemetry must never affect the command: every entry point swallows its + * own failures to log.debug and the send is capped at TELEMETRY_TIMEOUT_MS. + */ + +import { DEFAULT_TELEMETRY_ENDPOINT, TELEMETRY_TIMEOUT_MS } from "./constants.ts"; +import { + ensureMachineUuid, + getTelemetryDisabled, + markTelemetryNoticeShown, + resolveProfile, +} from "./config.ts"; +import { + detectAiAgent, + detectInScreen, + detectInstallMethod, + detectInTmux, + detectTerminalProgram, + optOutEnvVar, + type EnvLike, + type OptOutEnvVar, +} from "./env-signals.ts"; +import { getCurrentEnvName } from "./environment.ts"; +import { ApiError, CliError, EXIT_CODE, UserAbortError, isPromptExitError } from "./errors.ts"; +import { loggedFetch } from "./fetch.ts"; +import { log } from "./log.ts"; +import { getMode } from "../mode.ts"; +import { DEV_CLI_VERSION, resolveCliVersion } from "./version.ts"; + +export type TelemetryResult = { + outcome: "success" | "error" | "abort"; + exitCode: number; + errorCode?: string; +}; + +/** Structural slice of Commander's Command — avoids its generic types. */ +export type TelemetryCommand = { + name(): string; + options: readonly { name(): string; attributeName(): string }[]; + getOptionValueSource(key: string): string | undefined; + parent: TelemetryCommand | null; +}; + +type TelemetryContext = { + command: string; + flags: string; + startedAt: number; +}; + +let context: TelemetryContext | null = null; + +/** Pure env + version check; the persisted opt-out lives in getTelemetryStatus. */ +export function telemetryEnabled( + env: EnvLike = process.env, + version: string = resolveCliVersion() ?? DEV_CLI_VERSION, +): boolean { + if (optOutEnvVar(env)) return false; + if (env.CLERK_TELEMETRY_URL) return true; + return version !== DEV_CLI_VERSION; +} + +export type TelemetryStatus = + | { enabled: true } + | { enabled: false; reason: "env"; envVar: OptOutEnvVar } + | { enabled: false; reason: "config" } + | { enabled: false; reason: "dev-build" }; + +/** + * Effective enablement with the winning reason, in precedence order: + * env opt-out > persisted `clerk telemetry disable` > dev-build guard. + */ +export async function getTelemetryStatus( + env: EnvLike = process.env, + version: string = resolveCliVersion() ?? DEV_CLI_VERSION, +): Promise { + const envVar = optOutEnvVar(env); + if (envVar) return { enabled: false, reason: "env", envVar }; + if (await getTelemetryDisabled()) return { enabled: false, reason: "config" }; + if (!telemetryEnabled(env, version)) return { enabled: false, reason: "dev-build" }; + return { enabled: true }; +} + +/** "users list" for `clerk users list` — root name excluded, never raw argv. */ +function commandPathOf(cmd: TelemetryCommand): string { + const parts: string[] = []; + for (let c: TelemetryCommand | null = cmd; c && c.parent; c = c.parent) { + parts.unshift(c.name()); + } + return parts.join(" "); +} + +/** Names of flags explicitly set on the CLI (own + inherited), never values. */ +function collectSetFlagNames(cmd: TelemetryCommand): string[] { + const names: string[] = []; + for (let c: TelemetryCommand | null = cmd; c; c = c.parent) { + for (const option of c.options) { + if (c.getOptionValueSource(option.attributeName()) === "cli") { + names.push(option.name()); + } + } + } + return names; +} + +/** Pure in-memory; never throws. */ +export function startCommandTelemetry(actionCommand: TelemetryCommand): void { + try { + context = { + command: commandPathOf(actionCommand), + flags: collectSetFlagNames(actionCommand).join(","), + startedAt: Date.now(), + }; + } catch (error) { + log.debug(`telemetry: failed to start context: ${error}`); + } +} + +export function telemetryResultForError(error: unknown): TelemetryResult { + if (error instanceof UserAbortError || isPromptExitError(error)) { + return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; + } + if (error instanceof CliError) { + return { outcome: "error", exitCode: error.exitCode, errorCode: error.code ?? "cli_error" }; + } + if (error instanceof ApiError) { + return { outcome: "error", exitCode: EXIT_CODE.GENERAL, errorCode: error.code ?? "api_error" }; + } + return { outcome: "error", exitCode: EXIT_CODE.GENERAL, errorCode: "unexpected_error" }; +} + +/** + * Build + send the event, and surface the one-time disclosure notice. + * Awaited by runProgram before process.exit; must never throw or exceed + * `deadlineMs` by more than scheduling noise. The deadline covers the entire + * job — config I/O, git profile lookup, and the POST — not just the fetch; + * on timeout the event is dropped (`deadlineMs` is overridden in tests). + */ +export async function finalizeAndSendTelemetry( + result: TelemetryResult, + deadlineMs: number = TELEMETRY_TIMEOUT_MS, +): Promise { + const current = context; + context = null; + if (!current) return; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), deadlineMs); + try { + const work = buildAndSend(current, result, controller.signal).catch((error: unknown) => { + log.debug(`telemetry: send failed: ${error}`); + }); + await Promise.race([work, abortedToResolved(controller.signal)]); + } finally { + clearTimeout(timer); + } +} + +function abortedToResolved(signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) return resolve(); + signal.addEventListener("abort", () => resolve(), { once: true }); + }); +} + +async function buildAndSend( + current: TelemetryContext, + result: TelemetryResult, + signal: AbortSignal, +): Promise { + // Re-checked here (not just at start) so `clerk telemetry disable` itself + // sees the freshly persisted opt-out and sends nothing. + if (!(await getTelemetryStatus()).enabled) return; + + // The notice tells the user "Nothing has been sent during this run" — honor it. + if (await maybeShowTelemetryNotice()) return; + + const machineUuid = await ensureMachineUuid(); + const resolved = await resolveProfile(process.cwd()).catch(() => undefined); + const version = resolveCliVersion() ?? DEV_CLI_VERSION; + + const event = { + sdk: "clerk-cli", + sdkv: version, + event: "CLI_COMMAND_EXECUTED", + payload: { + command: current.command, + flags: current.flags, + outcome: result.outcome, + exit_code: result.exitCode, + error_code: result.errorCode ?? null, + duration_ms: Date.now() - current.startedAt, + machine_uuid: machineUuid, + install_method: detectInstallMethod(process.env, process.execPath), + ai_agent: detectAiAgent(process.env), + terminal_program: detectTerminalProgram(process.env), + mode: getMode(), + os: process.platform, + arch: process.arch, + ci: Boolean(process.env.CI), + in_tmux: detectInTmux(process.env), + in_screen: detectInScreen(process.env), + env: getCurrentEnvName(), + workspace_id: resolved?.profile.workspaceId ?? null, + app_id: resolved?.profile.appId ?? null, + }, + }; + + log.debug(`telemetry: event ${JSON.stringify(event)}`); + + const url = process.env.CLERK_TELEMETRY_URL ?? DEFAULT_TELEMETRY_ENDPOINT; + await loggedFetch(url, { + tag: "telemetry", + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ events: [event] }), + signal, + bestEffort: true, + }); +} + +/** + * One-time stderr disclosure for humans and agents alike. Returns true when + * the notice was just shown — that run sends nothing, so disclosure always + * precedes a machine's first event. CI is exempt from both the notice and + * the grace run: ephemeral CI machines are always on their "first run", so + * a grace there would mean CI never sends at all. + */ +async function maybeShowTelemetryNotice(): Promise { + if (process.env.CI) return false; + if (!(await markTelemetryNoticeShown())) return false; + log.blank(); + log.info( + "The Clerk CLI collects usage telemetry to help improve the CLI: command name, flag names,", + ); + log.info( + "duration, outcome, a random machine identifier — and your workspace and app IDs when a", + ); + log.info("project is linked. Nothing has been sent during this run."); + log.info("Opt out: `clerk telemetry disable` — details: https://clerk.com/docs/telemetry"); + log.blank(); + return true; +} diff --git a/packages/cli-core/src/lib/user-agent.test.ts b/packages/cli-core/src/lib/user-agent.test.ts index 98d038f0b..63e850760 100644 --- a/packages/cli-core/src/lib/user-agent.test.ts +++ b/packages/cli-core/src/lib/user-agent.test.ts @@ -1,34 +1,45 @@ -import { test, expect, describe, afterEach } from "bun:test"; +import { test, expect, describe } from "bun:test"; import { buildUserAgent } from "./user-agent.ts"; describe("buildUserAgent", () => { - const originalCi = process.env.CI; - afterEach(() => { - if (originalCi === undefined) delete process.env.CI; - else process.env.CI = originalCi; - }); - test("starts with Clerk-CLI/", () => { - expect(buildUserAgent()).toMatch(/^Clerk-CLI\/\S+ /); + expect(buildUserAgent({})).toMatch(/^Clerk-CLI\/\S+ /); }); test("includes Bun/ and platform-arch", () => { - const ua = buildUserAgent(); + const ua = buildUserAgent({}); expect(ua).toContain(`Bun/${Bun.version}`); expect(ua).toContain(`${process.platform}-${process.arch}`); }); test("appends ci segment when CI env is set", () => { - process.env.CI = "1"; - expect(buildUserAgent()).toMatch(/; ci\)$/); + const ua = buildUserAgent({ CI: "1" }); + expect(ua).toMatch(/; ci\)$/); }); test("omits ci segment when CI env is unset", () => { - delete process.env.CI; - expect(buildUserAgent()).not.toMatch(/; ci\)/); + const ua = buildUserAgent({}); + expect(ua).not.toMatch(/; ci\)/); }); test("uses only printable ASCII characters", () => { - expect(buildUserAgent()).toMatch(/^[\x20-\x7e]+$/); + expect(buildUserAgent({})).toMatch(/^[\x20-\x7e]+$/); + }); + + test("appends an AIAgent segment when an agent is detected", () => { + const ua = buildUserAgent({ CLAUDECODE: "1" }); + expect(ua).toMatch(/; AIAgent\/claude_code\)$/); + }); + + test("ci and AIAgent segments compose inside the parens", () => { + expect(buildUserAgent({ CI: "1", CLAUDECODE: "1" })).toMatch(/; ci; AIAgent\/claude_code\)$/); + }); + + test("no AIAgent segment when no agent env is present", () => { + expect(buildUserAgent({})).not.toContain("AIAgent/"); + }); + + test("agentToken: false omits the segment even when an agent is detected", () => { + expect(buildUserAgent({ CLAUDECODE: "1" }, { agentToken: false })).not.toContain("AIAgent/"); }); }); diff --git a/packages/cli-core/src/lib/user-agent.ts b/packages/cli-core/src/lib/user-agent.ts index 5b136fc2d..008b3117b 100644 --- a/packages/cli-core/src/lib/user-agent.ts +++ b/packages/cli-core/src/lib/user-agent.ts @@ -4,17 +4,28 @@ * we fall through to Bun's default `User-Agent: Bun/`, which is * indistinguishable from any other Bun-based client. * - * Format: `Clerk-CLI/ (Bun/; -[; ci])` + * Format: `Clerk-CLI/ (Bun/; -[; ci][; AIAgent/])` * - : darwin | linux | win32 | … (process.platform) * - : arm64 | x64 | … (process.arch) * - `ci` segment is appended when running under a recognized CI environment. + * - `AIAgent/` segment is appended when an AI agent is detected — + * analytics-purposed, so the fetch layer omits it when telemetry is + * opted out. */ +import { detectAiAgent, type EnvLike } from "./env-signals.ts"; import { DEV_CLI_VERSION, resolveCliVersion } from "./version.ts"; -export function buildUserAgent(): string { +export function buildUserAgent( + env: EnvLike = process.env, + options: { agentToken?: boolean } = {}, +): string { const version = resolveCliVersion() ?? DEV_CLI_VERSION; const segments = [`Bun/${Bun.version}`, `${process.platform}-${process.arch}`]; - if (process.env.CI) segments.push("ci"); + if (env.CI) segments.push("ci"); + if (options.agentToken !== false) { + const agent = detectAiAgent(env); + if (agent) segments.push(`AIAgent/${agent}`); + } return `Clerk-CLI/${version} (${segments.join("; ")})`; } diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts new file mode 100644 index 000000000..a2133ee34 --- /dev/null +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -0,0 +1,292 @@ +/** + * Telemetry is exercised via the CLERK_TELEMETRY_URL escape hatch (tests run + * as 0.0.0-dev, where telemetry is otherwise off). The harness mocks all + * fetch, so events are captured from http.requests. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { clerk, http, useIntegrationTestHarness } from "./lib/harness.ts"; +import { useCaptureLog } from "../lib/stubs.ts"; + +useIntegrationTestHarness(); + +const TELEMETRY_URL = "https://test-telemetry.clerk.com/v1/event"; + +// Isolate from ambient env: clear before each test too (not just after) so a +// pre-set CLERK_TELEMETRY_URL/CLERK_TELEMETRY_DISABLED/DO_NOT_TRACK in the +// shell can't change whether telemetry is enabled for the first test. +beforeEach(() => { + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; +}); +afterEach(() => { + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; +}); + +function telemetryEvents() { + const requests = http.requests.filter((r) => r.url.startsWith(TELEMETRY_URL)); + return requests.map((r) => JSON.parse(r.body ?? "{}") as { events: Record[] }); +} + +// The run that first shows the disclosure notice deliberately sends nothing, +// so tests that expect an event pre-mark the notice as already shown. +// Dynamic import per the harness rule: config.ts transitively imports mocked +// modules, so it must load after the harness registers its mocks. +async function markNoticeAlreadyShown() { + const { markTelemetryNoticeShown } = await import("../../lib/config.ts"); + await markTelemetryNoticeShown(); +} + +test("sends one event for a successful command", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + await clerk("completion", "zsh"); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + expect(bodies[0]!.events).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.sdk).toBe("clerk-cli"); + expect(event.event).toBe("CLI_COMMAND_EXECUTED"); + // Command path is subcommand names only — "zsh" is an argument value and + // is deliberately excluded. + expect(event.payload.command).toBe("completion"); + expect(event.payload.outcome).toBe("success"); + expect(event.payload.exit_code).toBe(0); + expect(event.payload.machine_uuid).toMatch(/^[0-9a-f-]{36}$/); + expect(typeof event.payload.duration_ms).toBe("number"); + // Env-dependent fields are strings, values depend on the host machine. + expect(typeof event.payload.ai_agent).toBe("string"); + expect(typeof event.payload.install_method).toBe("string"); + // Never collected: the argument value ("zsh") must not leak into any event + // field — fails if command path or flags ever start carrying argv values. + expect(JSON.stringify(event)).not.toContain("zsh"); +}); + +test("records failures with error code and reuses the machine uuid", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + const first = await clerk("completion", "zsh"); + expect(first.exitCode).toBe(0); + const firstUuid = telemetryEvents()[0]!.events[0]!.payload.machine_uuid; + + // `apps list` fails because its PLAPI route is not mocked (the mock fetch + // throws) — a real error path through runProgram's catch, unlike Commander + // usage errors (invalid .choices() values), which exit inside Commander in + // production and never produce telemetry. + http.mock({ "test-telemetry.clerk.com": {} }); + const result = await clerk.raw("apps", "list"); + expect(result.exitCode).toBe(1); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("apps list"); + expect(event.payload.outcome).toBe("error"); + expect(event.payload.exit_code).toBe(1); + expect(event.payload.error_code).toBe("unexpected_error"); + expect(event.payload.machine_uuid).toBe(firstUuid); +}); + +test("maps a soft failure (process.exitCode set without throwing) to outcome error", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + try { + // Simulates commands that report failure via process.exitCode instead of throwing. + process.exitCode = 1; + await clerk.raw("completion", "zsh"); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.outcome).toBe("error"); + expect(event.payload.exit_code).toBe(1); + } finally { + process.exitCode = undefined; + } +}); + +test("an invalid --mode value still produces an error event", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + const result = await clerk.raw("--mode", "banana", "completion", "zsh"); + expect(result.exitCode).toBe(2); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("completion"); + expect(event.payload.outcome).toBe("error"); + expect(event.payload.exit_code).toBe(2); +}); + +// payload.command comes from the resolved Command objects' registered names, +// never from argv — a typo or pasted secret either becomes an argument (never +// serialized) or dies inside Commander before any hook runs (no event). +test.each([[["users", "sk_test_secret123"]], [["sk_live_pasted"]]])( + "mistyped input %o never reaches an event", + async (argv) => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock(); // any fetch would record and throw + + const result = await clerk.raw(...argv); + expect(result.exitCode).not.toBe(0); + expect(telemetryEvents()).toHaveLength(0); + const allRequestBodies = http.requests.map((r) => r.body ?? "").join("\n"); + expect(allRequestBodies).not.toContain("sk_test_secret123"); + expect(allRequestBodies).not.toContain("sk_live_pasted"); + }, +); + +test("command succeeds even when the telemetry endpoint is down", async () => { + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock(); // no routes: every fetch throws, including the telemetry send + const result = await clerk.raw("completion", "zsh"); + expect(result.exitCode).toBe(0); +}); + +test("no telemetry traffic without CLERK_TELEMETRY_URL (dev build guard)", async () => { + http.mock(); // guard mock: any fetch would throw + await clerk("completion", "zsh"); + expect(http.requests).toHaveLength(0); +}); + +describe("error rendering is not blocked by the telemetry send", () => { + // Own capture buffer so the stub below can inspect stderr mid-run. + const captured = useCaptureLog(); + + test("the error is on stderr before the telemetry POST fires", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + + let stderrWhenTelemetryFired: string | null = null; + http.stub(async (url) => { + if (url.startsWith(TELEMETRY_URL)) { + stderrWhenTelemetryFired = captured.err; + return Response.json({ ok: true }); + } + throw new Error(`Unmocked fetch route: ${url}`); + }); + + const result = await clerk.raw("apps", "list"); + expect(result.exitCode).toBe(1); + expect(stderrWhenTelemetryFired).not.toBeNull(); + expect(stderrWhenTelemetryFired!).toContain("Unmocked fetch route"); + }); +}); + +test("the first human run shows the notice and sends nothing; the next run sends", async () => { + const originalCi = process.env.CI; + delete process.env.CI; // the notice is suppressed in CI environments + try { + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + const first = await clerk("completion", "zsh"); + expect(first.stderr).toContain("Nothing has been sent during this run"); + expect(first.stderr).toContain("clerk telemetry disable"); + expect(telemetryEvents()).toHaveLength(0); + + const second = await clerk("completion", "zsh"); + expect(second.stderr).not.toContain("Nothing has been sent during this run"); + expect(telemetryEvents()).toHaveLength(1); + } finally { + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + } +}); + +test("agent-mode first run also gets the notice and sends nothing", async () => { + const originalCi = process.env.CI; + delete process.env.CI; + try { + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + const first = await clerk("--mode", "agent", "completion", "zsh"); + expect(first.stderr).toContain("Nothing has been sent during this run"); + expect(telemetryEvents()).toHaveLength(0); + + const second = await clerk("--mode", "agent", "completion", "zsh"); + expect(second.stderr).not.toContain("Nothing has been sent during this run"); + expect(telemetryEvents()).toHaveLength(1); + } finally { + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + // --mode agent sets the mocked mode module's state, which the harness does + // not reset between tests in the same file — restore it. + const { setMode } = await import("../../mode.ts"); + setMode("human"); + } +}); + +test("`clerk telemetry disable` itself sends nothing, and the opt-out persists", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock(); // any fetch would record and throw — none may happen + + await clerk("telemetry", "disable"); + await clerk("completion", "zsh"); + expect(http.requests).toHaveLength(0); +}); + +test("`clerk telemetry enable` turns events back on", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + await clerk("telemetry", "disable"); // sends nothing: opt-out visible at finalize + await clerk("telemetry", "enable"); // sends: the user just opted back in + await clerk("completion", "zsh"); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(2); + expect(bodies[0]!.events[0]!.payload.command).toBe("telemetry enable"); + expect(bodies[1]!.events[0]!.payload.command).toBe("completion"); +}); + +test("`clerk telemetry status` reports the state and the winning reason", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; // lifts the dev-build guard + http.mock({ "test-telemetry.clerk.com": {} }); + + const enabled = await clerk("telemetry", "status"); + expect(enabled.stdout.trim()).toBe("enabled"); + + // Broadened env parsing honored end-to-end: "yes" opts out. + process.env.CLERK_TELEMETRY_DISABLED = "yes"; + const disabledByEnv = await clerk("telemetry", "status"); + expect(disabledByEnv.stdout.trim()).toBe("disabled"); + expect(disabledByEnv.stderr).toContain("CLERK_TELEMETRY_DISABLED"); + delete process.env.CLERK_TELEMETRY_DISABLED; + + await clerk("telemetry", "disable"); + const disabledByConfig = await clerk("telemetry", "status"); + expect(disabledByConfig.stdout.trim()).toBe("disabled"); + expect(disabledByConfig.stderr).toContain("clerk telemetry enable"); +}); + +test("`clerk telemetry enable` warns when the dev-build guard keeps telemetry off", async () => { + http.mock(); // dev build without the URL escape hatch: no network at all + const result = await clerk("telemetry", "enable"); + expect(result.stderr).toContain("Telemetry enabled"); + expect(result.stderr).toContain("dev build"); +}); + +test("`clerk telemetry status` explains the dev-build guard", async () => { + http.mock(); // dev build without the URL escape hatch: no network at all + const result = await clerk("telemetry", "status"); + expect(result.stdout.trim()).toBe("disabled"); + expect(result.stderr).toContain("dev build"); +}); diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index 70598ff99..9c27eccdd 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -158,6 +158,11 @@ export const configStubs = { resolveAppContext: async () => ({ appId: "", appLabel: "", instanceId: "", instanceLabel: "" }), profileLabel: (profile: { appName?: string; appId: string }) => profile.appName ? `${profile.appName} (${profile.appId})` : profile.appId, + ensureMachineUuid: async () => "00000000-0000-4000-8000-000000000000", + markTelemetryNoticeShown: async () => false, + getTelemetryNoticeShown: async () => true, + getTelemetryDisabled: async () => false, + setTelemetryDisabled: noop, }; // Same wholesale-replacement rule as configStubs: this must cover every