From 762489700460dc97af7c1e9d2577eafa63b32e0e Mon Sep 17 00:00:00 2001 From: Durga Date: Wed, 22 Jul 2026 13:29:09 -0400 Subject: [PATCH] Remove Verdict and experimentation support from shopify-dev-tools Assisted-By: devx/30621b40-06c0-4a5e-839a-64b122f7967d --- package.json | 5 - packages/shopify-dev-tools/CHANGELOG.md | 16 -- packages/shopify-dev-tools/package.json | 5 - .../scripts/generate-agent-skills.ts | 6 +- .../src/experiments/experiments.test.ts | 213 ------------------ .../src/experiments/experiments.ts | 129 ----------- .../src/experiments/index.ts | 20 -- .../src/experiments/install-id.test.ts | 136 ----------- .../src/experiments/install-id.ts | 109 --------- .../src/experiments/monorail-logger.test.ts | 95 -------- .../src/experiments/monorail-logger.ts | 75 ------ .../src/experiments/opt-out.test.ts | 74 ------ .../src/experiments/opt-out.ts | 52 ----- .../src/instructions/.gitignore | 4 - .../shopify-dev-tools/src/types/verdict.d.ts | 34 --- .../shopify-dev-tools/test-stubs/verdict.ts | 34 --- packages/shopify-dev-tools/vite.config.js | 7 - pnpm-lock.yaml | 1 - 18 files changed, 3 insertions(+), 1012 deletions(-) delete mode 100644 packages/shopify-dev-tools/src/experiments/experiments.test.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/experiments.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/index.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/install-id.test.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/install-id.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/monorail-logger.test.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/monorail-logger.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/opt-out.test.ts delete mode 100644 packages/shopify-dev-tools/src/experiments/opt-out.ts delete mode 100644 packages/shopify-dev-tools/src/instructions/.gitignore delete mode 100644 packages/shopify-dev-tools/src/types/verdict.d.ts delete mode 100644 packages/shopify-dev-tools/test-stubs/verdict.ts diff --git a/package.json b/package.json index 12b09f44b27..1318aa713ba 100644 --- a/package.json +++ b/package.json @@ -109,9 +109,6 @@ "use-resize-observer>react-dom": "19.2.4" } }, - "overrides": { - "@shopify/shopify-dev-tools>@shopify/verdict": "-" - }, "onlyBuiltDependencies": [ "@parcel/watcher", "esbuild", @@ -397,7 +394,6 @@ "project": [ "src/**/*.ts!", "scripts/*.{js,ts}!", - "test-stubs/**/*.ts!", "vite.config.js!" ], "ignore": [ @@ -411,7 +407,6 @@ "@shopify/hydrogen", "@shopify/hydrogen-react", "@shopify/polaris-types", - "@shopify/verdict", "@types/react", "isbot", "preact", diff --git a/packages/shopify-dev-tools/CHANGELOG.md b/packages/shopify-dev-tools/CHANGELOG.md index 4207c33b0f4..94a1fecbec7 100644 --- a/packages/shopify-dev-tools/CHANGELOG.md +++ b/packages/shopify-dev-tools/CHANGELOG.md @@ -67,33 +67,17 @@ ### Minor Changes - 716d22b: `shopify-app-store-review` skill now points the agent at the canonical shopify.dev requirements page (https://shopify.dev/docs/apps/launch/app-store-review/app-store-ai-self-review-requirements) instead of carrying a hand-maintained inline copy, and adds 5.x category-specific requirements. Output format and status taxonomy are unchanged. (Retroactive changeset for #722, which merged without one.) -- b2e0788: Add a shippable experiments helper with explicit stable install-id lifecycle, opt-out short-circuiting, revocation support, and hardened Monorail error reporting. - 08f0921: Add a releasable UCP topic that publishes the standalone `ucp` skill and learn_shopify_api instructions. - 716d22b: Skill validate scripts (`validate_graphql`, `validate_components`, `validate_functions`, `validate_theme`) now emit the same markdown summary the MCP `validate_*_codeblocks` tools return, including artifact ID and revision lines. The id is auto-minted when not supplied and echoed back to the agent, matching the MCP behavior so retries can chain across revisions on either surface. ### Patch Changes - e236bad: fix polaris skills to use camelCase not kebab-case -- d2545fd: Support eval-time instruction experiments. Drop a sibling - `.experiment.md` next to any `src/instructions/.md` and run - `pnpm --filter @shopify/ai-toolkit-evals run eval:experiment promptfoo/.yaml` - to see baseline and experiment columns side-by-side, plus a `None` - baseline (no instructions, no tools). - - Internals: the api-instructions and agent-skills generators are now - callable as functions with `{ inputDir, outputDir, apiFilter }` (existing - `tsx`-invoked behavior is unchanged). The dev-mcp loader honors a new - `MCP_INSTRUCTIONS_OVERRIDE_DIR` env var that, when set, prepends an - overlay directory to its candidate list — falls back to canonical for - any api the overlay doesn't cover. Has no effect when the env var is - unset. - - c2b8ff2: Update the Shopify CLI guidance topic to prefix `shopify` commands with `SHOPIFY_CLI_AGENT*` environment variables so agent-driven CLI executions can be attributed in Shopify CLI analytics. - f8d1abd: Disclose default-on telemetry more clearly in mirrored plugin install surfaces and generated skill privacy notices, including the opt-out environment variable. Clarify that validation and search scripts report specific request data to `shopify.dev/mcp/usage`. - 40cd7cd: Include the response body in `shopifyDevFetch` errors for 5xx responses, restoring the diagnostic detail previously surfaced by `search_docs.ts` and improving 5xx error messages for the MCP. - 9add652: Quote the packed `SHOPIFY_CLI_AGENT_INFO` and `SHOPIFY_CLI_AGENT_IDS` values in the Shopify CLI topic so exact executed command examples remain valid shell commands. Tighten the matching eval to require both packed env vars, including ordered `s:` / `r:` / `i:` ID segments. - 716d22b: Inject `--target` guidance into SKILL.md for extension-surface skills (Admin Extensions, Checkout Extensions, Customer Account Extensions, POS UI). The validate.mjs invocation example now includes `--target ` plus a one-line note with a representative target value, since validation rejects extension-surface code without one. Non-extension skills are unchanged. -- c976654: Fix Experiments constructor to thread config.env through to install-ID resolution. - 01d0288: Mirror the dual-mode pattern from `validate_graphql.ts` in `validate_functions.ts`: bundled per-skill builds inject a new `__BUNDLED__` esbuild define and resolve schemas via diff --git a/packages/shopify-dev-tools/package.json b/packages/shopify-dev-tools/package.json index 1d4df256de0..d2ec9847c5c 100644 --- a/packages/shopify-dev-tools/package.json +++ b/packages/shopify-dev-tools/package.json @@ -57,10 +57,6 @@ "./config/api-mappings": { "import": "./dist/config/api-mappings.js", "types": "./dist/config/api-mappings.d.ts" - }, - "./experiments": { - "import": "./dist/experiments/index.js", - "types": "./dist/experiments/index.d.ts" } }, "scripts": { @@ -87,7 +83,6 @@ "@shopify/theme-check-common": "3.24.0", "@shopify/theme-check-docs-updater": "3.24.0", "@shopify/theme-check-node": "3.24.0", - "@shopify/verdict": "7.3.0", "@types/react": "catalog:", "graphql": "catalog:", "html-tags": "5.1.0", diff --git a/packages/shopify-dev-tools/scripts/generate-agent-skills.ts b/packages/shopify-dev-tools/scripts/generate-agent-skills.ts index 08f22aad50e..f8bef065386 100644 --- a/packages/shopify-dev-tools/scripts/generate-agent-skills.ts +++ b/packages/shopify-dev-tools/scripts/generate-agent-skills.ts @@ -633,9 +633,9 @@ export async function generateAgentSkills( await generateSkill(apiName, ctx); } - // Generic search skill is build-time canon, not part of any single api's - // instruction surface — skip it when scoped to a single api (e.g. experiments), - // except when the filter explicitly targets the generic skill by name. + // Generic search skill is build-time canon, not part of any single API's + // instruction surface. Skip it during filtered generation unless the filter + // explicitly targets the generic skill by name. if (!apiFilter || apiFilter === GENERIC_SKILL_NAME) { await generateGenericSearchSkill(ctx); } diff --git a/packages/shopify-dev-tools/src/experiments/experiments.test.ts b/packages/shopify-dev-tools/src/experiments/experiments.test.ts deleted file mode 100644 index f117a183164..00000000000 --- a/packages/shopify-dev-tools/src/experiments/experiments.test.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { describe, it, expect, vi } from "vitest"; - -import { Experiments, type ExperimentsVerdictClient } from "./experiments.js"; -import { revokeInstallId } from "./install-id.js"; - -function makeVerdictClient( - overrides: Partial = {}, -): ExperimentsVerdictClient { - return { - assignVariant: vi.fn().mockResolvedValue("control"), - flagEnabled: vi.fn().mockResolvedValue(true), - ...overrides, - }; -} - -describe("Experiments", () => { - it("deactivates when SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS is set", () => { - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: { SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS: "1" }, - }); - - expect(experiments.isActive).toBe(false); - expect(experiments.optOutReason).toBe( - "SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS set", - ); - expect(experiments.debugSubjectId).toBeNull(); - }); - - it("returns null/false without disk or network setup when opted out", async () => { - const installIdProvider = vi.fn(() => "should-not-be-read"); - const verdictFactory = vi.fn(() => makeVerdictClient()); - const fetchImpl = vi.fn(); - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: { CI: "true" }, - installIdProvider, - verdictFactory, - fetchImpl, - }); - - expect(experiments.isActive).toBe(false); - expect(await experiments.assign("e_foo")).toBeNull(); - expect(await experiments.flag("f_foo")).toBe(false); - expect(installIdProvider).not.toHaveBeenCalled(); - expect(verdictFactory).not.toHaveBeenCalled(); - expect(fetchImpl).not.toHaveBeenCalled(); - }); - - it("constructs with an install-id and becomes active when env is clean", () => { - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdOverride: "fixed-test-install-id", - verdictFactory: () => makeVerdictClient(), - }); - - expect(experiments.isActive).toBe(true); - expect(experiments.optOutReason).toBeUndefined(); - expect(experiments.debugSubjectId).toBe("fixed-test-install-id"); - }); - - it("assigns experiments with the stable default subject", async () => { - const verdictClient = makeVerdictClient({ - assignVariant: vi.fn().mockResolvedValue("treatment"), - }); - const verdictFactory = vi.fn(() => verdictClient); - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdProvider: () => "install-abc", - verdictFactory, - }); - - await expect(experiments.assign("e_foo")).resolves.toBe("treatment"); - expect(verdictFactory).toHaveBeenCalledWith( - expect.objectContaining({ - apiKey: "pub_test", - appName: "dev-mcp-test", - useEdgeApi: true, - fetchOptions: { timeout: 2000 }, - }), - ); - expect(verdictClient.assignVariant).toHaveBeenCalledWith("e_foo", { - type: "default", - subjectId: "install-abc", - }); - }); - - it("checks flags with the stable default subject", async () => { - const verdictClient = makeVerdictClient({ - flagEnabled: vi.fn().mockResolvedValue(false), - }); - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdProvider: () => "install-abc", - verdictFactory: () => verdictClient, - }); - - await expect(experiments.flag("f_foo")).resolves.toBe(false); - expect(verdictClient.flagEnabled).toHaveBeenCalledWith("f_foo", { - type: "default", - subjectId: "install-abc", - }); - }); - - it("contains observer failures from Verdict onError", () => { - const verdictError = new Error("verdict exploded"); - const onError = vi.fn(() => { - throw new Error("observer failed"); - }); - let verdictOnError: ((err: unknown) => void) | undefined; - - new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdProvider: () => "install-abc", - verdictFactory: (config) => { - verdictOnError = config.onError as (err: unknown) => void; - return makeVerdictClient(); - }, - onError, - }); - - expect(() => verdictOnError?.(verdictError)).not.toThrow(); - expect(onError).toHaveBeenCalledWith(verdictError); - }); - - it("returns safe defaults and reports unexpected Verdict errors", async () => { - const assignError = new Error("assign exploded"); - const flagError = new Error("flag exploded"); - const onError = vi.fn(); - const verdictClient = makeVerdictClient({ - assignVariant: vi.fn().mockRejectedValue(assignError), - flagEnabled: vi.fn().mockRejectedValue(flagError), - }); - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdProvider: () => "install-abc", - verdictFactory: () => verdictClient, - onError, - }); - - await expect(experiments.assign("e_foo")).resolves.toBeNull(); - await expect(experiments.flag("f_foo")).resolves.toBe(false); - expect(onError).toHaveBeenCalledWith(assignError); - expect(onError).toHaveBeenCalledWith(flagError); - }); - - it("keeps safe defaults when the error reporter throws", async () => { - const verdictClient = makeVerdictClient({ - assignVariant: vi.fn().mockRejectedValue(new Error("assign exploded")), - flagEnabled: vi.fn().mockRejectedValue(new Error("flag exploded")), - }); - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env: {}, - installIdProvider: () => "install-abc", - verdictFactory: () => verdictClient, - onError: () => { - throw new Error("observer failed"); - }, - }); - - await expect(experiments.assign("e_foo")).resolves.toBeNull(); - await expect(experiments.flag("f_foo")).resolves.toBe(false); - }); - - it("threads config.env to getOrCreateInstallId so tests use a sandbox", () => { - const sandbox = mkdtempSync(join(tmpdir(), "experiments-env-test-")); - const env = { XDG_STATE_HOME: sandbox }; - const verdictFactory = vi.fn(() => makeVerdictClient()); - - try { - const experiments = new Experiments({ - apiKey: "pub_test", - appName: "dev-mcp-test", - env, - verdictFactory, - }); - - expect(experiments.isActive).toBe(true); - const id = experiments.debugSubjectId; - expect(id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - - // Verify the ID was persisted in the sandbox, not the real home dir - const persisted = readFileSync( - join(sandbox, "shopify-ai-toolkit", "install-id"), - "utf-8", - ); - expect(persisted).toBe(id); - } finally { - revokeInstallId({ env }); - rmSync(sandbox, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/shopify-dev-tools/src/experiments/experiments.ts b/packages/shopify-dev-tools/src/experiments/experiments.ts deleted file mode 100644 index 980d46f1a6c..00000000000 --- a/packages/shopify-dev-tools/src/experiments/experiments.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { Verdict } from "@shopify/verdict/node"; -import type { - VariantAssignment, - VerdictError, - VerdictUserConfiguration, -} from "@shopify/verdict/node"; - -import { getOrCreateInstallId } from "./install-id.js"; -import { - createEdgeMonorailLogger, - type MonorailLogger, -} from "./monorail-logger.js"; -import { detectOptOut, type OptOutResult } from "./opt-out.js"; - -export interface ExperimentsConfig { - apiKey: string; - appName: string; - onError?: (err: VerdictError | unknown) => void; - loggerOverride?: MonorailLogger; - installIdOverride?: string; - installIdProvider?: () => string; - verdictFactory?: ( - config: VerdictUserConfiguration, - ) => ExperimentsVerdictClient; - fetchImpl?: typeof fetch; - env?: NodeJS.ProcessEnv; -} - -export type ExperimentSubject = { type: "default"; subjectId: string }; - -export interface ExperimentsVerdictClient { - assignVariant( - experimentHandle: string, - subject: ExperimentSubject, - ): Promise; - flagEnabled(flagHandle: string, subject: ExperimentSubject): Promise; -} - -export class Experiments { - private readonly verdict: ExperimentsVerdictClient | null; - private readonly subject: ExperimentSubject | null; - private readonly optOut: OptOutResult; - private readonly installId: string | null; - private readonly onError?: (err: VerdictError | unknown) => void; - - constructor(config: ExperimentsConfig) { - this.optOut = detectOptOut(config.env); - this.onError = config.onError; - - if (this.optOut.optedOut) { - this.verdict = null; - this.subject = null; - this.installId = null; - return; - } - - this.installId = - config.installIdProvider?.() ?? - config.installIdOverride ?? - getOrCreateInstallId({ env: config.env }); - this.subject = { type: "default", subjectId: this.installId }; - - const logger = - config.loggerOverride ?? - createEdgeMonorailLogger({ - fetchImpl: config.fetchImpl, - onError: config.onError, - }); - - const verdictConfig: VerdictUserConfiguration = { - apiKey: config.apiKey, - appName: config.appName, - useEdgeApi: true, - cacheConfiguration: { - cacheTTLSeconds: 300, - staleWhileRevalidateSeconds: 3600, - staleIfErrorSeconds: 86400, - }, - fetchOptions: { timeout: 2000 }, - logger, - onError: (err) => this.reportError(err), - }; - - this.verdict = config.verdictFactory - ? config.verdictFactory(verdictConfig) - : new Verdict(verdictConfig); - } - - async assign(handle: string): Promise { - if (!this.verdict || !this.subject) return null; - try { - return await this.verdict.assignVariant(handle, this.subject); - } catch (err) { - this.reportError(err); - return null; - } - } - - async flag(handle: string): Promise { - if (!this.verdict || !this.subject) return false; - try { - return await this.verdict.flagEnabled(handle, this.subject); - } catch (err) { - this.reportError(err); - return false; - } - } - - private reportError(err: VerdictError | unknown): void { - try { - this.onError?.(err); - } catch { - // Experiment failures must degrade to safe defaults, even if the error - // observer itself throws. - } - } - - get isActive(): boolean { - return this.verdict !== null; - } - - get optOutReason(): string | undefined { - return this.optOut.optedOut ? this.optOut.reason : undefined; - } - - get debugSubjectId(): string | null { - return this.installId; - } -} diff --git a/packages/shopify-dev-tools/src/experiments/index.ts b/packages/shopify-dev-tools/src/experiments/index.ts deleted file mode 100644 index 9212d748e20..00000000000 --- a/packages/shopify-dev-tools/src/experiments/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export { - Experiments, - type ExperimentSubject, - type ExperimentsConfig, - type ExperimentsVerdictClient, -} from "./experiments.js"; -export { - getOrCreateInstallId, - resolveInstallIdDir, - resolveInstallIdPath, - revokeInstallId, - type InstallIdOptions, -} from "./install-id.js"; -export { detectOptOut, type OptOutResult } from "./opt-out.js"; -export { - createEdgeMonorailLogger, - type MonorailLogger, - type MonorailLoggerOptions, - type VerdictTelemetryEvent, -} from "./monorail-logger.js"; diff --git a/packages/shopify-dev-tools/src/experiments/install-id.test.ts b/packages/shopify-dev-tools/src/experiments/install-id.test.ts deleted file mode 100644 index d1e0a5ab82d..00000000000 --- a/packages/shopify-dev-tools/src/experiments/install-id.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - statSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterEach, beforeEach, describe, expect, it } from "vitest"; - -import { - getOrCreateInstallId, - resolveInstallIdDir, - revokeInstallId, -} from "./install-id.js"; - -const UUID_PATTERN = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; - -function installIdFile(sandbox: string): string { - return join(sandbox, "shopify-ai-toolkit", "install-id"); -} - -describe("getOrCreateInstallId", () => { - let sandbox: string; - const originalXdg = process.env.XDG_STATE_HOME; - - beforeEach(() => { - sandbox = mkdtempSync(join(tmpdir(), "install-id-test-")); - process.env.XDG_STATE_HOME = sandbox; - }); - - afterEach(() => { - revokeInstallId({ env: { XDG_STATE_HOME: sandbox } }); - if (originalXdg === undefined) delete process.env.XDG_STATE_HOME; - else process.env.XDG_STATE_HOME = originalXdg; - }); - - it("creates a UUID on first call and persists it to disk", () => { - const id = getOrCreateInstallId(); - expect(id).toMatch(UUID_PATTERN); - expect(readFileSync(installIdFile(sandbox), "utf-8")).toBe(id); - }); - - it.skipIf(process.platform === "win32")( - "writes the install-id file with 0o600 permissions", - () => { - getOrCreateInstallId(); - expect(statSync(installIdFile(sandbox)).mode & 0o777).toBe(0o600); - }, - ); - - it("returns the same id on repeat calls", () => { - const first = getOrCreateInstallId(); - const second = getOrCreateInstallId(); - expect(second).toBe(first); - }); - - it("read-caches the id for the current process", () => { - const first = getOrCreateInstallId(); - writeFileSync(installIdFile(sandbox), "different-id-on-disk"); - - expect(getOrCreateInstallId()).toBe(first); - }); - - it("reads an existing id without overwriting it", () => { - const dir = join(sandbox, "shopify-ai-toolkit"); - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, "install-id"), "preexisting-id-value\n"); - - expect(getOrCreateInstallId()).toBe("preexisting-id-value"); - }); - - it("ignores whitespace-only files and regenerates", () => { - const dir = join(sandbox, "shopify-ai-toolkit"); - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, "install-id"), " \n"); - - const id = getOrCreateInstallId(); - expect(id).toMatch(UUID_PATTERN); - }); - - it("revokes the persisted id and clears the process cache", () => { - const first = getOrCreateInstallId(); - - expect(revokeInstallId()).toBe(true); - expect(existsSync(installIdFile(sandbox))).toBe(false); - - const second = getOrCreateInstallId(); - expect(second).toMatch(UUID_PATTERN); - expect(second).not.toBe(first); - expect(readFileSync(installIdFile(sandbox), "utf-8")).toBe(second); - }); - - it("returns false when revoking an id that does not exist", () => { - expect(revokeInstallId()).toBe(false); - }); -}); - -describe("resolveInstallIdDir", () => { - it("prefers XDG_STATE_HOME when set, on any platform", () => { - expect(resolveInstallIdDir({ XDG_STATE_HOME: "/xdg" }, "linux")).toMatch( - /(^|[\\/])xdg[\\/]shopify-ai-toolkit$/, - ); - expect(resolveInstallIdDir({ XDG_STATE_HOME: "/xdg" }, "darwin")).toMatch( - /(^|[\\/])xdg[\\/]shopify-ai-toolkit$/, - ); - expect( - resolveInstallIdDir( - { XDG_STATE_HOME: "C:\\xdg", LOCALAPPDATA: "C:\\AppData\\Local" }, - "win32", - ), - ).toMatch(/xdg[\\/]shopify-ai-toolkit$/); - }); - - it("uses %LOCALAPPDATA% on Windows when XDG_STATE_HOME is unset", () => { - const dir = resolveInstallIdDir( - { LOCALAPPDATA: "C:\\Users\\tester\\AppData\\Local" }, - "win32", - ); - expect(dir).toMatch(/AppData[\\/]Local[\\/]shopify-ai-toolkit$/); - }); - - it("falls back to ~/.config on Unix when XDG_STATE_HOME is unset", () => { - const dir = resolveInstallIdDir({}, "linux"); - expect(dir).toMatch(/\.config[\\/]shopify-ai-toolkit$/); - }); - - it("falls back to ~/.config on Windows when LOCALAPPDATA is missing", () => { - const dir = resolveInstallIdDir({}, "win32"); - expect(dir).toMatch(/\.config[\\/]shopify-ai-toolkit$/); - }); -}); diff --git a/packages/shopify-dev-tools/src/experiments/install-id.ts b/packages/shopify-dev-tools/src/experiments/install-id.ts deleted file mode 100644 index 5785bbd6d79..00000000000 --- a/packages/shopify-dev-tools/src/experiments/install-id.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { join } from "node:path"; - -const DIR_NAME = "shopify-ai-toolkit"; -const FILE_NAME = "install-id"; - -export interface InstallIdOptions { - env?: NodeJS.ProcessEnv; - platform?: NodeJS.Platform; -} - -let cachedInstallId: { file: string; id: string } | undefined; - -/** - * Resolves the directory that holds the install-id file. - * - * Stable ID strategy: - * - Generate one random UUID per local machine/user profile on first active use. - * - Persist it under the OS-specific per-user state location so future processes - * get stable Verdict bucketing for the same install. - * - Never read or create this file while experiments are opted out. - * - Revoke by deleting the file with `revokeInstallId()`; the next active use - * creates a fresh UUID. Existing `Experiments` instances keep their in-memory - * subject, so recreate them after revocation. Revocation is local and cannot - * delete telemetry that was already emitted. - * - * Precedence: - * 1. `XDG_STATE_HOME` — respected everywhere so tests (and Unix users who set it) - * can override the default. - * 2. `%LOCALAPPDATA%` on Windows — the idiomatic per-user, non-roaming location. - * 3. `~/.config/` — Unix fallback (works on Linux/macOS; matches gh, npm conventions). - * - * `env` and `platform` are injectable for tests; defaults are the live process values. - */ -export function resolveInstallIdDir( - env: NodeJS.ProcessEnv = process.env, - platform: NodeJS.Platform = process.platform, -): string { - const xdgStateHome = env.XDG_STATE_HOME; - if (xdgStateHome) return join(xdgStateHome, DIR_NAME); - - if (platform === "win32") { - const localAppData = env.LOCALAPPDATA; - if (localAppData) return join(localAppData, DIR_NAME); - } - - return join(homedir(), ".config", DIR_NAME); -} - -export function resolveInstallIdPath(options: InstallIdOptions = {}): string { - return join(resolveInstallIdDir(options.env, options.platform), FILE_NAME); -} - -function readValidId(file: string): string | null { - try { - const existing = readFileSync(file, "utf-8").trim(); - return existing || null; - } catch { - return null; - } -} - -function rememberInstallId(file: string, id: string): string { - cachedInstallId = { file, id }; - return id; -} - -export function getOrCreateInstallId(options: InstallIdOptions = {}): string { - const file = resolveInstallIdPath(options); - if (cachedInstallId?.file === file) return cachedInstallId.id; - - const existing = readValidId(file); - if (existing) return rememberInstallId(file, existing); - - mkdirSync(resolveInstallIdDir(options.env, options.platform), { - recursive: true, - }); - const id = randomUUID(); - - try { - // `wx` = create-only; fails with EEXIST if another process won the race. - // `mode` is honored on POSIX and silently ignored on Windows. - writeFileSync(file, id, { mode: 0o600, flag: "wx" }); - return rememberInstallId(file, id); - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; - // Another process wrote first. Prefer their id; if the pre-existing file was - // corrupt (empty/whitespace), overwrite with ours. - const winner = readValidId(file); - if (winner) return rememberInstallId(file, winner); - writeFileSync(file, id, { mode: 0o600 }); - return rememberInstallId(file, id); - } -} - -export function revokeInstallId(options: InstallIdOptions = {}): boolean { - const file = resolveInstallIdPath(options); - if (cachedInstallId?.file === file) cachedInstallId = undefined; - - try { - unlinkSync(file); - return true; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") return false; - throw err; - } -} diff --git a/packages/shopify-dev-tools/src/experiments/monorail-logger.test.ts b/packages/shopify-dev-tools/src/experiments/monorail-logger.test.ts deleted file mode 100644 index 0816fdc4303..00000000000 --- a/packages/shopify-dev-tools/src/experiments/monorail-logger.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; - -import { createEdgeMonorailLogger } from "./monorail-logger.js"; - -const sampleEvent = { - schemaId: "experiment_assignments/1.2", - payload: { - subjectId: "install-abc", - experiment: "e_foo", - group: "control", - sourceApp: "dev-mcp", - manualAssignment: false, - }, -}; - -function response(status: number): Response { - return new Response(null, { status }); -} - -describe("createEdgeMonorailLogger", () => { - it("POSTs to the edge Monorail endpoint with schema_id and payload", async () => { - const fetchImpl = vi.fn().mockResolvedValue(response(200)); - const logger = createEdgeMonorailLogger({ fetchImpl }); - - await logger.produce(sampleEvent); - - expect(fetchImpl).toHaveBeenCalledTimes(1); - const [url, init] = fetchImpl.mock.calls[0]; - expect(url).toBe("https://monorail-edge.shopifysvc.com/v1/produce"); - expect(init?.method).toBe("POST"); - expect(init?.headers).toMatchObject({ - "Content-Type": "application/json; charset=utf-8", - }); - expect(init?.headers).toHaveProperty("X-Monorail-Edge-Event-Created-At-Ms"); - expect(JSON.parse(init?.body as string)).toEqual({ - schema_id: sampleEvent.schemaId, - payload: sampleEvent.payload, - }); - }); - - it("respects a custom endpoint override", async () => { - const fetchImpl = vi.fn().mockResolvedValue(response(200)); - const logger = createEdgeMonorailLogger({ - fetchImpl, - endpoint: "https://example.test/ingest", - }); - - await logger.produce(sampleEvent); - - expect(fetchImpl.mock.calls[0][0]).toBe("https://example.test/ingest"); - }); - - it("swallows network errors and reports via onError", async () => { - const boom = new Error("network down"); - const fetchImpl = vi.fn().mockRejectedValue(boom); - const onError = vi.fn(); - const logger = createEdgeMonorailLogger({ fetchImpl, onError }); - - await expect(logger.produce(sampleEvent)).resolves.toBeUndefined(); - expect(onError).toHaveBeenCalledWith(boom); - }); - - it("reports non-2xx responses via onError without throwing", async () => { - const fetchImpl = vi.fn().mockResolvedValue(response(500)); - const onError = vi.fn(); - const logger = createEdgeMonorailLogger({ fetchImpl, onError }); - - await expect(logger.produce(sampleEvent)).resolves.toBeUndefined(); - expect(onError).toHaveBeenCalledTimes(1); - expect(onError.mock.calls[0][0]).toBeInstanceOf(Error); - expect((onError.mock.calls[0][0] as Error).message).toContain("HTTP 500"); - }); - - it("does not throw when the error reporter throws", async () => { - const fetchImpl = vi.fn().mockResolvedValue(response(500)); - const logger = createEdgeMonorailLogger({ - fetchImpl, - onError: () => { - throw new Error("observer failed"); - }, - }); - - await expect(logger.produce(sampleEvent)).resolves.toBeUndefined(); - }); - - it("passes an AbortSignal tied to the timeout", async () => { - const fetchImpl = vi.fn().mockResolvedValue(response(200)); - const logger = createEdgeMonorailLogger({ fetchImpl, timeoutMs: 50 }); - - await logger.produce(sampleEvent); - - const init = fetchImpl.mock.calls[0][1]; - expect(init?.signal).toBeInstanceOf(AbortSignal); - }); -}); diff --git a/packages/shopify-dev-tools/src/experiments/monorail-logger.ts b/packages/shopify-dev-tools/src/experiments/monorail-logger.ts deleted file mode 100644 index 86bd2220169..00000000000 --- a/packages/shopify-dev-tools/src/experiments/monorail-logger.ts +++ /dev/null @@ -1,75 +0,0 @@ -const EDGE_MONORAIL_ENDPOINT = - "https://monorail-edge.shopifysvc.com/v1/produce"; - -/** - * Structural shape of a Verdict telemetry event. Verdict's `EventLogger` - * interface expects `produce(event: { schemaId; payload })` — the exact union - * of schemas isn't re-exported from `@shopify/verdict/node`, so we type it - * structurally here. - */ -export interface VerdictTelemetryEvent { - schemaId: string; - payload: unknown; -} - -export interface MonorailLoggerOptions { - endpoint?: string; - fetchImpl?: typeof fetch; - onError?: (err: unknown) => void; - timeoutMs?: number; -} - -export interface MonorailLogger { - produce(event: VerdictTelemetryEvent): Promise; -} - -function reportError(onError: (err: unknown) => void, err: unknown): void { - try { - onError(err); - } catch { - // Telemetry failures, including failure handlers, must never break callers. - } -} - -export function createEdgeMonorailLogger( - options: MonorailLoggerOptions = {}, -): MonorailLogger { - const endpoint = options.endpoint ?? EDGE_MONORAIL_ENDPOINT; - const fetchImpl = options.fetchImpl ?? fetch; - const onError = options.onError ?? (() => {}); - const timeoutMs = options.timeoutMs ?? 2000; - - return { - async produce(event: VerdictTelemetryEvent): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - const response = await fetchImpl(endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json; charset=utf-8", - "X-Monorail-Edge-Event-Created-At-Ms": String(Date.now()), - }, - body: JSON.stringify({ - schema_id: event.schemaId, - payload: event.payload, - }), - signal: controller.signal, - }); - - if (!response.ok) { - reportError( - onError, - new Error( - `Monorail produce failed with HTTP ${response.status} ${response.statusText}`.trim(), - ), - ); - } - } catch (err) { - reportError(onError, err); - } finally { - clearTimeout(timer); - } - }, - }; -} diff --git a/packages/shopify-dev-tools/src/experiments/opt-out.test.ts b/packages/shopify-dev-tools/src/experiments/opt-out.test.ts deleted file mode 100644 index 62bbb23b74f..00000000000 --- a/packages/shopify-dev-tools/src/experiments/opt-out.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, it, expect } from "vitest"; - -import { detectOptOut } from "./opt-out.js"; - -describe("detectOptOut", () => { - it("returns not opted out for empty env", () => { - expect(detectOptOut({})).toEqual({ optedOut: false }); - }); - - it.each([ - ["SHOPIFY_DEV_TOOLS_DISABLE_EXPERIMENTS", "1"], - ["SHOPIFY_DEV_TOOLS_DISABLE_EXPERIMENTS", "true"], - ["SHOPIFY_DEV_TOOLS_DISABLE_EXPERIMENTS", "TRUE"], - ["SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS", "1"], - ["SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS", "true"], - ])("opts out when %s=%s", (name, value) => { - const result = detectOptOut({ [name]: value }); - expect(result).toEqual({ - optedOut: true, - reason: `${name} set`, - }); - }); - - it.each([ - "SHOPIFY_DEV_TOOLS_DISABLE_EXPERIMENTS", - "SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS", - ])("does not opt out for %s=0", (name) => { - expect(detectOptOut({ [name]: "0" })).toEqual({ - optedOut: false, - }); - }); - - it("opts out when DO_NOT_TRACK=1", () => { - expect(detectOptOut({ DO_NOT_TRACK: "1" })).toEqual({ - optedOut: true, - reason: "DO_NOT_TRACK=1", - }); - }); - - it("opts out when DNT=1", () => { - expect(detectOptOut({ DNT: "1" })).toEqual({ - optedOut: true, - reason: "DNT=1", - }); - }); - - it.each([ - "CI", - "CONTINUOUS_INTEGRATION", - "GITHUB_ACTIONS", - "BUILDKITE", - "JENKINS_URL", - "CIRCLECI", - "GITLAB_CI", - "TRAVIS", - ])("opts out when %s is set", (name) => { - const result = detectOptOut({ [name]: "anything-truthy" }); - expect(result).toEqual({ - optedOut: true, - reason: `CI detected (${name})`, - }); - }); - - it("prefers explicit disable flag over CI detection", () => { - const result = detectOptOut({ - SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS: "1", - CI: "true", - }); - expect(result.optedOut).toBe(true); - if (result.optedOut) { - expect(result.reason).toBe("SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS set"); - } - }); -}); diff --git a/packages/shopify-dev-tools/src/experiments/opt-out.ts b/packages/shopify-dev-tools/src/experiments/opt-out.ts deleted file mode 100644 index 7333c10f52b..00000000000 --- a/packages/shopify-dev-tools/src/experiments/opt-out.ts +++ /dev/null @@ -1,52 +0,0 @@ -const DISABLE_EXPERIMENTS_ENV_VARS = [ - "SHOPIFY_DEV_TOOLS_DISABLE_EXPERIMENTS", - "SHOPIFY_DEV_MCP_DISABLE_EXPERIMENTS", -] as const; - -const CI_ENV_VARS = [ - "CI", - "CONTINUOUS_INTEGRATION", - "GITHUB_ACTIONS", - "BUILDKITE", - "JENKINS_URL", - "CIRCLECI", - "GITLAB_CI", - "TRAVIS", -] as const; - -export type OptOutResult = - | { optedOut: false } - | { optedOut: true; reason: string }; - -function envFlagEnabled(value: string | undefined): boolean { - return value === "1" || value?.toLowerCase() === "true"; -} - -export function detectOptOut( - env: NodeJS.ProcessEnv = process.env, -): OptOutResult { - for (const name of DISABLE_EXPERIMENTS_ENV_VARS) { - if (envFlagEnabled(env[name])) { - return { - optedOut: true, - reason: `${name} set`, - }; - } - } - - if (env.DO_NOT_TRACK === "1") { - return { optedOut: true, reason: "DO_NOT_TRACK=1" }; - } - - if (env.DNT === "1") { - return { optedOut: true, reason: "DNT=1" }; - } - - for (const name of CI_ENV_VARS) { - if (env[name]) { - return { optedOut: true, reason: `CI detected (${name})` }; - } - } - - return { optedOut: false }; -} diff --git a/packages/shopify-dev-tools/src/instructions/.gitignore b/packages/shopify-dev-tools/src/instructions/.gitignore deleted file mode 100644 index 2d8bda52a9a..00000000000 --- a/packages/shopify-dev-tools/src/instructions/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Sibling overlays for `pnpm eval --experiment`. Drop a -# `.experiment.md` next to the canonical `.md` to A/B-test -# instructions without committing the experiment. -*.experiment.md diff --git a/packages/shopify-dev-tools/src/types/verdict.d.ts b/packages/shopify-dev-tools/src/types/verdict.d.ts deleted file mode 100644 index 32db92d3afa..00000000000 --- a/packages/shopify-dev-tools/src/types/verdict.d.ts +++ /dev/null @@ -1,34 +0,0 @@ -declare module "@shopify/verdict/node" { - export type VariantAssignment = string; - export type VerdictError = unknown; - - export interface VerdictUserConfiguration { - apiKey: string; - appName: string; - useEdgeApi: boolean; - cacheConfiguration: { - cacheTTLSeconds: number; - staleWhileRevalidateSeconds: number; - staleIfErrorSeconds: number; - }; - fetchOptions: { timeout: number }; - logger: { - produce(event: { schemaId: string; payload: unknown }): Promise; - }; - onError(err: unknown): void; - } - - export class Verdict { - constructor(config: VerdictUserConfiguration); - - assignVariant( - experimentHandle: string, - subject: { type: "default"; subjectId: string }, - ): Promise; - - flagEnabled( - flagHandle: string, - subject: { type: "default"; subjectId: string }, - ): Promise; - } -} diff --git a/packages/shopify-dev-tools/test-stubs/verdict.ts b/packages/shopify-dev-tools/test-stubs/verdict.ts deleted file mode 100644 index 84a2777dae0..00000000000 --- a/packages/shopify-dev-tools/test-stubs/verdict.ts +++ /dev/null @@ -1,34 +0,0 @@ -export type VariantAssignment = string; -export type VerdictError = unknown; - -export interface VerdictUserConfiguration { - apiKey: string; - appName: string; - useEdgeApi: boolean; - cacheConfiguration: { - cacheTTLSeconds: number; - staleWhileRevalidateSeconds: number; - staleIfErrorSeconds: number; - }; - fetchOptions: { timeout: number }; - logger: { - produce(event: { schemaId: string; payload: unknown }): Promise; - }; - onError(err: unknown): void; -} - -export class Verdict { - constructor(_config: VerdictUserConfiguration) {} - - async assignVariant(): Promise { - throw new Error( - "The Verdict client is unavailable in the public repository test environment.", - ); - } - - async flagEnabled(): Promise { - throw new Error( - "The Verdict client is unavailable in the public repository test environment.", - ); - } -} diff --git a/packages/shopify-dev-tools/vite.config.js b/packages/shopify-dev-tools/vite.config.js index 307a1239cea..0f22fd47d2b 100644 --- a/packages/shopify-dev-tools/vite.config.js +++ b/packages/shopify-dev-tools/vite.config.js @@ -136,9 +136,6 @@ export default defineConfig({ "http/index": fileURLToPath( new URL("./src/http/index.ts", import.meta.url), ), - "experiments/index": fileURLToPath( - new URL("./src/experiments/index.ts", import.meta.url), - ), "internal/index": fileURLToPath( new URL("./src/internal/index.ts", import.meta.url), ), @@ -163,7 +160,6 @@ export default defineConfig({ "graphql", "typescript", "html-tags", - "@shopify/verdict/node", ], output: { interop: "auto", @@ -185,9 +181,6 @@ export default defineConfig({ alias: { // Similar to the moduleNameMapper in Jest config "^(\\.{1,2}/.*)\\.js$": "$1", - "@shopify/verdict/node": fileURLToPath( - new URL("./test-stubs/verdict.ts", import.meta.url), - ), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7b7e6950e8..d3b2534ef75 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,7 +81,6 @@ overrides: '@shopify/cli-hydrogen>@shopify/plugin-cloudflare': link:./packages/plugin-cloudflare nanoid: 3.3.8 graphql: 16.14.2 - '@shopify/shopify-dev-tools>@shopify/verdict': '-' importers: