From 8b3a77a544e9ac4ef79a87b8e1df2fbf9ab0dccc Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 13 Jul 2026 15:37:13 -0700 Subject: [PATCH 01/15] feat(migrations): Unify schema and data journals Add a publishable migration package that executes SQL and TypeScript entries from one Drizzle-compatible journal. Route core and plugin upgrades through the shared runner so schema changes and resumable data migrations use one ordered ledger. Keep historical scripts isolated behind versioned capabilities, and wire the package into release, CI, Craft, documentation, and package validation. Co-Authored-By: GPT-5 Codex --- .craft.yml | 3 + .github/workflows/ci.yml | 3 + CONTRIBUTING.md | 1 + README.md | 1 + .../rules/no-core-plugin-dynamic-imports.yml | 2 +- ast-grep/rules/no-core-plugin-reexports.yml | 2 +- .../rules/no-core-plugin-static-imports.yml | 2 +- package.json | 8 +- packages/docs/src/content/docs/cli/upgrade.md | 111 +- .../src/content/docs/contribute/releasing.md | 1 + packages/junior-memory/README.md | 2 + packages/junior-memory/package.json | 2 + packages/junior-migrations/README.md | 39 + packages/junior-migrations/package.json | 47 + packages/junior-migrations/src/cli.ts | 24 + packages/junior-migrations/src/generate.ts | 68 + packages/junior-migrations/src/index.ts | 18 + packages/junior-migrations/src/journal.ts | 142 ++ packages/junior-migrations/src/runner.ts | 311 ++++ packages/junior-migrations/src/types.ts | 90 ++ .../junior-migrations/tests/generate.test.ts | 99 ++ .../junior-migrations/tests/journal.test.ts | 77 + .../junior-migrations/tests/runner.test.ts | 276 ++++ .../junior-migrations/tsconfig.build.json | 11 + packages/junior-migrations/tsconfig.json | 14 + packages/junior-migrations/tsup.config.ts | 22 + packages/junior-plugin-api/README.md | 12 +- .../junior-plugin-api/src/registration.ts | 3 + packages/junior-scheduler/README.md | 4 + packages/junior-scheduler/package.json | 2 + packages/junior/migrations/README.md | 40 +- packages/junior/package.json | 2 + .../src/chat/conversations/sql/migrations.ts | 303 ++-- .../junior/src/chat/plugins/migrations.ts | 198 +-- packages/junior/src/cli/upgrade.ts | 116 +- .../cli/upgrade/migrations/core-journal.ts | 60 + .../cli/upgrade/migrations/plugin-journal.ts | 70 + .../src/cli/upgrade/migrations/plugin-sql.ts | 23 - packages/junior/src/cli/upgrade/types.ts | 16 +- packages/junior/src/db/db.ts | 2 - packages/junior/src/db/neon.ts | 9 - packages/junior/src/db/postgres.ts | 9 - .../tests/component/cli/upgrade-cli.test.ts | 1308 ++++++++++++++++- .../component/memory-plugin-storage.test.ts | 77 +- .../component/scheduler-sql-plugin.test.ts | 317 +++- .../tests/fixtures/postgres/executor.ts | 6 - packages/junior/tests/fixtures/sql.ts | 2 - .../junior/tests/unit/sql/executor.test.ts | 1 - packages/junior/tsup.config.ts | 1 + pnpm-lock.yaml | 50 +- scripts/bump-release-versions.mjs | 1 + 51 files changed, 3537 insertions(+), 471 deletions(-) create mode 100644 packages/junior-migrations/README.md create mode 100644 packages/junior-migrations/package.json create mode 100644 packages/junior-migrations/src/cli.ts create mode 100644 packages/junior-migrations/src/generate.ts create mode 100644 packages/junior-migrations/src/index.ts create mode 100644 packages/junior-migrations/src/journal.ts create mode 100644 packages/junior-migrations/src/runner.ts create mode 100644 packages/junior-migrations/src/types.ts create mode 100644 packages/junior-migrations/tests/generate.test.ts create mode 100644 packages/junior-migrations/tests/journal.test.ts create mode 100644 packages/junior-migrations/tests/runner.test.ts create mode 100644 packages/junior-migrations/tsconfig.build.json create mode 100644 packages/junior-migrations/tsconfig.json create mode 100644 packages/junior-migrations/tsup.config.ts create mode 100644 packages/junior/src/cli/upgrade/migrations/core-journal.ts create mode 100644 packages/junior/src/cli/upgrade/migrations/plugin-journal.ts delete mode 100644 packages/junior/src/cli/upgrade/migrations/plugin-sql.ts diff --git a/.craft.yml b/.craft.yml index be494ed2d..caf81919d 100644 --- a/.craft.yml +++ b/.craft.yml @@ -7,6 +7,9 @@ targets: - name: npm id: "@sentry/junior" includeNames: /^sentry-junior-\d.*\.tgz$/ + - name: npm + id: "@sentry/junior-migrations" + includeNames: /^sentry-junior-migrations-\d.*\.tgz$/ - name: npm id: "@sentry/junior-plugin-api" includeNames: /^sentry-junior-plugin-api-\d.*\.tgz$/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fea7428b..a1abd30ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: - .github/actions/** core: - packages/junior/** + - packages/junior-migrations/** - packages/junior-plugin-api/** - packages/junior-scheduler/** - packages/junior-testing/** @@ -216,6 +217,7 @@ jobs: - run: pnpm --filter @sentry/junior-memory build - run: pnpm --filter @sentry/junior-github build - run: pnpm --filter @sentry/junior-vercel build + - run: pnpm --filter @sentry/junior-migrations test - run: pnpm --filter @sentry/junior-memory test - run: pnpm --filter @sentry/junior-github test - run: pnpm --filter @sentry/junior-vercel test @@ -343,6 +345,7 @@ jobs: run: | mkdir -p artifacts pnpm --filter @sentry/junior pack --pack-destination artifacts + pnpm --filter @sentry/junior-migrations pack --pack-destination artifacts pnpm --filter @sentry/junior-plugin-api pack --pack-destination artifacts pnpm --filter @sentry/junior-agent-browser pack --pack-destination artifacts pnpm --filter @sentry/junior-amplitude pack --pack-destination artifacts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aed6be2b..25619e00f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,6 +119,7 @@ pnpm build:pkg This repo uses Craft for manual lockstep npm releases of: - `@sentry/junior` +- `@sentry/junior-migrations` - `@sentry/junior-plugin-api` - `@sentry/junior-agent-browser` - `@sentry/junior-amplitude` diff --git a/README.md b/README.md index 8ecc578a1..165d50abb 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Start here: | Package | Purpose | | ------------------------------ | ---------------------------------------------------------------------------- | | `@sentry/junior` | Core Slack bot runtime | +| `@sentry/junior-migrations` | Mixed Drizzle SQL and TypeScript migration format and runner | | `@sentry/junior-plugin-api` | Lightweight plugin API types and helpers | | `@sentry/junior-agent-browser` | Agent Browser plugin package for browser automation | | `@sentry/junior-amplitude` | Read-only Amplitude product analytics through Amplitude's hosted MCP server | diff --git a/ast-grep/rules/no-core-plugin-dynamic-imports.yml b/ast-grep/rules/no-core-plugin-dynamic-imports.yml index 9ae0f5e4d..a36ee7071 100644 --- a/ast-grep/rules/no-core-plugin-dynamic-imports.yml +++ b/ast-grep/rules/no-core-plugin-dynamic-imports.yml @@ -12,4 +12,4 @@ constraints: all: - regex: '^["@'']@sentry/junior-.+["@'']$' - not: - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/ast-grep/rules/no-core-plugin-reexports.yml b/ast-grep/rules/no-core-plugin-reexports.yml index adb19e779..c4504cd33 100644 --- a/ast-grep/rules/no-core-plugin-reexports.yml +++ b/ast-grep/rules/no-core-plugin-reexports.yml @@ -14,4 +14,4 @@ rule: - not: has: kind: string - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/ast-grep/rules/no-core-plugin-static-imports.yml b/ast-grep/rules/no-core-plugin-static-imports.yml index c2af1d8e9..c26903f92 100644 --- a/ast-grep/rules/no-core-plugin-static-imports.yml +++ b/ast-grep/rules/no-core-plugin-static-imports.yml @@ -14,4 +14,4 @@ rule: - not: has: kind: string - regex: '^["@'']@sentry/junior-plugin-api["@'']$' + regex: '^["@'']@sentry/junior-(?:plugin-api|migrations)["@'']$' diff --git a/package.json b/package.json index 506a1fa1b..75079e953 100644 --- a/package.json +++ b/package.json @@ -12,19 +12,19 @@ "worktree:setup": "node scripts/worktree.mjs setup", "cloudflare:token": "node scripts/refresh-cloudflare-tunnel-token.mjs", "prepare": "simple-git-hooks", - "lint": "pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm --filter @sentry/junior-vercel lint && pnpm ast-grep:lint && pnpm package:lint", + "lint": "pnpm --filter @sentry/junior-migrations lint && pnpm --filter @sentry/junior lint && pnpm --filter @sentry/junior-memory lint && pnpm --filter @sentry/junior-github lint && pnpm --filter @sentry/junior-vercel lint && pnpm ast-grep:lint && pnpm package:lint", "lint:fix": "pnpm --filter @sentry/junior lint:fix", "ast-grep:lint": "ast-grep scan", "lint-staged": "lint-staged", - "build": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-vercel build && pnpm --filter @sentry/junior-dashboard build", + "build": "pnpm --filter @sentry/junior-migrations build && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-vercel build && pnpm --filter @sentry/junior-dashboard build", "build:example": "pnpm --filter @sentry/junior-example build", "docs:dev": "pnpm --filter @sentry/junior-docs dev", "docs:build": "pnpm --filter @sentry/junior-docs build", "docs:check": "pnpm --filter @sentry/junior-docs check", - "package:lint": "for pkg in packages/junior packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done", + "package:lint": "for pkg in packages/junior packages/junior-migrations packages/junior-plugin-api packages/junior-scheduler packages/junior-memory packages/junior-dashboard packages/junior-github packages/junior-agent-browser packages/junior-amplitude packages/junior-cloudflare packages/junior-datadog packages/junior-hex packages/junior-linear packages/junior-maintenance packages/junior-notion packages/junior-sentry packages/junior-vercel; do pnpm exec publint \"$pkg\" || exit $?; done", "release:check": "node scripts/check-release-config.mjs", "start": "pnpm --filter @sentry/junior-example dev", - "test": "pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-vercel build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-vercel test && pnpm --filter @sentry/junior-dashboard test", + "test": "pnpm --filter @sentry/junior-migrations test && pnpm --filter @sentry/junior build && pnpm --filter @sentry/junior-memory build && pnpm --filter @sentry/junior-github build && pnpm --filter @sentry/junior-vercel build && pnpm --filter @sentry/junior-dashboard build && pnpm --filter @sentry/junior test && pnpm --filter @sentry/junior-memory test && pnpm --filter @sentry/junior-github test && pnpm --filter @sentry/junior-vercel test && pnpm --filter @sentry/junior-dashboard test", "test:e2e:dashboard": "pnpm --filter @sentry/junior-dashboard build && playwright test -c packages/junior-dashboard/playwright.config.ts", "test:watch": "pnpm --filter @sentry/junior test:watch", "evals": "pnpm --filter @sentry/junior-evals evals", diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index 44434f54a..7052383cd 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -1,8 +1,8 @@ --- title: "junior upgrade" -description: "Apply Junior and plugin SQL schema migrations." +description: "Apply Junior schema and data migrations." type: reference -summary: Bring the configured Junior SQL database up to the installed schema. +summary: Move configured SQL schemas and persisted state forward after upgrades. prerequisites: - /start-here/quickstart/ related: @@ -11,65 +11,102 @@ related: - /cli/snapshot-create/ --- -Use `junior upgrade` after installing a new Junior release. The command applies -pending Drizzle migrations to the database configured by `DATABASE_URL`. +Use `junior upgrade` after installing a Junior release that includes schema or +data migrations. The command mutates the configured SQL database and state +stores, so run it from the same app environment that has the production state +and SQL environment variables configured for the deployment you are upgrading. ## Usage -Run the command from your Junior app: +Run it from a project that already has `@sentry/junior` installed: ```bash pnpm exec junior upgrade ``` -The command takes no arguments. It migrates the core Junior schema first, then -the schemas owned by enabled plugins. Already-applied migrations are recognized -as up to date and are not rerun. +The command takes no extra arguments. -## Upgrade bridge for older databases +## What it does -An existing Junior database without core Drizzle migration history must -complete the `0.107.1` upgrade before upgrading to a later release. Before -starting this bridge, block new ingress, drain active and resumable work, stop -all old workers and queue consumers, and keep them stopped until the later -release is ready. If Junior reports this unsupported database state: +`junior upgrade` runs migrations sequentially. Core and plugin migration +directories use Drizzle Kit's ordered journal and may contain either generated +SQL schema migrations or TypeScript data migrations targeting a versioned +host capability API. Current +upgrade work includes: -1. Install `@sentry/junior@0.107.1`. -2. Run `pnpm exec junior upgrade` and confirm it completes successfully. -3. Restore the intended Junior version. -4. Run `pnpm exec junior upgrade` again. -5. Deploy the intended version, then restart workers and reopen ingress. +- Apply core and enabled-plugin SQL schema migrations. +- Rewrite retained turn-session records from legacy storage shapes before the + new runtime reads them. +- Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed. +- Backfill retained conversation records into the shared Junior SQL database. The upgrade requires `DATABASE_URL`. +- Apply the SQL schema cutover and rewrite legacy Pi-message rows into canonical conversation events. +- Repair legacy token and estimated-cost rollups from durable SQL conversation events in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle. -Fresh databases without Junior tables do not require the bridge release. +Completed journal entries are tracked individually, and TypeScript migrations +can checkpoint progress for safe retries. Legacy backfills remain idempotent: +rerunning them skips records that were already moved, removes stale legacy +index entries that no longer have a record, and upserts SQL conversation rows. +The conversation-history import paginates through every conversation in the +retained activity index; orphaned or expired Redis keys outside that index are +not treated as retained history. After cutover, SQL owns durable conversation +metadata and event history. + +## Hard-cutover upgrade sequence + +The canonical conversation-event cutover is not rolling-compatible. Do not run it inside a Vercel build while the previous deployment can still accept work. Use this operator sequence: + +1. Block new ingress and enqueueing while leaving the previous release's workers and continuation consumers running. +2. Let existing work drain, then verify that no turns remain running or awaiting resume. +3. Stop every old worker, queue consumer, and heartbeat. Keep the old deployment stopped for the rest of the procedure. +4. Run the upgrade from an operator environment with the production `REDIS_URL`, `JUNIOR_STATE_KEY_PREFIX`, and `DATABASE_URL`. +5. Confirm the history import and message-event seal complete with no missing rows. +6. Run `junior check`, deploy the new release, and only then reopen ingress and start the new workers. + +Run the upgrade as a separate operator command: + +```bash +pnpm exec junior upgrade +pnpm exec junior check +``` + +The checkpoint and message-event rewrites fail closed if resumable work remains. After the drain succeeds, each rewrite invalidates stale resume state before changing physical event positions. Checkpoint normalization closes deletion gaps, and the message migration resequences the streams it changes while preserving reporting summaries. + +If the command exits nonzero, leave the deployment stopped, correct the reported state, and rerun it. Do not restart workers after only part of the sequence completes. ## Example output -An already-current database reports its migrations as existing: +Typical logs look like this: ```text -Checking database migrations... - junior: up to date (7 migrations) - junior-github: up to date (4 migrations) - junior-memory: up to date (5 migrations) - junior-scheduler: up to date (2 migrations) -Database is up to date (18 migrations). +Running Junior upgrade migrations... +Running migration core-migrations... +Finished migration core-migrations: scanned=8 migrated=4 existing=4 missing=0 skipped=0 +Running migration repair-conversation-usage... +Finished migration repair-conversation-usage: scanned=2 migrated=1 existing=1 missing=0 +Running migration plugin-migrations... +Finished migration plugin-migrations: scanned=8 migrated=8 existing=0 missing=0 skipped=0 +Junior upgrade complete. ``` ## Failure behavior -The command exits nonzero when it cannot connect to SQL, encounters an -unsupported pre-Drizzle database, or a migration fails. Treat that as a deploy -blocker: correct the reported database or migration error, then rerun the -command. +If the configured state store is unavailable or a legacy record is malformed, the CLI exits non-zero and prints the underlying error: + +```text +junior command failed: Legacy conversation work state is invalid for slack:C123:1712345.0001 +``` + +Treat that as a deploy blocker for the affected environment. Check `REDIS_URL`, `JUNIOR_STATE_KEY_PREFIX`, `DATABASE_URL`, and the reported legacy record before retrying. ## Verification -Confirm that the command exits successfully and lists `junior` plus each -enabled plugin that owns migrations. The final line reports whether the -database was already current or how many migrations were applied. +After running the command: + +1. Confirm the final log line includes `Junior upgrade complete`. +2. Confirm `backfill-conversation-events-sql` scanned the complete retained activity index and did not stop at one page. +3. Confirm `move-conversation-messages-to-events` reports `missing=0`. The runtime now uses the copied events. The legacy message table remains available so a later upgrade can recover messages written by old workers during deployment. +4. Run `pnpm exec junior check` before building or deploying the app. ## Next step -Run [junior check](/cli/check/) before deploying, then continue with -[junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox -dependencies. +Run [junior check](/cli/check/) after the upgrade, then continue with [junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox dependencies. diff --git a/packages/docs/src/content/docs/contribute/releasing.md b/packages/docs/src/content/docs/contribute/releasing.md index 5556fcb5c..9950005a3 100644 --- a/packages/docs/src/content/docs/contribute/releasing.md +++ b/packages/docs/src/content/docs/contribute/releasing.md @@ -12,6 +12,7 @@ related: Junior uses lockstep package releases for: - `@sentry/junior` +- `@sentry/junior-migrations` - `@sentry/junior-plugin-api` - `@sentry/junior-agent-browser` - `@sentry/junior-amplitude` diff --git a/packages/junior-memory/README.md b/packages/junior-memory/README.md index f6c634a13..1eb814815 100644 --- a/packages/junior-memory/README.md +++ b/packages/junior-memory/README.md @@ -55,6 +55,8 @@ exported types, tools, and tests are authoritative. - `MEMORY_RECALL_MAX_VECTOR_DISTANCE` or `recallMaxVectorDistance` configures the vector candidate threshold. - Generate schema changes with `pnpm --filter @sentry/junior-memory db:generate`. +- Generate self-contained data migrations with + `pnpm --filter @sentry/junior-memory db:generate:data --name `. Follow `../../policies/data-redaction.md`, `../../policies/security.md`, and the plugin contract in `../junior-plugin-api/README.md`. diff --git a/packages/junior-memory/package.json b/packages/junior-memory/package.json index 07b877453..75cf5e45c 100644 --- a/packages/junior-memory/package.json +++ b/packages/junior-memory/package.json @@ -25,6 +25,7 @@ "scripts": { "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts", "prepare": "pnpm run build", "prepack": "pnpm run build", @@ -39,6 +40,7 @@ "zod": "catalog:" }, "devDependencies": { + "@sentry/junior-migrations": "workspace:*", "@sentry/junior-testing": "workspace:*", "@types/node": "^25.9.1", "drizzle-kit": "catalog:", diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md new file mode 100644 index 000000000..c78ddc14c --- /dev/null +++ b/packages/junior-migrations/README.md @@ -0,0 +1,39 @@ +# `@sentry/junior-migrations` + +Junior's migration format extends a Drizzle Kit journal with TypeScript data +migrations. Every journal entry resolves to exactly one `.sql` or +`.ts` file. Drizzle Kit continues to own numbering and schema snapshots; +the Junior migration runner owns execution and exact per-entry tracking. + +TypeScript migrations target a versioned capability API and must not import +Junior runtime internals. New parsers and transforms belong in the migration +file so application refactors cannot break pending upgrades. A journal can +also invoke a versioned host task when adopting legacy migration code that +already shipped before the mixed journal existed; the task name then becomes +part of the permanent migration ABI. + +## Authoring + +Generate schema migrations with the owning package's normal Drizzle command. +Generate data migrations through this package's wrapper: + +```bash +junior-migrations generate \ + --config drizzle.config.ts \ + --out migrations \ + --name backfill_actor +``` + +The wrapper asks Drizzle Kit to create a custom journal entry and snapshot, +then replaces the empty SQL file with a `MigrationV1` TypeScript scaffold. + +## Compatibility + +Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock +`migrate()` function is not a supported executor for mixed journals because it +requires every entry to have a SQL file. Call `runMigrationJournal` instead and +provide the SQL, state, loader, and locking capabilities owned by the host. + +The runner validates that each TypeScript migration has no runtime imports. +Only a type-only import from `@sentry/junior-migrations` is accepted. Add a new +API version rather than changing an existing migration capability contract. diff --git a/packages/junior-migrations/package.json b/packages/junior-migrations/package.json new file mode 100644 index 000000000..391e14d0f --- /dev/null +++ b/packages/junior-migrations/package.json @@ -0,0 +1,47 @@ +{ + "name": "@sentry/junior-migrations", + "version": "0.101.0", + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/getsentry/junior.git", + "directory": "packages/junior-migrations" + }, + "bin": { + "junior-migrations": "dist/cli.js" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", + "prepare": "pnpm run build", + "prepack": "pnpm run build", + "lint": "oxlint --config ../junior/.oxlintrc.json --deny-warnings src tests tsup.config.ts", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "peerDependencies": { + "drizzle-kit": "catalog:" + }, + "devDependencies": { + "@types/node": "^25.9.1", + "drizzle-kit": "catalog:", + "drizzle-orm": "catalog:", + "oxlint": "^1.66.0", + "tsup": "^8.5.1", + "typescript": "^6.0.3", + "vitest": "^4.1.7" + } +} diff --git a/packages/junior-migrations/src/cli.ts b/packages/junior-migrations/src/cli.ts new file mode 100644 index 000000000..22714b788 --- /dev/null +++ b/packages/junior-migrations/src/cli.ts @@ -0,0 +1,24 @@ +import { parseArgs } from "node:util"; +import { generateTypeScriptMigration } from "./generate"; + +const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + config: { type: "string" }, + name: { type: "string" }, + out: { type: "string", default: "./migrations" }, + }, +}); + +if (positionals[0] !== "generate" || !values.config || !values.name) { + throw new Error( + "Usage: junior-migrations generate --config --name [--out ]", + ); +} + +const path = await generateTypeScriptMigration({ + configPath: values.config, + migrationsFolder: values.out, + name: values.name, +}); +console.log(`Created TypeScript migration ${path}`); diff --git a/packages/junior-migrations/src/generate.ts b/packages/junior-migrations/src/generate.ts new file mode 100644 index 000000000..3c78c79a3 --- /dev/null +++ b/packages/junior-migrations/src/generate.ts @@ -0,0 +1,68 @@ +import { spawn } from "node:child_process"; +import { readFile, rename, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { readMigrationJournal } from "./journal"; + +/** Drizzle Kit inputs used to create one TypeScript migration entry. */ +export interface GenerateTypeScriptMigrationOptions { + configPath: string; + cwd?: string; + migrationsFolder: string; + name: string; +} + +function run(command: string, args: string[], cwd: string): Promise { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { cwd, stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) { + resolvePromise(); + } else { + reject(new Error(`${command} exited with code ${code ?? "unknown"}`)); + } + }); + }); +} + +function scaffold(): string { + return `import type { MigrationV1 } from "@sentry/junior-migrations";\n\nconst migration = {\n apiVersion: 1,\n async up(context) {\n void context;\n },\n} satisfies MigrationV1;\n\nexport default migration;\n`; +} + +/** Create a journaled TypeScript migration through Drizzle Kit's custom generator. */ +export async function generateTypeScriptMigration( + options: GenerateTypeScriptMigrationOptions, +): Promise { + const cwd = resolve(options.cwd ?? process.cwd()); + const folder = resolve(cwd, options.migrationsFolder); + const before = await readMigrationJournal(folder).catch(() => []); + await run( + "drizzle-kit", + [ + "generate", + "--custom", + "--config", + options.configPath, + "--name", + options.name, + ], + cwd, + ); + const after = await readMigrationJournal(folder); + if (after.length !== before.length + 1) { + throw new Error("Drizzle Kit did not create exactly one migration entry"); + } + const entry = after.at(-1); + if (!entry) { + throw new Error("Drizzle Kit did not create a migration entry"); + } + const sqlPath = resolve(folder, `${entry.tag}.sql`); + const source = await readFile(sqlPath, "utf8"); + if (!source.includes("Custom SQL migration file")) { + throw new Error(`Refusing to replace non-custom migration ${entry.tag}`); + } + const typescriptPath = resolve(folder, `${entry.tag}.ts`); + await rename(sqlPath, typescriptPath); + await writeFile(typescriptPath, scaffold(), "utf8"); + return typescriptPath; +} diff --git a/packages/junior-migrations/src/index.ts b/packages/junior-migrations/src/index.ts new file mode 100644 index 000000000..44e810595 --- /dev/null +++ b/packages/junior-migrations/src/index.ts @@ -0,0 +1,18 @@ +export { generateTypeScriptMigration } from "./generate"; +export { readMigrationJournal, resolveMigrations } from "./journal"; +export { runMigrationJournal } from "./runner"; +export type { GenerateTypeScriptMigrationOptions } from "./generate"; +export type { RunMigrationJournalOptions } from "./runner"; +export type { + MigrationContextV1, + MigrationJsonValue, + MigrationJournalEntry, + MigrationProgressV1, + MigrationRunResult, + MigrationSqlExecutor, + MigrationStateV1, + MigrationTasksV1, + MigrationV1, + ResolvedMigration, + TypeScriptMigrationLoader, +} from "./types"; diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts new file mode 100644 index 000000000..14bd1dc8b --- /dev/null +++ b/packages/junior-migrations/src/journal.ts @@ -0,0 +1,142 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { MigrationJournalEntry, ResolvedMigration } from "./types"; + +interface DrizzleJournalEntry { + breakpoints?: unknown; + idx?: unknown; + tag?: unknown; + when?: unknown; +} + +interface DrizzleJournal { + dialect?: unknown; + entries?: unknown; +} + +function parseJournalEntry( + value: DrizzleJournalEntry, + position: number, +): MigrationJournalEntry { + if ( + typeof value.idx !== "number" || + !Number.isInteger(value.idx) || + value.idx !== position || + typeof value.when !== "number" || + !Number.isFinite(value.when) || + typeof value.tag !== "string" || + !value.tag || + typeof value.breakpoints !== "boolean" + ) { + throw new Error(`Invalid Drizzle journal entry at index ${position}`); + } + return { + breakpoints: value.breakpoints, + index: value.idx, + tag: value.tag, + when: value.when, + }; +} + +async function optionalFile(path: string): Promise { + try { + return await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +function validateTypeScriptSource(tag: string, source: string): void { + const imports = source.matchAll( + /^\s*import\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, + ); + for (const match of imports) { + const clause = match[1]?.trim(); + const specifier = match[2]; + if ( + !clause?.startsWith("type ") || + specifier !== "@sentry/junior-migrations" + ) { + throw new Error( + `TypeScript migration ${tag} may only import migration types`, + ); + } + } + if ( + /^\s*import\s*["']/m.test(source) || + /^\s*export\s+.+\s+from\s+["']/m.test(source) || + /\b(?:import\s*\(|require\s*\()/.test(source) + ) { + throw new Error(`TypeScript migration ${tag} cannot load runtime modules`); + } +} + +/** Read and validate the ordered Drizzle journal entries in one migration directory. */ +export async function readMigrationJournal( + migrationsFolder: string, +): Promise { + const journalPath = join(migrationsFolder, "meta", "_journal.json"); + let source: string; + try { + source = await readFile(journalPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error("Can't find meta/_journal.json file", { cause: error }); + } + throw error; + } + const parsed = JSON.parse(source) as DrizzleJournal; + if (parsed.dialect !== "postgresql" || !Array.isArray(parsed.entries)) { + throw new Error(`Unsupported Drizzle journal in ${journalPath}`); + } + const entries = parsed.entries.map((entry, index) => + parseJournalEntry(entry as DrizzleJournalEntry, index), + ); + const timestamps = new Set(); + for (const entry of entries) { + if (timestamps.has(entry.when)) { + throw new Error(`Duplicate migration timestamp ${entry.when}`); + } + timestamps.add(entry.when); + } + return entries; +} + +/** Resolve every journal entry to exactly one immutable SQL or TypeScript source file. */ +export async function resolveMigrations( + migrationsFolder: string, +): Promise { + const entries = await readMigrationJournal(migrationsFolder); + return await Promise.all( + entries.map(async (entry) => { + const sqlPath = join(migrationsFolder, `${entry.tag}.sql`); + const typescriptPath = join(migrationsFolder, `${entry.tag}.ts`); + const [sqlSource, typescriptSource] = await Promise.all([ + optionalFile(sqlPath), + optionalFile(typescriptPath), + ]); + if ((sqlSource === undefined) === (typescriptSource === undefined)) { + throw new Error( + `Migration ${entry.tag} must have exactly one .sql or .ts file`, + ); + } + const kind = sqlSource === undefined ? "typescript" : "sql"; + const source = sqlSource ?? typescriptSource ?? ""; + const path = kind === "sql" ? sqlPath : typescriptPath; + if (kind === "typescript") { + validateTypeScriptSource(entry.tag, source); + } + return { + ...entry, + hash: createHash("sha256").update(source).digest("hex"), + kind, + path, + source, + }; + }), + ); +} diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts new file mode 100644 index 000000000..48912d40b --- /dev/null +++ b/packages/junior-migrations/src/runner.ts @@ -0,0 +1,311 @@ +import type { + MigrationContextV1, + MigrationRunResult, + MigrationSqlExecutor, + MigrationV1, + ResolvedMigration, + TypeScriptMigrationLoader, +} from "./types"; +import { resolveMigrations } from "./journal"; + +interface MigrationRow { + createdAt: string; + hash: string; + progress: unknown; + status: string | null; +} + +interface RunMigrationJournalBaseOptions { + beforeRun?: () => Promise; + executor: MigrationSqlExecutor; + migrationsFolder: string; + migrationsTable: string; +} + +interface RunAllMigrationJournalOptions extends RunMigrationJournalBaseOptions { + createContext: (args: { + migration: ResolvedMigration; + progress: MigrationContextV1["progress"]; + }) => MigrationContextV1; + loadTypeScript: TypeScriptMigrationLoader; + mode?: "all"; +} + +interface RunSqlMigrationJournalOptions extends RunMigrationJournalBaseOptions { + createContext?: never; + loadTypeScript?: never; + mode: "sql"; +} + +/** Host capabilities and execution mode for one mixed migration journal. */ +export type RunMigrationJournalOptions = + | RunAllMigrationJournalOptions + | RunSqlMigrationJournalOptions; + +function identifier(value: string): string { + if (!/^[a-z_][a-z0-9_]*$/.test(value)) { + throw new Error(`Invalid migration table identifier: ${value}`); + } + return value; +} + +function qualifiedTable(table: string): string { + return `drizzle.${identifier(table)}`; +} + +async function ensureMigrationTable( + executor: MigrationSqlExecutor, + table: string, +): Promise { + const qualified = qualifiedTable(table); + await executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); + await executor.execute(` +CREATE TABLE IF NOT EXISTS ${qualified} ( + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL, + created_at BIGINT +) +`); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS name TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS kind TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS status TEXT`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS progress JSONB`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS result JSONB`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ`, + ); + await executor.execute( + `ALTER TABLE ${qualified} ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ`, + ); +} + +async function migrationRows( + executor: MigrationSqlExecutor, + table: string, +): Promise> { + const rows = await executor.query(` +SELECT + created_at::text AS "createdAt", + hash, + progress, + status +FROM ${qualifiedTable(table)} +WHERE created_at IS NOT NULL +ORDER BY created_at ASC, id ASC +`); + return new Map(rows.map((row) => [Number(row.createdAt), row])); +} + +async function adoptLegacySqlPrefix(args: { + executor: MigrationSqlExecutor; + migrations: readonly ResolvedMigration[]; + rows: Map; + table: string; +}): Promise { + const latestAppliedAt = Math.max( + ...args.rows.keys(), + Number.NEGATIVE_INFINITY, + ); + if (!Number.isFinite(latestAppliedAt)) { + return; + } + for (const migration of args.migrations) { + if ( + migration.when >= latestAppliedAt || + migration.kind !== "sql" || + args.rows.has(migration.when) + ) { + continue; + } + await args.executor.execute( + `INSERT INTO ${qualifiedTable(args.table)} + (hash, created_at, name, kind, status, completed_at) + VALUES ($1, $2, $3, 'sql', 'completed', NOW())`, + [migration.hash, migration.when, migration.tag], + ); + } +} + +function sqlStatements(migration: ResolvedMigration): string[] { + return migration.source + .split("--> statement-breakpoint") + .map((statement) => statement.trim()) + .filter(Boolean); +} + +function migrationV1(value: unknown, tag: string): MigrationV1 { + const candidate = + typeof value === "object" && value !== null && "default" in value + ? (value as { default?: unknown }).default + : value; + if ( + typeof candidate !== "object" || + candidate === null || + (candidate as { apiVersion?: unknown }).apiVersion !== 1 || + typeof (candidate as { up?: unknown }).up !== "function" + ) { + throw new Error(`TypeScript migration ${tag} does not export MigrationV1`); + } + return candidate as MigrationV1; +} + +async function runSqlMigration(args: { + executor: MigrationSqlExecutor; + migration: ResolvedMigration; + table: string; +}): Promise { + await args.executor.transaction(async () => { + for (const statement of sqlStatements(args.migration)) { + await args.executor.execute(statement); + } + await args.executor.execute( + `INSERT INTO ${qualifiedTable(args.table)} + (hash, created_at, name, kind, status, started_at, completed_at) + VALUES ($1, $2, $3, 'sql', 'completed', NOW(), NOW())`, + [args.migration.hash, args.migration.when, args.migration.tag], + ); + }); +} + +async function runTypeScriptMigration(args: { + createContext: RunAllMigrationJournalOptions["createContext"]; + executor: MigrationSqlExecutor; + loadTypeScript: TypeScriptMigrationLoader; + migration: ResolvedMigration; + row: MigrationRow | undefined; + table: string; +}): Promise { + const qualified = qualifiedTable(args.table); + if (args.row && args.row.hash !== args.migration.hash) { + throw new Error(`Migration ${args.migration.tag} changed after it started`); + } + if (args.row) { + await args.executor.execute( + `UPDATE ${qualified} + SET name = $1, kind = 'typescript', status = 'running', started_at = NOW() + WHERE created_at = $2`, + [args.migration.tag, args.migration.when], + ); + } else { + await args.executor.execute( + `INSERT INTO ${qualified} + (hash, created_at, name, kind, status, started_at) + VALUES ($1, $2, $3, 'typescript', 'running', NOW())`, + [args.migration.hash, args.migration.when, args.migration.tag], + ); + } + const progress: MigrationContextV1["progress"] = { + async load() { + const [current] = await args.executor.query<{ + progress: Awaited>; + }>(`SELECT progress FROM ${qualified} WHERE created_at = $1 LIMIT 1`, [ + args.migration.when, + ]); + return current?.progress ?? undefined; + }, + async save(value) { + await args.executor.execute( + `UPDATE ${qualified} SET progress = $1, status = 'running' WHERE created_at = $2`, + [value, args.migration.when], + ); + }, + }; + try { + const context = args.createContext({ migration: args.migration, progress }); + const migration = migrationV1( + await args.loadTypeScript(args.migration.path), + args.migration.tag, + ); + const result = await migration.up(context); + await args.executor.execute( + `UPDATE ${qualified} + SET status = 'completed', result = $1, completed_at = NOW() + WHERE created_at = $2`, + [result ?? null, args.migration.when], + ); + } catch (error) { + await args.executor.execute( + `UPDATE ${qualified} SET status = 'failed' WHERE created_at = $1`, + [args.migration.when], + ); + throw error; + } +} + +/** Execute one mixed Drizzle journal with exact SQL and TypeScript entry tracking. */ +export async function runMigrationJournal( + options: RunMigrationJournalOptions, +): Promise { + const migrations = await resolveMigrations(options.migrationsFolder); + const mode = options.mode ?? "all"; + return await options.executor.withMigrationLock( + options.migrationsTable, + async () => { + await options.beforeRun?.(); + await ensureMigrationTable(options.executor, options.migrationsTable); + let rows = await migrationRows(options.executor, options.migrationsTable); + await adoptLegacySqlPrefix({ + executor: options.executor, + migrations, + rows, + table: options.migrationsTable, + }); + rows = await migrationRows(options.executor, options.migrationsTable); + const result: MigrationRunResult = { + existing: 0, + migrated: 0, + scanned: migrations.length, + skipped: 0, + }; + for (const migration of migrations) { + const row = rows.get(migration.when); + if (row && row.hash !== migration.hash) { + throw new Error( + `Migration ${migration.tag} changed after it started`, + ); + } + if (row?.status === "completed" || (row && row.status === null)) { + result.existing += 1; + continue; + } + if (mode === "sql" && migration.kind === "typescript") { + result.skipped += 1; + continue; + } + if (migration.kind === "sql") { + await runSqlMigration({ + executor: options.executor, + migration, + table: options.migrationsTable, + }); + } else { + if (!options.createContext || !options.loadTypeScript) { + throw new Error( + `TypeScript migration ${migration.tag} requires a loader and context`, + ); + } + await runTypeScriptMigration({ + createContext: options.createContext, + executor: options.executor, + loadTypeScript: options.loadTypeScript, + migration, + row, + table: options.migrationsTable, + }); + } + result.migrated += 1; + } + return result; + }, + ); +} diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts new file mode 100644 index 000000000..20a45206d --- /dev/null +++ b/packages/junior-migrations/src/types.ts @@ -0,0 +1,90 @@ +/** SQL capabilities available to the mixed migration runner. */ +export interface MigrationSqlExecutor { + execute(statement: string, parameters?: readonly unknown[]): Promise; + query( + statement: string, + parameters?: readonly unknown[], + ): Promise; + transaction(callback: () => Promise): Promise; + withMigrationLock( + migrationTable: string, + callback: () => Promise, + ): Promise; +} + +/** Stable state-store capabilities exposed to v1 TypeScript migrations. */ +export interface MigrationStateV1 { + appendToList( + key: string, + value: unknown, + options?: { maxLength?: number; ttlMs?: number }, + ): Promise; + delete(key: string): Promise; + get(key: string): Promise; + getList(key: string): Promise; + set(key: string, value: unknown, ttlMs?: number): Promise; +} + +/** Resumable progress storage scoped to one TypeScript migration. */ +export interface MigrationProgressV1 { + load(): Promise; + save(value: MigrationJsonValue): Promise; +} + +/** Versioned host tasks available to migrations that predate the public ABI. */ +export interface MigrationTasksV1 { + run(name: string): Promise; +} + +/** JSON-compatible value persisted in migration progress and result columns. */ +export type MigrationJsonValue = + | boolean + | number + | string + | null + | MigrationJsonValue[] + | { [key: string]: MigrationJsonValue }; + +/** Permanent capability contract for apiVersion 1 migrations. */ +export interface MigrationContextV1 { + log(message: string): void; + progress: MigrationProgressV1; + sql: Pick; + state: MigrationStateV1; + tasks: MigrationTasksV1; +} + +/** Isolated TypeScript data migration targeting the v1 ABI. */ +export interface MigrationV1 { + apiVersion: 1; + up(context: MigrationContextV1): Promise; +} + +/** One ordered entry from Drizzle Kit's journal metadata. */ +export interface MigrationJournalEntry { + breakpoints: boolean; + index: number; + tag: string; + when: number; +} + +/** Journal entry paired with its unique SQL or TypeScript source file. */ +export interface ResolvedMigration extends MigrationJournalEntry { + hash: string; + kind: "sql" | "typescript"; + path: string; + source: string; +} + +/** Aggregate counts returned after one journal execution. */ +export interface MigrationRunResult { + existing: number; + migrated: number; + scanned: number; + skipped: number; +} + +/** Host-provided loader for an isolated TypeScript migration module. */ +export interface TypeScriptMigrationLoader { + (path: string): Promise; +} diff --git a/packages/junior-migrations/tests/generate.test.ts b/packages/junior-migrations/tests/generate.test.ts new file mode 100644 index 000000000..399d368a8 --- /dev/null +++ b/packages/junior-migrations/tests/generate.test.ts @@ -0,0 +1,99 @@ +import { spawn } from "node:child_process"; +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { generateTypeScriptMigration } from "../src/generate"; +import { readMigrationJournal } from "../src/journal"; + +const temporaryDirectories: string[] = []; + +function runDrizzle(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("drizzle-kit", args, { cwd, stdio: "pipe" }); + let output = ""; + child.stdout.on("data", (chunk) => { + output += String(chunk); + }); + child.stderr.on("data", (chunk) => { + output += String(chunk); + }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(output)); + } + }); + }); +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +it("keeps Drizzle schema generation working across a TypeScript entry", async () => { + const relativeRoot = `.tmp-mixed-migrations-${Date.now()}`; + const root = join(process.cwd(), relativeRoot); + temporaryDirectories.push(root); + await mkdir(root); + await writeFile( + join(root, "schema.ts"), + 'import { pgTable, text } from "drizzle-orm/pg-core";\nexport const users = pgTable("users", { id: text("id").primaryKey() });\n', + ); + await writeFile( + join(root, "drizzle.config.ts"), + `import { defineConfig } from "drizzle-kit";\nexport default defineConfig({ dialect: "postgresql", schema: "./${relativeRoot}/schema.ts", out: "./${relativeRoot}/migrations" });\n`, + ); + + await runDrizzle( + [ + "generate", + "--config", + `${relativeRoot}/drizzle.config.ts`, + "--name", + "initial", + ], + process.cwd(), + ); + const typescriptPath = await generateTypeScriptMigration({ + configPath: `${relativeRoot}/drizzle.config.ts`, + cwd: process.cwd(), + migrationsFolder: `${relativeRoot}/migrations`, + name: "backfill", + }); + await expect(access(typescriptPath)).resolves.toBeUndefined(); + await expect( + access(join(root, "migrations", "0001_backfill.sql")), + ).rejects.toThrow("ENOENT"); + + await writeFile( + join(root, "schema.ts"), + 'import { pgTable, text } from "drizzle-orm/pg-core";\nexport const users = pgTable("users", { id: text("id").primaryKey(), name: text("name") });\n', + ); + await runDrizzle( + [ + "generate", + "--config", + `${relativeRoot}/drizzle.config.ts`, + "--name", + "add_name", + ], + process.cwd(), + ); + + await expect( + readMigrationJournal(join(root, "migrations")), + ).resolves.toMatchObject([ + { index: 0, tag: "0000_initial" }, + { index: 1, tag: "0001_backfill" }, + { index: 2, tag: "0002_add_name" }, + ]); + await expect( + readFile(join(root, "migrations", "0002_add_name.sql"), "utf8"), + ).resolves.toContain('ADD COLUMN "name" text'); +}); diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts new file mode 100644 index 000000000..005586efd --- /dev/null +++ b/packages/junior-migrations/tests/journal.test.ts @@ -0,0 +1,77 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveMigrations } from "../src/journal"; + +const temporaryDirectories: string[] = []; + +async function migrationFolder(entries: string[]): Promise { + const root = await mkdtemp(join(tmpdir(), "junior-migrations-")); + temporaryDirectories.push(root); + await mkdir(join(root, "meta")); + await writeFile( + join(root, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: entries.map((tag, index) => ({ + idx: index, + version: "7", + when: 1_000 + index, + tag, + breakpoints: true, + })), + }), + ); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("resolveMigrations", () => { + it("resolves one ordered SQL or TypeScript file per journal entry", async () => { + const folder = await migrationFolder(["0000_initial", "0001_backfill"]); + await writeFile(join(folder, "0000_initial.sql"), "SELECT 1;"); + await writeFile( + join(folder, "0001_backfill.ts"), + 'import type { MigrationV1 } from "@sentry/junior-migrations";\nexport default { apiVersion: 1, async up() {} } satisfies MigrationV1;\n', + ); + + await expect(resolveMigrations(folder)).resolves.toMatchObject([ + { index: 0, kind: "sql", tag: "0000_initial", when: 1_000 }, + { index: 1, kind: "typescript", tag: "0001_backfill", when: 1_001 }, + ]); + }); + + it("rejects migrations that import runtime modules", async () => { + const folder = await migrationFolder(["0000_unsafe"]); + await writeFile( + join(folder, "0000_unsafe.ts"), + 'import { getDb } from "@/db";\nexport default { apiVersion: 1, async up() { getDb(); } };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "may only import migration types", + ); + }); + + it("rejects ambiguous migration files", async () => { + const folder = await migrationFolder(["0000_ambiguous"]); + await writeFile(join(folder, "0000_ambiguous.sql"), "SELECT 1;"); + await writeFile( + join(folder, "0000_ambiguous.ts"), + "export default { apiVersion: 1, async up() {} };\n", + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "exactly one .sql or .ts file", + ); + }); +}); diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts new file mode 100644 index 000000000..0c1c21575 --- /dev/null +++ b/packages/junior-migrations/tests/runner.test.ts @@ -0,0 +1,276 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runMigrationJournal } from "../src/runner"; +import type { + MigrationContextV1, + MigrationSqlExecutor, + MigrationV1, +} from "../src/types"; + +interface StoredRow { + createdAt: number; + hash: string; + progress?: unknown; + status: string | null; +} + +class FakeExecutor implements MigrationSqlExecutor { + readonly rows = new Map(); + readonly statements: string[] = []; + + async execute(statement: string, parameters: readonly unknown[] = []) { + const normalized = statement.trim(); + if ( + normalized.startsWith("CREATE SCHEMA") || + normalized.startsWith("CREATE TABLE IF NOT EXISTS drizzle.") || + normalized.startsWith("ALTER TABLE drizzle.") + ) { + return; + } + if (normalized.startsWith("INSERT INTO drizzle.")) { + const hash = String(parameters[0]); + const createdAt = Number(parameters[1]); + const kind = normalized.includes("'typescript'") + ? "running" + : "completed"; + this.rows.set(createdAt, { createdAt, hash, status: kind }); + return; + } + if (normalized.startsWith("UPDATE drizzle.")) { + const createdAt = Number(parameters.at(-1)); + const row = this.rows.get(createdAt); + if (!row) { + throw new Error(`Missing row ${createdAt}`); + } + if (normalized.includes("progress = $1")) { + row.progress = parameters[0]; + } + if (normalized.includes("status = 'running'")) { + row.status = "running"; + } else if (normalized.includes("status = 'completed'")) { + row.status = "completed"; + } else if (normalized.includes("status = 'failed'")) { + row.status = "failed"; + } + return; + } + this.statements.push(normalized); + } + + async query(statement: string, parameters: readonly unknown[] = []) { + if (statement.includes('created_at::text AS "createdAt"')) { + return [...this.rows.values()].map((row) => ({ + createdAt: String(row.createdAt), + hash: row.hash, + progress: row.progress, + status: row.status, + })) as T[]; + } + if (statement.includes("SELECT progress")) { + const row = this.rows.get(Number(parameters[0])); + return (row ? [{ progress: row.progress ?? null }] : []) as T[]; + } + return []; + } + + async transaction(callback: () => Promise): Promise { + return await callback(); + } + + async withMigrationLock( + _migrationTable: string, + callback: () => Promise, + ): Promise { + return await callback(); + } +} + +const temporaryDirectories: string[] = []; + +async function mixedFolder(): Promise { + const root = await mkdtemp(join(tmpdir(), "junior-migrations-runner-")); + temporaryDirectories.push(root); + await mkdir(join(root, "meta")); + await writeFile( + join(root, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: ["0000_schema", "0001_data", "0002_schema"].map( + (tag, index) => ({ + idx: index, + version: "7", + when: 2_000 + index, + tag, + breakpoints: true, + }), + ), + }), + ); + await writeFile(join(root, "0000_schema.sql"), "SELECT 'schema-zero';"); + await writeFile( + join(root, "0001_data.ts"), + 'import type { MigrationV1 } from "@sentry/junior-migrations";\nexport default {} satisfies MigrationV1;\n', + ); + await writeFile(join(root, "0002_schema.sql"), "SELECT 'schema-two';"); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("runMigrationJournal", () => { + it("runs mixed entries in journal order and is a no-op on rerun", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + const order: string[] = []; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + order.push("typescript"); + await context.progress.save({ cursor: 1 }); + }, + }; + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + log: () => {}, + progress, + sql: executor, + state: { + appendToList: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + set: async () => {}, + }, + tasks: { + run: async () => undefined, + }, + }), + }), + ).resolves.toEqual({ existing: 0, migrated: 3, scanned: 3, skipped: 0 }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + expect(order).toEqual(["typescript"]); + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + log: () => {}, + progress, + sql: executor, + state: { + appendToList: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + set: async () => {}, + }, + tasks: { + run: async () => undefined, + }, + }), + }), + ).resolves.toEqual({ existing: 3, migrated: 0, scanned: 3, skipped: 0 }); + }); + + it("allows schema-only execution to leave a TypeScript gap pending", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + mode: "sql", + }), + ).resolves.toEqual({ existing: 0, migrated: 2, scanned: 3, skipped: 1 }); + expect([...executor.rows.keys()].sort()).toEqual([2_000, 2_002]); + }); + + it("resumes a failed TypeScript migration from saved progress", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + let attempts = 0; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + attempts += 1; + const progress = await context.progress.load(); + if (!progress) { + await context.progress.save({ cursor: 1 }); + throw new Error("interrupted"); + } + return { resumed: true }; + }, + }; + const options = { + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }) => ({ + log: () => {}, + progress, + sql: executor, + state: { + appendToList: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + set: async () => {}, + }, + tasks: { + run: async () => undefined, + }, + }), + }; + + await expect(runMigrationJournal(options)).rejects.toThrow("interrupted"); + expect(executor.rows.get(2_001)).toMatchObject({ + progress: { cursor: 1 }, + status: "failed", + }); + expect(executor.statements).toEqual(["SELECT 'schema-zero';"]); + + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 1, + migrated: 2, + scanned: 3, + skipped: 0, + }); + expect(attempts).toBe(2); + expect(executor.rows.get(2_001)).toMatchObject({ + progress: { cursor: 1 }, + status: "completed", + }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + }); +}); diff --git a/packages/junior-migrations/tsconfig.build.json b/packages/junior-migrations/tsconfig.build.json new file mode 100644 index 000000000..7d583629c --- /dev/null +++ b/packages/junior-migrations/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "noEmit": false, + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/junior-migrations/tsconfig.json b/packages/junior-migrations/tsconfig.json new file mode 100644 index 000000000..165b4315c --- /dev/null +++ b/packages/junior-migrations/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM"], + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "isolatedModules": true, + "noEmit": true, + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/packages/junior-migrations/tsup.config.ts b/packages/junior-migrations/tsup.config.ts new file mode 100644 index 000000000..254c4d2c6 --- /dev/null +++ b/packages/junior-migrations/tsup.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from "tsup"; + +const shared = { + format: "esm" as const, + tsconfig: "tsconfig.build.json", + dts: false, + outDir: "dist", +}; + +export default defineConfig([ + { + ...shared, + entry: { index: "src/index.ts" }, + clean: true, + }, + { + ...shared, + entry: { cli: "src/cli.ts" }, + clean: false, + banner: { js: "#!/usr/bin/env node" }, + }, +]); diff --git a/packages/junior-plugin-api/README.md b/packages/junior-plugin-api/README.md index 2c1537c1f..3eeae30d0 100644 --- a/packages/junior-plugin-api/README.md +++ b/packages/junior-plugin-api/README.md @@ -5,12 +5,14 @@ exported TypeScript types and runtime validators are authoritative. ## Registration -Use `defineJuniorPlugin({ manifest, hooks, tasks, cli, model })`. A plugin name -is a lowercase identifier and is unique within the enabled app plugin set. +Use `defineJuniorPlugin({ manifest, hooks, tasks, migrationTasks, cli, model })`. +A plugin name is a lowercase identifier and is unique within the enabled app +plugin set. A plugin may instead be a declarative `plugin.yaml` package when it has no -host-executed hooks. Do not combine an inline manifest with a second YAML -definition for the same plugin. +host-executed hooks, tasks, migration tasks, CLI, or model implementation. Do +not combine an inline manifest with a second YAML definition for the same +plugin. ## Manifest @@ -58,6 +60,8 @@ reports, and other typed hook surfaces exported by this package. - Packaged migrations create plugin-owned tables through the host migration runner. +- TypeScript journal entries may invoke explicitly versioned `migrationTasks`; + tasks run only when their owning journal entry is pending. - Generate migration artifacts from the package schema; do not hand-maintain a second schema contract. - Runtime hooks and CLI actions use host-provided `ctx.db`. diff --git a/packages/junior-plugin-api/src/registration.ts b/packages/junior-plugin-api/src/registration.ts index 1800108ef..17c2b282e 100644 --- a/packages/junior-plugin-api/src/registration.ts +++ b/packages/junior-plugin-api/src/registration.ts @@ -1,6 +1,7 @@ import type { PluginCliDefinition } from "./cli"; import type { PluginHooks } from "./hooks"; import type { PluginManifest } from "./manifest"; +import type { PluginMigrationTask } from "./operations"; import type { PluginTasks } from "./tasks"; export interface PluginModelConfig { @@ -14,6 +15,8 @@ export type PluginRegistrationInput = { cli?: PluginCliDefinition; hooks?: PluginHooks; manifest: PluginManifest; + /** Permanently versioned data tasks referenced by this plugin's migration journal. */ + migrationTasks?: Record; model?: PluginModelConfig; packageName?: string; tasks?: PluginTasks; diff --git a/packages/junior-scheduler/README.md b/packages/junior-scheduler/README.md index 073710f19..18b08c2ae 100644 --- a/packages/junior-scheduler/README.md +++ b/packages/junior-scheduler/README.md @@ -55,6 +55,10 @@ Junior's durable agent runtime. The plugin exposes create, update, delete, list, and run-now tools plus bounded operational reporting. Generate schema changes with `pnpm --filter @sentry/junior-scheduler db:generate`. +Use `pnpm --filter @sentry/junior-scheduler db:generate:data --name ` +for a TypeScript data migration in the same journal. New migrations should use +the stable migration capabilities directly; the existing scheduler backfill +uses a versioned task to preserve its pre-journal implementation. Follow `../../policies/serverless-background-work.md`, `../../policies/context-bound-systems.md`, and diff --git a/packages/junior-scheduler/package.json b/packages/junior-scheduler/package.json index 43af4ebc2..b39dbff63 100644 --- a/packages/junior-scheduler/package.json +++ b/packages/junior-scheduler/package.json @@ -25,6 +25,7 @@ "scripts": { "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "prepare": "pnpm run build", "prepack": "pnpm run build", "typecheck": "tsc --noEmit" @@ -35,6 +36,7 @@ "zod": "catalog:" }, "devDependencies": { + "@sentry/junior-migrations": "workspace:*", "@types/node": "^25.9.1", "drizzle-kit": "catalog:", "tsup": "^8.5.1", diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index 39d948d82..f72a90dab 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -1,17 +1,16 @@ # SQL migrations `src/db/schema.ts` is the Drizzle schema entrypoint. This directory is the -append-only history used to bring an existing database to that schema. +append-only history used to bring an existing database to that schema. It +extends Drizzle Kit's journal with self-contained +TypeScript data migrations. Drizzle Kit owns numbering, snapshots, and journal +entries; `junior upgrade` executes each entry as either `.sql` or +`.ts` through `@sentry/junior-migrations`. -`junior upgrade` applies core migrations and enabled plugins' packaged Drizzle -migrations. Core SQL is recorded in `drizzle.__drizzle_junior_core`. Reruns -check that journal before taking the migration lock, so an already-current -schema returns without opening a second SQL connection. - -An existing Junior schema without the core Drizzle journal cannot be upgraded -directly. Upgrade it with `@sentry/junior@0.107.1` first so that bridge release -can establish the journal, then continue to the target version. A database with -no Junior tables remains a normal fresh install. +`junior upgrade` runs the ordered migration list in `src/cli/upgrade.ts`. Core +SQL runs under a migration lock and is recorded in +`drizzle.__drizzle_junior_core`, so reruns do not reapply it. Register backfills +after the schema migration they require. ## Add a migration @@ -20,11 +19,26 @@ no Junior tables remains a normal fresh install. 3. Review the generated SQL and commit it with `meta/_journal.json` and its snapshot. -- Put deterministic row transformations in the SQL migration that requires - them. +- Prefer SQL for schema changes and deterministic transformations of rows in + the same database. +- Use an application data migration only for external data or work that must be + decoded in application code; keep it bounded and rerunnable. - Never edit, rename, reorder, or delete an applied SQL migration or its metadata. Add a new migration to correct it. -Migration loading, locking, and the bridge-version guard live in +For a data migration, run +`pnpm --filter @sentry/junior db:generate:data --name `. +TypeScript migrations target the versioned migration capability API and must +not import Junior runtime internals or current feature schemas. Entries that +adopt pre-journal upgrade code call a versioned host task; new data migrations +should use the stable SQL, state, and progress capabilities directly. + +The `0000_initial.sql` baseline represents the schema already deployed by the +pre-Drizzle Junior migration runner. During upgrade, existing installations +adopt that baseline once; new installations execute it normally. All later +migrations are applied by Junior in journal order. Drizzle ORM's stock +`migrate()` function is not compatible with TypeScript entries in this folder. + +Migration loading, locking, and legacy baseline adoption live in `src/chat/conversations/sql/migrations.ts`. Their integration coverage lives in `tests/integration/conversation-sql.test.ts`. diff --git a/packages/junior/package.json b/packages/junior/package.json index d8132d2b8..8ca163d44 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -55,6 +55,7 @@ "prepack": "pnpm run build", "build": "tsup && tsc -p tsconfig.build.json --emitDeclarationOnly", "db:generate": "pnpm exec drizzle-kit generate --config drizzle.config.ts", + "db:generate:data": "pnpm exec junior-migrations generate --config drizzle.config.ts --out migrations", "lint": "oxlint --config .oxlintrc.json --deny-warnings src tests scripts bin tsup.config.ts && depcruise --config .dependency-cruiser.mjs src/chat", "lint:fix": "oxlint --config .oxlintrc.json --deny-warnings --fix src tests scripts bin tsup.config.ts", "test": "vitest run --maxWorkers=4", @@ -74,6 +75,7 @@ "@modelcontextprotocol/sdk": "1.29.0", "@neondatabase/serverless": "^1.1.0", "@sentry/junior-plugin-api": "workspace:*", + "@sentry/junior-migrations": "workspace:*", "@sentry/node": "catalog:", "@sinclair/typebox": "^0.34.49", "@slack/web-api": "^7.16.0", diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index 979029770..aa25310d6 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -1,12 +1,42 @@ /** SQL schema migrations for durable Junior records. */ import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { readMigrationFiles, type MigrationMeta } from "drizzle-orm/migrator"; +import { + resolveMigrations, + runMigrationJournal, + type MigrationContextV1, + type MigrationRunResult, + type MigrationStateV1, + type TypeScriptMigrationLoader, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; -import { isPostgresErrorCode } from "@/db/postgres-error"; import { juniorSqlSchema as schema } from "@/db/schema"; -const CORE_MIGRATION_BRIDGE_VERSION = "0.107.1"; +const LEGACY_CORE_MIGRATION_IDS = [ + "0001_conversation_core", + "0002_slack_destination_visibility_backfill", + "0003_user_identities", + "0004_actor_cutover", + "0005_conversation_transcripts", +] as const; +const LEGACY_METRICS_MIGRATION_ID = "0006_conversation_metrics"; +// Pinned output of the pre-Drizzle runner's SHA-256(statement + NUL) algorithm. +const LEGACY_CORE_MIGRATION_CHECKSUMS = { + "0001_conversation_core": + "78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651", + "0002_slack_destination_visibility_backfill": + "fb590a09fa51db471a748e3d7abb4137f521ee8df97f6e9ef5563121be98c394", + "0003_user_identities": + "67d9c9c26cbd76213614eb6d7a7cc7e2501fc20e92321eb5176a08ce39cd2efb", + "0004_actor_cutover": + "d41b8bfa66b8a88d69e84af38950025ba4c9be56341565cbe1411f0ca50c1dc2", + "0005_conversation_transcripts": + "add299d1b254e023f89b5993c417dd2248dc009e874efdeaf31ec0732e0d4fb4", + "0006_conversation_metrics": + "7c7ca5c9e11ed4b0e14737fd90d3348ea46e306c88fdf31199b7afb2a11c6a41", +} as const; +type LegacyCoreMigrationId = keyof typeof LEGACY_CORE_MIGRATION_CHECKSUMS; const MIGRATIONS_TABLE = "__drizzle_junior_core"; /** Resolve the packaged Drizzle migration directory in source or built output. */ @@ -21,116 +51,207 @@ function migrationFolder(): string { return join(packageRoot, "migrations"); } -interface CoreMigrationState { - appliedAt?: number; - hasJuniorTables: boolean; -} - -interface CoreMigrationResult { - existing: number; - migrated: number; - scanned: number; -} - -/** Read core journal progress and detect existing pre-journal Junior tables. */ -async function loadCoreMigrationState( +async function adoptLegacyMigrationState( executor: JuniorSqlMigrationExecutor, -): Promise { - try { - const [migration] = await executor.query<{ appliedAt: string | null }>(` -SELECT created_at::text AS "appliedAt" -FROM drizzle.__drizzle_junior_core -ORDER BY created_at DESC -LIMIT 1 + migrationsFolder: string, +): Promise { + const [tables] = await executor.query<{ + drizzleTable: string | null; + legacyTable: string | null; + }>(` +SELECT + to_regclass('drizzle.__drizzle_junior_core')::text AS "drizzleTable", + to_regclass('public.junior_schema_migrations')::text AS "legacyTable" `); - if (migration?.appliedAt !== null && migration?.appliedAt !== undefined) { - return { - appliedAt: Number(migration.appliedAt), - hasJuniorTables: true, - }; - } - } catch (error) { - if (!isPostgresErrorCode(error, "42P01")) { - throw error; - } + if (!tables?.legacyTable || tables.drizzleTable) { + return; } - const [tables] = await executor.query<{ hasJuniorTables: boolean }>(` -SELECT EXISTS ( + const migrations = await resolveMigrations(migrationsFolder); + const [metrics] = await executor.query<{ columnCount: number }>(` +SELECT count(*)::integer AS "columnCount" +FROM information_schema.columns +WHERE table_schema = 'public' + AND table_name = 'junior_conversations' + AND column_name IN ( + 'duration_ms', + 'usage_json', + 'execution_duration_ms', + 'execution_usage_json' + ) +`); + const legacyRecords = await executor.query<{ + checksum: string; + id: string; + }>("SELECT id, checksum FROM junior_schema_migrations"); + const legacyRecordsById = new Map( + legacyRecords.map((record) => [record.id, record.checksum]), + ); + const metricColumnCount = metrics?.columnCount ?? 0; + if (metricColumnCount !== 0 && metricColumnCount !== 4) { + throw new Error( + `Cannot adopt partial legacy metrics state: found ${metricColumnCount} of 4 required columns`, + ); + } + const metricsComplete = metricColumnCount === 4; + const hasMetricsRecord = legacyRecordsById.has(LEGACY_METRICS_MIGRATION_ID); + if (metricsComplete !== hasMetricsRecord) { + throw new Error( + "Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns", + ); + } + const expectedIds: readonly LegacyCoreMigrationId[] = metricsComplete + ? [...LEGACY_CORE_MIGRATION_IDS, LEGACY_METRICS_MIGRATION_ID] + : [...LEGACY_CORE_MIGRATION_IDS]; + const missingIds = expectedIds.filter((id) => !legacyRecordsById.has(id)); + if (missingIds.length > 0) { + throw new Error( + `Cannot adopt partial legacy core migration state; missing: ${missingIds.join(", ")}`, + ); + } + const checksumMismatches = expectedIds.filter( + (id) => legacyRecordsById.get(id) !== LEGACY_CORE_MIGRATION_CHECKSUMS[id], + ); + if (checksumMismatches.length > 0) { + throw new Error( + `Cannot adopt legacy core migration state: checksum mismatch: ${checksumMismatches.join(", ")}`, + ); + } + + const [baseline] = await executor.query<{ + conversationEventsTable: string | null; + legacyAgentStepsTable: boolean; + metricRunIdColumn: boolean; + searchIndex: string | null; + }>(` +SELECT + to_regclass('public.junior_conversation_events')::text AS "conversationEventsTable", + to_regclass('public.junior_conversation_messages_search_idx')::text AS "searchIndex", + EXISTS ( SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' - AND table_name LIKE 'junior\\_%' ESCAPE '\\' - ) AS "hasJuniorTables" + AND table_name = 'junior_agent_steps' + AND table_type = 'BASE TABLE' + ) AS "legacyAgentStepsTable", + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'junior_conversations' + AND column_name = 'metric_run_id' + ) AS "metricRunIdColumn" `); - return { - hasJuniorTables: tables?.hasJuniorTables ?? false, - }; -} + if (!baseline?.legacyAgentStepsTable || baseline.conversationEventsTable) { + throw new Error( + "Cannot adopt legacy core migration state: expected the pre-Drizzle junior_agent_steps table and no junior_conversation_events table", + ); + } + const postBaselineMarkers = [ + baseline.searchIndex + ? "junior_conversation_messages_search_idx" + : undefined, + baseline.metricRunIdColumn + ? "junior_conversations.metric_run_id" + : undefined, + ].filter((marker): marker is string => marker !== undefined); + if (postBaselineMarkers.length > 0) { + throw new Error( + `Cannot adopt legacy core migration state: post-baseline schema markers are already present: ${postBaselineMarkers.join(", ")}`, + ); + } -function isCurrentMigrationState( - state: CoreMigrationState, - latestMigrationAt: number, -): boolean { - return state.appliedAt !== undefined && state.appliedAt >= latestMigrationAt; + const migration = metricsComplete ? migrations[1] : migrations[0]; + if (!migration) { + throw new Error("No core Drizzle migrations were packaged"); + } + + await executor.transaction(async () => { + await executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); + await executor.execute(` +CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( + id SERIAL PRIMARY KEY, + hash TEXT NOT NULL, + created_at BIGINT +) +`); + await executor.execute( + `INSERT INTO drizzle.__drizzle_junior_core (hash, created_at) + VALUES ($1, $2)`, + [migration.hash, migration.when], + ); + }); } -function migrationResult( - migrations: readonly MigrationMeta[], - state: CoreMigrationState, -): CoreMigrationResult { - const appliedAt = state.appliedAt; - const existing = - appliedAt === undefined - ? 0 - : migrations.filter((migration) => migration.folderMillis <= appliedAt) - .length; +function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { return { - existing, - migrated: migrations.length - existing, - scanned: migrations.length, + appendToList: async (key, value, options) => { + await stateAdapter.appendToList(key, value, options); + }, + delete: async (key) => { + await stateAdapter.delete(key); + }, + get: async (key) => await stateAdapter.get(key), + getList: async (key) => await stateAdapter.getList(key), + set: async (key, value, ttlMs) => { + await stateAdapter.set(key, value, ttlMs); + }, }; } -/** Reject legacy databases that must first run the bridge release upgrade. */ -function assertSupportedMigrationState(state: CoreMigrationState): void { - if (!state.hasJuniorTables || state.appliedAt !== undefined) { - return; - } - throw new Error( - `Existing Junior SQL tables have no core Drizzle migration history. Stop old Junior workers, install @sentry/junior@${CORE_MIGRATION_BRIDGE_VERSION}, run \`junior upgrade\`, then restore this Junior version and rerun \`junior upgrade\` before restarting workers.`, - ); +/** Project the executor onto the permanent SQL migration capability. */ +function migrationSql( + executor: JuniorSqlMigrationExecutor, +): MigrationContextV1["sql"] { + return { + execute: executor.execute.bind(executor), + query: executor.query.bind(executor), + transaction: executor.transaction.bind(executor), + }; } +export type MigrateSchemaOptions = + | { mode?: "sql" } + | { + loadTypeScript: TypeScriptMigrationLoader; + log?: MigrationContextV1["log"]; + mode: "all"; + runTask: MigrationContextV1["tasks"]["run"]; + stateAdapter: StateAdapter; + }; + export { schema }; -/** Apply the packaged Drizzle migrations during `junior upgrade`. */ +/** Apply the packaged mixed migration journal, or SQL entries alone in tests. */ export async function migrateSchema( executor: JuniorSqlMigrationExecutor, -): Promise { + options: MigrateSchemaOptions = { mode: "sql" }, +): Promise { const migrationsFolder = migrationFolder(); - const migrations = readMigrationFiles({ migrationsFolder }); - const latestMigration = migrations.at(-1); - if (!latestMigration) { - throw new Error("No core Drizzle migrations were packaged"); - } - - const initialState = await loadCoreMigrationState(executor); - if (isCurrentMigrationState(initialState, latestMigration.folderMillis)) { - return migrationResult(migrations, initialState); + const runAll = options.mode === "all"; + const baseOptions = { + beforeRun: async () => { + await adoptLegacyMigrationState(executor, migrationsFolder); + }, + executor, + migrationsFolder, + migrationsTable: MIGRATIONS_TABLE, + }; + if (!runAll) { + return await runMigrationJournal({ ...baseOptions, mode: "sql" }); } - assertSupportedMigrationState(initialState); - - return await executor.withMigrationLock(MIGRATIONS_TABLE, async () => { - const lockedState = await loadCoreMigrationState(executor); - if (isCurrentMigrationState(lockedState, latestMigration.folderMillis)) { - return migrationResult(migrations, lockedState); - } - assertSupportedMigrationState(lockedState); - await executor.migrate({ - migrationsFolder, - migrationsTable: MIGRATIONS_TABLE, - }); - return migrationResult(migrations, lockedState); + return await runMigrationJournal({ + ...baseOptions, + createContext: ({ progress }): MigrationContextV1 => ({ + log: options.log ?? (() => {}), + progress, + sql: migrationSql(executor), + state: migrationState(options.stateAdapter), + tasks: { + run: options.runTask, + }, + }), + loadTypeScript: options.loadTypeScript, + mode: "all", }); } diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index d0073e21f..8a6dfe442 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -1,7 +1,15 @@ import { createHash } from "node:crypto"; -import { readMigrationFiles, type MigrationMeta } from "drizzle-orm/migrator"; +import { + resolveMigrations, + runMigrationJournal, + type MigrationContextV1, + type MigrationJsonValue, + type MigrationStateV1, + type ResolvedMigration, + type TypeScriptMigrationLoader, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; -import { isPostgresErrorCode } from "@/db/postgres-error"; interface PluginMigrationRoot { /** Absolute path to the plugin's Drizzle migrations directory. */ @@ -9,19 +17,25 @@ interface PluginMigrationRoot { pluginName: string; } -interface PluginMigrationResult { +type PluginMigrationResult = { existing: number; migrated: number; scanned: number; -} - -export interface PluginMigrationSummary extends PluginMigrationResult { - pluginName: string; -} + skipped?: number; +}; -interface PluginMigrationOptions { - onPluginMigration?: (summary: PluginMigrationSummary) => void; -} +type PluginMigrationOptions = + | { mode?: "sql" } + | { + loadTypeScript: TypeScriptMigrationLoader; + log?: MigrationContextV1["log"]; + mode: "all"; + runTask?: ( + pluginName: string, + taskName: string, + ) => Promise; + stateAdapter: StateAdapter; + }; const LEGACY_SCHEDULER_BASELINE_HASH = "d1d2f712181dd3a0557808f0fc67fd0722691d25f4c8cfb816b77c71d19e1e42"; @@ -39,28 +53,6 @@ function migrationTable(pluginName: string): string { return `__drizzle_${label}_${hash}`; } -async function appliedMigrationTime( - executor: JuniorSqlMigrationExecutor, - table: string, -): Promise { - try { - const [row] = await executor.query<{ createdAt: string | null }>( - `SELECT created_at::text AS "createdAt" - FROM drizzle.${table} - ORDER BY created_at DESC - LIMIT 1`, - ); - return row?.createdAt === null || row?.createdAt === undefined - ? undefined - : Number(row.createdAt); - } catch (error) { - if (isPostgresErrorCode(error, "42P01")) { - return undefined; - } - throw error; - } -} - async function legacyMigrationHashes( executor: JuniorSqlMigrationExecutor, pluginName: string, @@ -82,13 +74,13 @@ async function legacyMigrationHashes( } function adoptedMigration( - migrations: readonly MigrationMeta[], + migrations: readonly ResolvedMigration[], legacyHashes: ReadonlySet, pluginName: string, -): MigrationMeta | undefined { - let adopted: MigrationMeta | undefined; +): ResolvedMigration | undefined { + let adopted: ResolvedMigration | undefined; for (const migration of migrations) { - if (!legacyHashes.has(migration.hash)) { + if (migration.kind !== "sql" || !legacyHashes.has(migration.hash)) { break; } adopted = migration; @@ -105,10 +97,17 @@ function adoptedMigration( async function adoptLegacyMigrationState(args: { executor: JuniorSqlMigrationExecutor; - migrations: readonly MigrationMeta[]; + migrations: readonly ResolvedMigration[]; pluginName: string; table: string; -}): Promise { +}): Promise { + const [exists] = await args.executor.query<{ tableName: string | null }>( + `SELECT to_regclass($1)::text AS "tableName"`, + [`drizzle.${args.table}`], + ); + if (exists?.tableName) { + return; + } const legacyHashes = await legacyMigrationHashes( args.executor, args.pluginName, @@ -119,7 +118,7 @@ async function adoptLegacyMigrationState(args: { args.pluginName, ); if (!migration) { - return undefined; + return; } await args.executor.transaction(async () => { await args.executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); @@ -132,27 +131,43 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( `); await args.executor.execute( `INSERT INTO drizzle.${args.table} (hash, created_at) VALUES ($1, $2)`, - [migration.hash, migration.folderMillis], + [migration.hash, migration.when], ); }); - return migration.folderMillis; } -function appliedCount( - migrations: readonly MigrationMeta[], - createdAt: number | undefined, -): number { - return createdAt === undefined - ? 0 - : migrations.filter((migration) => migration.folderMillis <= createdAt) - .length; +function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { + return { + appendToList: async (key, value, options) => { + await stateAdapter.appendToList(key, value, options); + }, + delete: async (key) => { + await stateAdapter.delete(key); + }, + get: async (key) => await stateAdapter.get(key), + getList: async (key) => await stateAdapter.getList(key), + set: async (key, value, ttlMs) => { + await stateAdapter.set(key, value, ttlMs); + }, + }; +} + +/** Project the executor onto the permanent SQL migration capability. */ +function migrationSql( + executor: JuniorSqlMigrationExecutor, +): MigrationContextV1["sql"] { + return { + execute: executor.execute.bind(executor), + query: executor.query.bind(executor), + transaction: executor.transaction.bind(executor), + }; } -/** Apply enabled plugins' packaged Drizzle migrations in plugin-name order. */ +/** Apply enabled plugins' mixed migrations in plugin-name and journal order. */ export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, roots: readonly PluginMigrationRoot[], - options: PluginMigrationOptions = {}, + options: PluginMigrationOptions = { mode: "sql" }, ): Promise { const result: PluginMigrationResult = { existing: 0, @@ -163,52 +178,51 @@ export async function migratePluginSchemas( left.pluginName.localeCompare(right.pluginName), ); for (const root of orderedRoots) { - const migrations = readMigrationFiles({ migrationsFolder: root.dir }); - if (migrations.length === 0) { - continue; - } + const migrations = await resolveMigrations(root.dir); const table = migrationTable(root.pluginName); - const initialTime = await appliedMigrationTime(executor, table); - const initialExisting = appliedCount(migrations, initialTime); - if (initialExisting === migrations.length) { - const summary = { - existing: initialExisting, - migrated: 0, - pluginName: root.pluginName, - scanned: migrations.length, - }; - result.scanned += summary.scanned; - result.existing += summary.existing; - options.onPluginMigration?.(summary); - continue; - } - const summary = await executor.withMigrationLock(table, async () => { - const currentTime = - (await appliedMigrationTime(executor, table)) ?? - (await adoptLegacyMigrationState({ + const runAll = options.mode === "all"; + const baseOptions = { + beforeRun: async () => { + await adoptLegacyMigrationState({ executor, migrations, pluginName: root.pluginName, table, - })); - const existing = appliedCount(migrations, currentTime); - if (existing < migrations.length) { - await executor.migrate({ - migrationsFolder: root.dir, - migrationsTable: table, }); - } - return { - existing, - migrated: migrations.length - existing, - pluginName: root.pluginName, - scanned: migrations.length, - }; - }); - result.scanned += summary.scanned; - result.existing += summary.existing; - result.migrated += summary.migrated; - options.onPluginMigration?.(summary); + }, + executor, + migrationsFolder: root.dir, + migrationsTable: table, + }; + const pluginResult = runAll + ? await runMigrationJournal({ + ...baseOptions, + createContext: ({ progress }): MigrationContextV1 => ({ + log: options.log ?? (() => {}), + progress, + sql: migrationSql(executor), + state: migrationState(options.stateAdapter), + tasks: { + run: async (taskName) => { + if (!options.runTask) { + throw new Error( + `Unsupported migration task for plugin ${root.pluginName}: ${taskName}`, + ); + } + return await options.runTask(root.pluginName, taskName); + }, + }, + }), + loadTypeScript: options.loadTypeScript, + mode: "all", + }) + : await runMigrationJournal({ ...baseOptions, mode: "sql" }); + result.scanned += pluginResult.scanned; + result.existing += pluginResult.existing; + result.migrated += pluginResult.migrated; + if (pluginResult.skipped > 0) { + result.skipped = (result.skipped ?? 0) + pluginResult.skipped; + } } return result; } diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 288a352af..225a651c6 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -1,22 +1,19 @@ -/** - * SQL-only upgrade entry point. - * - * It applies core and enabled-plugin Drizzle migrations through one owned - * executor; legacy state conversion belongs to the required bridge release. - */ -import { getChatConfig } from "@/chat/config"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { + disconnectStateAdapter, + getConnectedStateContext, +} from "@/chat/state/adapter"; import { createJiti } from "jiti"; import { loadAppPluginSet } from "@/plugin-module"; -import { migratePluginsToSql } from "./upgrade/migrations/plugin-sql"; +import { migrateCoreJournal } from "./upgrade/migrations/core-journal"; +import { migratePluginJournals } from "./upgrade/migrations/plugin-journal"; +import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; +import { resolveUpgradePlugins } from "./upgrade/migrations/upgrade-plugins"; import type { - MigrationSummary, - UpgradeContext, + MigrationContext, + MigrationResult, UpgradeIo, } from "./upgrade/types"; import { type JuniorPluginSet } from "@/plugins"; -import type { JuniorSqlExecutor } from "@/db/db"; -import { createJuniorSqlExecutor } from "@/db/executor"; const DEFAULT_IO: UpgradeIo = { info: console.log, @@ -56,71 +53,64 @@ export async function resolveUpgradePluginSet(): Promise< ); } -function migrationCount(count: number): string { - return `${count} ${count === 1 ? "migration" : "migrations"}`; -} - -function formatOwnerSummary(owner: string, summary: MigrationSummary): string { - if (summary.migrated === 0) { - return ` ${owner}: up to date (${migrationCount(summary.scanned)})`; +function formatMigrationResult(result: MigrationResult): string { + const fields = [ + `scanned=${result.scanned}`, + `migrated=${result.migrated}`, + `existing=${result.existing}`, + `missing=${result.missing}`, + ]; + if (result.skipped !== undefined) { + fields.push(`skipped=${result.skipped}`); } - return ` ${owner}: applied ${migrationCount(summary.migrated)} (${summary.scanned} total)`; + return fields.join(" "); } -function pluginMigrationOwner(pluginName: string): string { - const unscopedName = pluginName.split("/").at(-1) ?? pluginName; - return unscopedName.startsWith("junior-") - ? unscopedName - : `junior-${unscopedName}`; -} - -async function runDatabaseMigrations( - context: UpgradeContext, - io: UpgradeIo, -): Promise { - io.info("Checking database migrations..."); - const core = await migrateSchema(context.sqlExecutor); - io.info(formatOwnerSummary("junior", core)); - - const plugins = await migratePluginsToSql(context, { - onPluginMigration: (summary) => { - io.info( - formatOwnerSummary(pluginMigrationOwner(summary.pluginName), summary), - ); - }, - }); - const migrated = core.migrated + plugins.migrated; - const scanned = core.scanned + plugins.scanned; - io.info( - migrated === 0 - ? `Database is up to date (${migrationCount(scanned)}).` - : `Applied ${migrationCount(migrated)} (${scanned} total).`, - ); +/** Run all registered upgrade migrations in order. */ +export async function runUpgradeMigrations( + context: MigrationContext, +): Promise { + const plugins = await resolveUpgradePlugins(context); + const migrationContext = { ...context, ...plugins }; + const results: MigrationResult[] = []; + const run = async ( + name: string, + migrate: (context: MigrationContext) => Promise, + ): Promise => { + migrationContext.io.info(`Running migration ${name}...`); + const result = await migrate(migrationContext); + migrationContext.io.info( + `Finished migration ${name}: ${formatMigrationResult(result)}`, + ); + results.push(result); + }; + await run("core-migrations", migrateCoreJournal); + await run("repair-conversation-usage", conversationUsageRepairMigration.run); + await run("plugin-migrations", migratePluginJournals); + return results; } -/** Apply Junior and enabled-plugin database migrations. */ +/** Run one-shot Junior upgrade migrations against the configured state store. */ export async function runUpgrade( io: UpgradeIo = DEFAULT_IO, options: { pluginSet?: JuniorPluginSet | null } = {}, ): Promise { - const { sql } = getChatConfig(); - const sqlExecutor: JuniorSqlExecutor = createJuniorSqlExecutor({ - connectionString: sql.databaseUrl, - driver: sql.driver, - }); try { + const { redisStateAdapter, stateAdapter } = + await getConnectedStateContext(); const pluginSet = options.pluginSet === undefined ? await resolveUpgradePluginSet() : (options.pluginSet ?? undefined); - await runDatabaseMigrations( - { - pluginSet, - sqlExecutor, - }, + io.info("Running Junior upgrade migrations..."); + await runUpgradeMigrations({ io, - ); + pluginSet, + redisStateAdapter, + stateAdapter, + }); + io.info("Junior upgrade complete."); } finally { - await sqlExecutor.close(); + await disconnectStateAdapter(); } } diff --git a/packages/junior/src/cli/upgrade/migrations/core-journal.ts b/packages/junior/src/cli/upgrade/migrations/core-journal.ts new file mode 100644 index 000000000..d753825bb --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/core-journal.ts @@ -0,0 +1,60 @@ +import { getChatConfig } from "@/chat/config"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import type { MigrationJsonValue } from "@sentry/junior-migrations"; +import { createJiti } from "jiti"; +import { migrateAgentTurnSessionActor } from "./agent-turn-session-actor"; +import { migrateConversationHistoryToSql } from "./conversations-history-sql"; +import { migrateConversationsToSql } from "./conversations-sql"; +import { migrateRedisConversationState } from "./redis-conversation-state"; +import type { MigrationContext, MigrationResult } from "../types"; + +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); + +async function runCoreMigrationTask( + context: MigrationContext, + name: string, +): Promise { + switch (name) { + case "agent-turn-session-actor-v1": + return await migrateAgentTurnSessionActor(context); + case "redis-conversation-state-v1": + return await migrateRedisConversationState(context); + case "conversations-to-sql-v1": + return await migrateConversationsToSql(context); + case "conversation-history-to-sql-v1": + return await migrateConversationHistoryToSql(context); + default: + throw new Error(`Unknown core migration task: ${name}`); + } +} + +/** Apply core schema migrations and versioned legacy data tasks in journal order. */ +export async function migrateCoreJournal( + context: MigrationContext, +): Promise { + const { sql } = getChatConfig(); + const executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + try { + const result = await migrateSchema(executor, { + loadTypeScript: async (path) => + await migrationLoader.import>(path), + log: context.io.info, + mode: "all", + runTask: async (name) => await runCoreMigrationTask(context, name), + stateAdapter: context.stateAdapter, + }); + return { + existing: result.existing, + migrated: result.migrated, + missing: 0, + scanned: result.scanned, + skipped: result.skipped, + }; + } finally { + await executor.close(); + } +} diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts new file mode 100644 index 000000000..4ab03e05f --- /dev/null +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -0,0 +1,70 @@ +import { getChatConfig } from "@/chat/config"; +import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; +import { createPluginLogger } from "@/chat/plugins/logging"; +import { createPluginState } from "@/chat/plugins/state"; +import { createJuniorSqlExecutor } from "@/db/executor"; +import { createJiti } from "jiti"; +import { resolveUpgradePlugins } from "./upgrade-plugins"; +import type { MigrationContext, MigrationResult } from "../types"; + +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); + +/** Apply mixed migration journals owned by explicitly enabled plugins. */ +export async function migratePluginJournals( + context: MigrationContext, +): Promise { + const { sql } = getChatConfig(); + const { pluginCatalogConfig, pluginSet } = + await resolveUpgradePlugins(context); + const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig); + const executor = createJuniorSqlExecutor({ + connectionString: sql.databaseUrl, + driver: sql.driver, + }); + try { + const registrations = new Map( + pluginSet + ? pluginSet.registrations.map((plugin) => [ + plugin.manifest.name, + plugin, + ]) + : [], + ); + const result = await migratePluginSchemas( + executor, + pluginCatalogRuntime.getMigrationRoots(), + { + loadTypeScript: async (path) => + await migrationLoader.import>(path), + log: context.io.info, + mode: "all", + runTask: async (pluginName, taskName) => { + const task = + registrations.get(pluginName)?.migrationTasks?.[taskName]; + if (!task) { + throw new Error( + `Plugin ${pluginName} does not provide migration task ${taskName}`, + ); + } + return await task({ + db: context.db ?? executor.db(), + log: createPluginLogger(pluginName), + state: createPluginState(pluginName, context.stateAdapter), + }); + }, + stateAdapter: context.stateAdapter, + }, + ); + return { + existing: result.existing, + migrated: result.migrated, + missing: 0, + scanned: result.scanned, + ...(result.skipped === undefined ? {} : { skipped: result.skipped }), + }; + } finally { + pluginCatalogRuntime.setConfig(previousConfig); + await executor.close(); + } +} diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-sql.ts b/packages/junior/src/cli/upgrade/migrations/plugin-sql.ts deleted file mode 100644 index 474351ab5..000000000 --- a/packages/junior/src/cli/upgrade/migrations/plugin-sql.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - migratePluginSchemas, - type PluginMigrationSummary, -} from "@/chat/plugins/migrations"; -import { createPluginCatalogRuntime } from "@/chat/plugins/registry"; -import { resolveUpgradePluginCatalog } from "./upgrade-plugins"; -import type { MigrationSummary, UpgradeContext } from "../types"; - -/** Apply SQL schema migrations owned by explicitly enabled plugins. */ -export async function migratePluginsToSql( - context: UpgradeContext, - options: { - onPluginMigration?: (summary: PluginMigrationSummary) => void; - } = {}, -): Promise { - const pluginCatalog = createPluginCatalogRuntime(); - pluginCatalog.setConfig(await resolveUpgradePluginCatalog(context)); - return await migratePluginSchemas( - context.sqlExecutor, - pluginCatalog.getMigrationRoots(), - options, - ); -} diff --git a/packages/junior/src/cli/upgrade/types.ts b/packages/junior/src/cli/upgrade/types.ts index fe128abdd..218ce83bf 100644 --- a/packages/junior/src/cli/upgrade/types.ts +++ b/packages/junior/src/cli/upgrade/types.ts @@ -1,19 +1,25 @@ +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import type { StateAdapter } from "chat"; import type { PluginCatalogConfig } from "@/chat/plugins/types"; -import type { JuniorSqlExecutor } from "@/db/db"; import type { JuniorPluginSet } from "@/plugins"; export interface UpgradeIo { info: (line: string) => void; } -export interface UpgradeContext { +export interface MigrationContext { + db?: unknown; + io: UpgradeIo; pluginCatalogConfig?: PluginCatalogConfig; pluginSet?: JuniorPluginSet; - sqlExecutor: JuniorSqlExecutor; + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; } -export interface MigrationSummary { +export type MigrationResult = { existing: number; migrated: number; + missing: number; scanned: number; -} + skipped?: number; +}; diff --git a/packages/junior/src/db/db.ts b/packages/junior/src/db/db.ts index 87bb719bc..36576d8d1 100644 --- a/packages/junior/src/db/db.ts +++ b/packages/junior/src/db/db.ts @@ -7,7 +7,6 @@ */ import type { PgDatabase } from "drizzle-orm/pg-core"; import type { PgQueryResultHKT } from "drizzle-orm/pg-core/session"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { juniorSqlSchema } from "./schema"; export type JuniorDatabase = PgDatabase< @@ -23,7 +22,6 @@ export interface JuniorSqlDatabase { export interface JuniorSqlMigrationExecutor extends JuniorSqlDatabase { execute(statement: string, params?: readonly unknown[]): Promise; - migrate(config: MigrationConfig): Promise; query( statement: string, params?: readonly unknown[], diff --git a/packages/junior/src/db/neon.ts b/packages/junior/src/db/neon.ts index 62e7aa041..0ce11d06a 100644 --- a/packages/junior/src/db/neon.ts +++ b/packages/junior/src/db/neon.ts @@ -6,8 +6,6 @@ import { type QueryResultRow, } from "@neondatabase/serverless"; import { drizzle } from "drizzle-orm/neon-serverless"; -import { migrate } from "drizzle-orm/neon-serverless/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "./db"; import { juniorSqlSchema } from "./schema"; @@ -45,13 +43,6 @@ class NeonExecutor implements NeonJuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate( - drizzle(this.queryClient(), { schema: juniorSqlSchema }), - config, - ); - } - async transaction(callback: () => Promise): Promise { const existingClient = this.transactionClient.getStore(); if (existingClient) { diff --git a/packages/junior/src/db/postgres.ts b/packages/junior/src/db/postgres.ts index 4124bd9ff..53020ce44 100644 --- a/packages/junior/src/db/postgres.ts +++ b/packages/junior/src/db/postgres.ts @@ -5,8 +5,6 @@ import pg, { type QueryResultRow, } from "pg"; import { drizzle } from "drizzle-orm/node-postgres"; -import { migrate } from "drizzle-orm/node-postgres/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "./db"; import { juniorSqlSchema } from "./schema"; @@ -43,13 +41,6 @@ class PostgresExecutor implements JuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate( - drizzle(this.queryClient(), { schema: juniorSqlSchema }), - config, - ); - } - async transaction(callback: () => Promise): Promise { const existingClient = this.transactionClient.getStore(); if (existingClient) { diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index c64d13f2c..021b386d1 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -1,17 +1,110 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import { createJiti } from "jiti"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getChatConfig } from "@/chat/config"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { + appendInboundMessage, + CONVERSATION_ACTIVE_INDEX_KEY, + CONVERSATION_BY_ACTIVITY_INDEX_KEY, + requestConversationWork, +} from "@/chat/task-execution/store"; +import { createSqlStore } from "@/chat/conversations/sql/store"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import type { ConversationStore } from "@/chat/conversations/store"; +import type { PiMessage } from "@/chat/pi/messages"; +import { persistThreadStateById } from "@/chat/runtime/thread-state"; +import { recordAgentTurnSessionSummary } from "@/chat/state/turn-session"; import { resolveUpgradePluginSet } from "@/cli/upgrade"; +import { migrateAgentTurnSessionActor } from "@/cli/upgrade/migrations/agent-turn-session-actor"; +import { repairConversationUsage } from "@/cli/upgrade/migrations/conversation-usage"; +import { migrateConversationsToSql } from "@/cli/upgrade/migrations/conversations-sql"; +import { migrateRedisConversationState } from "@/cli/upgrade/migrations/redis-conversation-state"; +import { + CONVERSATION_ID, + SLACK_DESTINATION, + inboundMessage, +} from "../../fixtures/conversation-work"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; +const ORIGINAL_ENV = vi.hoisted(() => { + const original = { + DATABASE_URL: process.env.DATABASE_URL, + JUNIOR_STATE_ADAPTER: process.env.JUNIOR_STATE_ADAPTER, + }; + process.env.DATABASE_URL = "postgres://configured.example.test/neon"; + process.env.JUNIOR_STATE_ADAPTER = "memory"; + return original; +}); const ORIGINAL_CWD = process.cwd(); +const OTHER_SLACK_DESTINATION = { + ...SLACK_DESTINATION, + channelId: "C999", +} as const; +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); + +const stateOnlyConversationStore: ConversationStore = { + get: async () => undefined, + getDestinationVisibility: async () => undefined, + recordActivity: async () => {}, + recordExecution: async () => {}, + listByActivity: async () => [], +}; -describe("upgrade CLI", () => { - afterEach(() => { +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name]; + return; + } + process.env[name] = value; +} + +async function persistActiveTurn( + conversationId: string, + activeTurnId?: string, +): Promise { + await persistThreadStateById(conversationId, { + conversation: { + schemaVersion: 1, + backfill: {}, + compactions: [], + messages: [], + processing: { + activeTurnId, + }, + stats: { + compactedMessageCount: 0, + estimatedContextTokens: 0, + totalMessageCount: 0, + updatedAtMs: 2_000, + }, + vision: { + byFileId: {}, + }, + }, + }); +} + +describe("upgrade CLI migrations", () => { + beforeEach(async () => { + process.env.DATABASE_URL = "postgres://configured.example.test/neon"; + process.env.JUNIOR_STATE_ADAPTER = "memory"; + await disconnectStateAdapter(); + }); + + afterEach(async () => { process.chdir(ORIGINAL_CWD); + await disconnectStateAdapter(); + restoreEnv("DATABASE_URL", ORIGINAL_ENV.DATABASE_URL); + restoreEnv("JUNIOR_STATE_ADAPTER", ORIGINAL_ENV.JUNIOR_STATE_ADAPTER); + vi.restoreAllMocks(); }); - it("loads source app plugins when virtual config is unavailable", async () => { + it("loads source app plugins for upgrade when virtual config is unavailable", async () => { const tempDir = mkdtempSync(path.join(tmpdir(), "junior-upgrade-plugins-")); writeFileSync( path.join(tempDir, "plugins.ts"), @@ -35,4 +128,1211 @@ export const plugins = { rmSync(tempDir, { force: true, recursive: true }); } }); + + it("migrates legacy requester fields in turn-session state", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const conversationId = "slack:C123:legacy-actor"; + const sessionId = "turn-legacy-actor"; + const requester = { + platform: "slack", + teamId: "T123", + userId: "U123", + userName: "alice", + }; + const summary = { + version: 1, + conversationId, + cumulativeDurationMs: 0, + lastProgressAtMs: 2, + requester, + sessionId, + sliceId: 1, + startedAtMs: 1, + state: "completed", + updatedAtMs: 3, + }; + + await stateAdapter.appendToList( + "junior:agent_turn_session:index", + summary, + { ttlMs: 60_000 }, + ); + await stateAdapter.appendToList( + `junior:agent_turn_session:conversation:${conversationId}:index`, + summary, + { ttlMs: 60_000 }, + ); + await stateAdapter.set( + `junior:agent_turn_session:${conversationId}:${sessionId}`, + { ...summary, committedSeq: -1 }, + 60_000, + ); + + await expect( + migrateAgentTurnSessionActor({ + io: { info: () => {} }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 3, + missing: 0, + scanned: 3, + }); + + const { listBoundedAgentTurnSessionSummariesForConversation } = + await import("@/chat/state/turn-session"); + await expect( + listBoundedAgentTurnSessionSummariesForConversation(conversationId), + ).resolves.toEqual([ + expect.objectContaining({ actor: requester, sessionId }), + ]); + const migratedRecord = await stateAdapter.get( + `junior:agent_turn_session:${conversationId}:${sessionId}`, + ); + expect(migratedRecord).toEqual( + expect.objectContaining({ actor: requester }), + ); + expect(migratedRecord).not.toHaveProperty("requester"); + + await expect( + migrateAgentTurnSessionActor({ + io: { info: () => {} }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 1, + migrated: 0, + missing: 0, + scanned: 3, + }); + }); + + it("discovers legacy turn sessions outside the bounded global index", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const conversationId = "slack:C123:older-legacy-actor"; + const sessionId = "turn-older-legacy-actor"; + const requester = { + platform: "slack", + teamId: "T123", + userId: "U123", + }; + const summary = { + version: 1, + conversationId, + cumulativeDurationMs: 0, + lastProgressAtMs: 2, + requester, + sessionId, + sliceId: 1, + startedAtMs: 1, + state: "completed", + updatedAtMs: 3, + }; + + await stateAdapter.appendToList( + `junior:agent_turn_session:conversation:${conversationId}:index`, + summary, + { ttlMs: 60_000 }, + ); + await stateAdapter.set( + `junior:agent_turn_session:${conversationId}:${sessionId}`, + { ...summary, committedSeq: -1 }, + 60_000, + ); + + const statePrefix = getChatConfig().state.keyPrefix; + const redisConversationIndexKey = [ + "chat-sdk:list", + ...(statePrefix ? [statePrefix] : []), + `junior:agent_turn_session:conversation:${conversationId}:index`, + ].join(":"); + const redisStateAdapter = { + getClient: () => ({ + sendCommand: async (args: readonly string[]) => { + expect(args).toEqual([ + "SCAN", + "0", + "MATCH", + `*:list:${statePrefix ? `${statePrefix}:` : ""}junior:agent_turn_session:conversation:*:index`, + "COUNT", + "500", + ]); + return ["0", [redisConversationIndexKey]]; + }, + }), + } as unknown as RedisStateAdapter; + + await expect( + migrateAgentTurnSessionActor({ + io: { info: () => {} }, + redisStateAdapter, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 2, + missing: 0, + scanned: 2, + }); + + const { listBoundedAgentTurnSessionSummariesForConversation } = + await import("@/chat/state/turn-session"); + await expect( + listBoundedAgentTurnSessionSummariesForConversation(conversationId), + ).resolves.toEqual([ + expect.objectContaining({ actor: requester, sessionId }), + ]); + await expect( + stateAdapter.get( + `junior:agent_turn_session:${conversationId}:${sessionId}`, + ), + ).resolves.toEqual(expect.objectContaining({ actor: requester })); + }); + it("runs journaled TypeScript state migrations exactly once", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const actor = { platform: "api", userId: "migration-user" }; + const summary = { + conversationId: CONVERSATION_ID, + sessionId: "session-one", + requester: actor, + }; + await stateAdapter.appendToList("junior:agent_turn_session:index", summary); + await stateAdapter.appendToList( + `junior:agent_turn_session:conversation:${CONVERSATION_ID}:index`, + summary, + ); + await stateAdapter.set( + `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, + summary, + ); + + try { + await expect( + migrateSchema(fixture.sql, { + loadTypeScript: async (migrationPath) => + await migrationLoader.import>( + migrationPath, + ), + mode: "all", + runTask: async (name) => { + if (name === "agent-turn-session-actor-v1") { + return await migrateAgentTurnSessionActor({ + io: { info: () => {} }, + stateAdapter, + }); + } + }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 8, + scanned: 8, + skipped: 0, + }); + await expect( + stateAdapter.getList("junior:agent_turn_session:index"), + ).resolves.toEqual([ + { + actor, + conversationId: CONVERSATION_ID, + sessionId: "session-one", + }, + ]); + await expect( + stateAdapter.get( + `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, + ), + ).resolves.toEqual({ + actor, + conversationId: CONVERSATION_ID, + sessionId: "session-one", + }); + await expect( + migrateSchema(fixture.sql, { + loadTypeScript: async (migrationPath) => + await migrationLoader.import>( + migrationPath, + ), + mode: "all", + runTask: async () => undefined, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 8, + migrated: 0, + scanned: 8, + skipped: 0, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("migrates legacy conversation work before SQL conversation backfill", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const legacyMessage = inboundMessage("legacy-sql"); + await stateAdapter.set( + `junior:conversation-work:state:${CONVERSATION_ID}`, + { + schemaVersion: 1, + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + messages: [legacyMessage], + needsRun: true, + updatedAtMs: 2_000, + }, + ); + await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + + try { + await migrateSchema(fixture.sql); + const context = { + io: { info: () => {} }, + stateAdapter, + }; + const results = [ + await migrateRedisConversationState(context), + await migrateConversationsToSql(context, { target: sqlStore }), + ]; + + expect(results).toEqual([ + { + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }, + { + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }, + ]); + await expect( + stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), + ).resolves.toMatchObject({ + conversationId: CONVERSATION_ID, + execution: { + inboundMessageIds: ["legacy-sql"], + pendingCount: 1, + status: "pending", + }, + }); + const sqlConversation = await sqlStore.get({ + conversationId: CONVERSATION_ID, + }); + expect(sqlConversation).toMatchObject({ + conversationId: CONVERSATION_ID, + execution: { + status: "pending", + }, + }); + expect(sqlConversation?.execution).not.toHaveProperty("pendingCount"); + expect(sqlConversation?.execution).not.toHaveProperty("pendingMessages"); + } finally { + await fixture.close(); + } + }, 15_000); + + it("copies a bounded SQL conversation backfill slice", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + + try { + await migrateSchema(fixture.sql); + for (let index = 0; index < 3; index++) { + const conversationId = `slack:C123:page-${index}`; + await appendInboundMessage({ + message: inboundMessage(`page-${index}`, { conversationId }), + nowMs: 1_000 + index, + conversationStore: stateOnlyConversationStore, + state: stateAdapter, + }); + } + + await expect( + migrateConversationsToSql( + { + io: { info: () => {} }, + stateAdapter, + }, + { batchSize: 2, target: sqlStore }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 2, + missing: 0, + scanned: 2, + }); + await expect(sqlStore.listByActivity({ limit: 10 })).resolves.toEqual([ + expect.objectContaining({ conversationId: "slack:C123:page-2" }), + expect.objectContaining({ conversationId: "slack:C123:page-1" }), + ]); + } finally { + await fixture.close(); + } + }, 15_000); + + it("backfills retained conversation metrics without replacing SQL totals", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + + try { + await migrateSchema(fixture.sql); + const seedMs = Date.now() - 1_000; + await sqlStore.recordExecution({ + conversationId: CONVERSATION_ID, + createdAtMs: seedMs, + destination: SLACK_DESTINATION, + execution: { + runId: "run-two", + status: "idle", + updatedAtMs: seedMs, + }, + lastActivityAtMs: seedMs, + metrics: null, + updatedAtMs: seedMs, + }); + await recordAgentTurnSessionSummary({ + conversationId: CONVERSATION_ID, + cumulativeDurationMs: 1_000, + cumulativeUsage: { + inputTokens: 40, + outputTokens: 10, + reasoningTokens: 3, + cost: { total: 0.001 }, + }, + destination: SLACK_DESTINATION, + conversationStore: stateOnlyConversationStore, + sessionId: "run-one", + sliceId: 1, + state: "completed", + surface: "slack", + }); + await recordAgentTurnSessionSummary({ + conversationId: CONVERSATION_ID, + cumulativeDurationMs: 2_000, + cumulativeUsage: { + inputTokens: 80, + outputTokens: 20, + reasoningTokens: 7, + cost: { total: 0.002 }, + }, + destination: SLACK_DESTINATION, + conversationStore: stateOnlyConversationStore, + sessionId: "run-two", + sliceId: 1, + state: "completed", + surface: "slack", + }); + + await migrateConversationsToSql( + { io: { info: () => {} }, stateAdapter }, + { target: sqlStore }, + ); + + const readMetrics = async () => { + const [row] = await fixture.sql.query<{ + durationMs: number; + executionDurationMs: number; + executionUsage: { + inputTokens?: number; + outputTokens?: number; + } | null; + usage: { + cost?: { total?: number }; + inputTokens?: number; + outputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + } | null; + }>( + ` +SELECT + duration_ms AS "durationMs", + usage_json AS usage, + execution_duration_ms AS "executionDurationMs", + execution_usage_json AS "executionUsage" +FROM junior_conversations +WHERE conversation_id = $1 +`, + [CONVERSATION_ID], + ); + return row; + }; + await expect(readMetrics()).resolves.toMatchObject({ + durationMs: 3_000, + executionDurationMs: 2_000, + executionUsage: { + inputTokens: 80, + outputTokens: 20, + }, + usage: { + inputTokens: 120, + outputTokens: 30, + reasoningTokens: 10, + cost: { total: 0.003 }, + }, + }); + + const futureMs = Date.now() + 60_000; + await sqlStore.recordExecution({ + conversationId: CONVERSATION_ID, + createdAtMs: futureMs, + execution: { + runId: "run-three", + status: "idle", + updatedAtMs: futureMs, + }, + lastActivityAtMs: futureMs, + metrics: { + durationMs: 500, + usage: { + inputTokens: 5, + outputTokens: 5, + reasoningTokens: 1, + cost: { total: 0.0005 }, + }, + }, + updatedAtMs: futureMs, + }); + await migrateConversationsToSql( + { io: { info: () => {} }, stateAdapter }, + { target: sqlStore }, + ); + + await expect(readMetrics()).resolves.toMatchObject({ + durationMs: 3_500, + executionDurationMs: 500, + usage: { + reasoningTokens: 11, + totalTokens: 160, + cost: { total: 0.0035 }, + }, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("repairs SQL usage in bounded batches without changing duration", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); + const mixedConversationId = `${CONVERSATION_ID}:mixed`; + const unsafeConversationId = `${CONVERSATION_ID}:unsafe`; + const firstAssistant = { + role: "assistant", + content: [{ type: "text", text: "first" }], + api: "responses", + provider: "openai", + model: "test-model", + stopReason: "stop", + timestamp: 2_000, + usage: { + input: 10, + output: 2, + cacheRead: 3, + cacheWrite: 1, + reasoning: 1, + totalTokens: 16, + cost: { + input: 0.001, + output: 0.002, + cacheRead: 0.0003, + cacheWrite: 0.0004, + total: 0.0037, + }, + }, + } as PiMessage; + const secondAssistant = { + role: "assistant", + content: [{ type: "text", text: "second" }], + api: "responses", + provider: "openai", + model: "test-model", + stopReason: "stop", + timestamp: 3_000, + usage: { + input: 20, + output: 5, + cacheRead: 0, + cacheWrite: 0, + reasoning: 2, + totalTokens: 25, + cost: { + input: 0.004, + output: 0.006, + cacheRead: 0, + cacheWrite: 0, + total: 0.01, + }, + }, + } as PiMessage; + + try { + await migrateSchema(fixture.sql); + await sqlStore.recordExecution({ + conversationId: CONVERSATION_ID, + createdAtMs: 1_000, + execution: { + runId: "run-two", + status: "idle", + updatedAtMs: 4_000, + }, + lastActivityAtMs: 4_000, + metrics: { + durationMs: 9_000, + usage: { totalTokens: 999, cost: { total: 0.999 } }, + }, + updatedAtMs: 4_000, + }); + await eventStore.append(CONVERSATION_ID, [ + { + data: { type: "agent_step", message: firstAssistant }, + createdAtMs: 2_000, + }, + ]); + await eventStore.replaceHistory(CONVERSATION_ID, { + createdAtMs: 2_500, + data: { + type: "compaction", + modelProfile: "standard", + modelId: "test-model", + replacementHistory: [{ message: firstAssistant }], + }, + }); + await eventStore.append(CONVERSATION_ID, [ + { + data: { type: "agent_step", message: secondAssistant }, + createdAtMs: 3_000, + }, + ]); + await sqlStore.recordExecution({ + conversationId: mixedConversationId, + createdAtMs: 1_000, + execution: { status: "idle", updatedAtMs: 4_000 }, + lastActivityAtMs: 4_000, + metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, + updatedAtMs: 4_000, + }); + await eventStore.append(mixedConversationId, [ + { + data: { + type: "agent_step", + message: { + role: "assistant", + usage: { input: 3, totalTokens: 999 }, + } as PiMessage, + }, + createdAtMs: 2_000, + }, + { + data: { + type: "agent_step", + message: { + role: "assistant", + usage: { totalTokens: 5 }, + } as PiMessage, + }, + createdAtMs: 3_000, + }, + { + data: { + type: "agent_step", + message: { + role: "user", + usage: { input: 1_000, output: 1_000 }, + } as unknown as PiMessage, + }, + createdAtMs: 3_500, + }, + ]); + await sqlStore.recordExecution({ + conversationId: unsafeConversationId, + createdAtMs: 1_000, + execution: { status: "idle", updatedAtMs: 4_000 }, + lastActivityAtMs: 4_000, + metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, + updatedAtMs: 4_000, + }); + await eventStore.append(unsafeConversationId, [ + { + data: { + type: "agent_step", + message: { + role: "assistant", + usage: { input: 9_007_199_254_740_992 }, + } as PiMessage, + }, + createdAtMs: 3_000, + }, + ]); + const context = { io: { info: vi.fn() }, stateAdapter }; + await expect( + repairConversationUsage(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 2, + missing: 1, + scanned: 3, + }); + const [metrics] = await fixture.sql.query<{ + durationMs: number; + executionDurationMs: number; + usage: Record; + }>( + ` +SELECT + duration_ms AS "durationMs", + execution_duration_ms AS "executionDurationMs", + usage_json AS usage +FROM junior_conversations +WHERE conversation_id = $1 +`, + [CONVERSATION_ID], + ); + expect(metrics).toEqual({ + durationMs: 9_000, + executionDurationMs: 9_000, + usage: { + inputTokens: 30, + outputTokens: 7, + cachedInputTokens: 3, + cacheCreationTokens: 1, + reasoningTokens: 3, + cost: { + input: 0.005, + output: 0.008, + cacheRead: 0.0003, + cacheWrite: 0.0004, + total: 0.0137, + }, + }, + }); + const [mixed] = await fixture.sql.query<{ usage: unknown }>( + `SELECT usage_json AS usage FROM junior_conversations WHERE conversation_id = $1`, + [mixedConversationId], + ); + expect(mixed?.usage).toEqual({ totalTokens: 8 }); + const [unsafe] = await fixture.sql.query<{ usage: unknown }>( + `SELECT usage_json AS usage FROM junior_conversations WHERE conversation_id = $1`, + [unsafeConversationId], + ); + expect(unsafe?.usage).toEqual({ totalTokens: 777 }); + + await expect( + repairConversationUsage(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toEqual({ + existing: 2, + migrated: 0, + missing: 1, + scanned: 3, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("commits completed usage batches before a later batch fails", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); + const conversationIds = ["local:usage-batch-a", "local:usage-batch-b"]; + + try { + await migrateSchema(fixture.sql); + for (const [index, conversationId] of conversationIds.entries()) { + await sqlStore.recordExecution({ + conversationId, + createdAtMs: 1_000, + execution: { status: "idle", updatedAtMs: 2_000 }, + lastActivityAtMs: 2_000, + metrics: { durationMs: 1_000, usage: { totalTokens: 999 } }, + updatedAtMs: 2_000, + }); + await eventStore.append(conversationId, [ + { + data: { + type: "agent_step", + message: { + role: "assistant", + usage: { input: index + 1, totalTokens: index + 1 }, + } as PiMessage, + }, + createdAtMs: 2_000, + }, + ]); + } + + let queryCount = 0; + const failingExecutor = new Proxy(fixture.sql, { + get(target, key, receiver) { + if (key === "query") { + return async ( + statement: string, + params: readonly unknown[] = [], + ) => { + queryCount += 1; + if (queryCount === 2) { + throw new Error("later usage batch failed"); + } + return target.query(statement, params); + }; + } + const value = Reflect.get(target, key, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const context = { io: { info: vi.fn() }, stateAdapter }; + await expect( + repairConversationUsage(context, { + batchSize: 1, + executor: failingExecutor, + }), + ).rejects.toThrow("later usage batch failed"); + + const beforeRetry = await fixture.sql.query<{ + conversationId: string; + usage: unknown; + }>( + `SELECT conversation_id AS "conversationId", usage_json AS usage + FROM junior_conversations ORDER BY conversation_id`, + ); + expect(beforeRetry).toEqual([ + { + conversationId: conversationIds[0], + usage: { inputTokens: 1 }, + }, + { conversationId: conversationIds[1], usage: { totalTokens: 999 } }, + ]); + await expect( + repairConversationUsage(context, { + batchSize: 1, + executor: fixture.sql, + }), + ).resolves.toEqual({ + existing: 1, + migrated: 1, + missing: 0, + scanned: 2, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("reports version-guard misses as skipped", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + + try { + let queryCount = 0; + const skippedExecutor = new Proxy(fixture.sql, { + get(target, key, receiver) { + if (key === "query") { + return async () => { + queryCount += 1; + return queryCount === 1 + ? [ + { + changed: false, + conversationId: CONVERSATION_ID, + matched: false, + repairable: true, + }, + ] + : []; + }; + } + const value = Reflect.get(target, key, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + await expect( + repairConversationUsage( + { io: { info: vi.fn() }, stateAdapter }, + { executor: skippedExecutor }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 0, + missing: 0, + scanned: 1, + skipped: 1, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("repairs a conversation on rerun after it becomes idle", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + const eventStore = createSqlConversationEventStore(fixture.sql); + + try { + await migrateSchema(fixture.sql); + await sqlStore.recordExecution({ + conversationId: CONVERSATION_ID, + createdAtMs: 1_000, + execution: { + runId: "active-run", + status: "running", + updatedAtMs: 2_000, + }, + lastActivityAtMs: 2_000, + metrics: { + durationMs: 7_777, + usage: { totalTokens: 999, cost: { total: 0.999 } }, + }, + updatedAtMs: 2_000, + }); + await eventStore.append(CONVERSATION_ID, [ + { + data: { + type: "agent_step", + message: { + role: "assistant", + usage: { input: 4, output: 2, totalTokens: 6 }, + } as PiMessage, + }, + createdAtMs: 2_000, + }, + ]); + const context = { io: { info: vi.fn() }, stateAdapter }; + await expect( + repairConversationUsage(context, { executor: fixture.sql }), + ).resolves.toEqual({ + existing: 0, + migrated: 0, + missing: 0, + scanned: 0, + }); + + await fixture.sql.execute( + `UPDATE junior_conversations SET execution_status = 'idle' WHERE conversation_id = $1`, + [CONVERSATION_ID], + ); + await expect( + repairConversationUsage(context, { executor: fixture.sql }), + ).resolves.toEqual({ + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }); + const [metrics] = await fixture.sql.query<{ + durationMs: number; + usage: Record; + }>( + `SELECT duration_ms AS "durationMs", usage_json AS usage + FROM junior_conversations WHERE conversation_id = $1`, + [CONVERSATION_ID], + ); + expect(metrics).toEqual({ + durationMs: 7_777, + usage: { inputTokens: 4, outputTokens: 2 }, + }); + } finally { + await fixture.close(); + } + }, 15_000); + + it("seeds active awaiting continuations into conversation work", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await recordAgentTurnSessionSummary({ + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + resumeReason: "timeout", + sessionId: "turn-timeout", + sliceId: 2, + state: "awaiting_resume", + conversationStore: stateOnlyConversationStore, + }); + await persistActiveTurn(CONVERSATION_ID, "turn-timeout"); + + await expect( + migrateRedisConversationState({ + io: { info: () => {} }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }); + await expect( + stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), + ).resolves.toMatchObject({ + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + execution: { + pendingCount: 0, + pendingMessages: [], + status: "pending", + }, + }); + await expect( + stateAdapter.get(CONVERSATION_ACTIVE_INDEX_KEY), + ).resolves.toEqual([ + { + conversationId: CONVERSATION_ID, + score: expect.any(Number), + }, + ]); + }); + + it("merges legacy pending work when the conversation record already exists", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await requestConversationWork({ + conversationId: CONVERSATION_ID, + conversationStore: stateOnlyConversationStore, + destination: SLACK_DESTINATION, + nowMs: 2_000, + state: stateAdapter, + }); + await stateAdapter.delete(CONVERSATION_BY_ACTIVITY_INDEX_KEY); + await stateAdapter.delete(CONVERSATION_ACTIVE_INDEX_KEY); + await stateAdapter.set( + `junior:conversation-work:state:${CONVERSATION_ID}`, + { + schemaVersion: 1, + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + messages: [inboundMessage("m1")], + needsRun: true, + updatedAtMs: 3_000, + }, + ); + await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); + + await expect( + migrateRedisConversationState({ + io: { info: () => {} }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 1, + migrated: 0, + missing: 0, + scanned: 1, + }); + await expect( + stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), + ).resolves.toBeNull(); + await expect( + stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), + ).resolves.toMatchObject({ + conversationId: CONVERSATION_ID, + lastActivityAtMs: 2_000, + updatedAtMs: 3_000, + execution: { + inboundMessageIds: ["m1"], + pendingCount: 1, + pendingMessages: [expect.objectContaining({ inboundMessageId: "m1" })], + status: "pending", + updatedAtMs: 3_000, + }, + }); + await expect( + stateAdapter.get("junior:conversation-work:index"), + ).resolves.toBeNull(); + await expect( + stateAdapter.get(CONVERSATION_BY_ACTIVITY_INDEX_KEY), + ).resolves.toEqual([ + { + conversationId: CONVERSATION_ID, + score: 2_000, + }, + ]); + await expect( + stateAdapter.get(CONVERSATION_ACTIVE_INDEX_KEY), + ).resolves.toEqual([ + { + conversationId: CONVERSATION_ID, + score: 3_000, + }, + ]); + }); + + it("does not merge legacy pending work with a different destination", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await requestConversationWork({ + conversationId: CONVERSATION_ID, + conversationStore: stateOnlyConversationStore, + destination: SLACK_DESTINATION, + nowMs: 2_000, + state: stateAdapter, + }); + await stateAdapter.set( + `junior:conversation-work:state:${CONVERSATION_ID}`, + { + schemaVersion: 1, + conversationId: CONVERSATION_ID, + destination: OTHER_SLACK_DESTINATION, + messages: [ + { + ...inboundMessage("m1"), + destination: OTHER_SLACK_DESTINATION, + }, + ], + needsRun: true, + updatedAtMs: 3_000, + }, + ); + await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); + + await expect( + migrateRedisConversationState({ + io: { info: () => {} }, + stateAdapter, + }), + ).rejects.toThrow( + `Legacy conversation work destination does not match conversation ${CONVERSATION_ID}`, + ); + await expect( + stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), + ).resolves.toEqual(expect.objectContaining({ needsRun: true })); + }); + + it("rejects legacy pending work with a different message destination", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await stateAdapter.set( + `junior:conversation-work:state:${CONVERSATION_ID}`, + { + schemaVersion: 1, + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + messages: [ + { + ...inboundMessage("m1"), + destination: OTHER_SLACK_DESTINATION, + }, + ], + needsRun: true, + updatedAtMs: 3_000, + }, + ); + await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); + + await expect( + migrateRedisConversationState({ + io: { info: () => {} }, + stateAdapter, + }), + ).rejects.toThrow( + `Legacy conversation work state is invalid for ${CONVERSATION_ID}`, + ); + await expect( + stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), + ).resolves.toEqual(expect.objectContaining({ needsRun: true })); + }); + + it("ignores malformed legacy conversation work indexes", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await stateAdapter.set("junior:conversation-work:index", { + conversationId: CONVERSATION_ID, + }); + + await expect( + migrateRedisConversationState({ + io: { info: () => {} }, + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 0, + missing: 0, + scanned: 0, + }); + }); + + it("backfills retained conversation record into SQL when configured", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + await requestConversationWork({ + conversationId: CONVERSATION_ID, + conversationStore: stateOnlyConversationStore, + destination: SLACK_DESTINATION, + nowMs: 2_000, + state: stateAdapter, + }); + const fixture = await createLocalJuniorSqlFixture(); + const sqlStore = createSqlStore(fixture.sql); + + try { + await migrateSchema(fixture.sql); + const context = { + io: { info: () => {} }, + stateAdapter, + }; + const results = [ + await migrateRedisConversationState(context), + await migrateConversationsToSql(context, { target: sqlStore }), + ]; + + expect(results).toEqual([ + { + existing: 0, + migrated: 0, + missing: 0, + scanned: 0, + }, + { + existing: 0, + migrated: 1, + missing: 0, + scanned: 1, + }, + ]); + const sqlConversation = await sqlStore.get({ + conversationId: CONVERSATION_ID, + }); + expect(sqlConversation).toMatchObject({ + conversationId: CONVERSATION_ID, + destination: SLACK_DESTINATION, + execution: { + status: "pending", + }, + }); + expect(sqlConversation?.execution).not.toHaveProperty("pendingCount"); + } finally { + await fixture.close(); + } + }, 15_000); }); diff --git a/packages/junior/tests/component/memory-plugin-storage.test.ts b/packages/junior/tests/component/memory-plugin-storage.test.ts index 914a113ce..d9600d282 100644 --- a/packages/junior/tests/component/memory-plugin-storage.test.ts +++ b/packages/junior/tests/component/memory-plugin-storage.test.ts @@ -1,5 +1,7 @@ import path from "node:path"; import { readdirSync } from "node:fs"; +import { createMemoryState } from "@chat-adapter/state-memory"; +import { resolveMigrations } from "@sentry/junior-migrations"; import { afterAll, describe, expect, it, vi } from "vitest"; import { createMemoryPlugin, @@ -8,16 +10,13 @@ import { } from "@sentry/junior-memory"; import { createSlackSource, - defineJuniorPlugin, PluginToolInputError, } from "@sentry/junior-plugin-api"; import { defineJuniorPlugins } from "@/plugins"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; import { migratePluginSchemas } from "@/chat/plugins/migrations"; -import { readMigrationFiles } from "drizzle-orm/migrator"; import { closeDb } from "@/chat/db"; -import { migratePluginsToSql } from "@/cli/upgrade/migrations/plugin-sql"; -import { runUpgrade } from "@/cli/upgrade"; +import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const NEON = vi.hoisted(() => ({ @@ -40,7 +39,6 @@ vi.mock("@/db/executor", () => ({ db: NEON.sql.db.bind(NEON.sql), execute: NEON.sql.execute.bind(NEON.sql), query: NEON.sql.query.bind(NEON.sql), - migrate: NEON.sql.migrate.bind(NEON.sql), transaction: NEON.sql.transaction.bind(NEON.sql), withLock: NEON.sql.withLock.bind(NEON.sql), withMigrationLock: NEON.sql.withMigrationLock.bind(NEON.sql), @@ -99,9 +97,7 @@ async function migrateMemorySchema( describe("memory plugin host wiring", () => { it("adopts exact legacy migration hashes without replaying them", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrations = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }); + const migrations = await resolveMigrations(memoryMigrationsDir()); const migrationFiles = memoryMigrationFiles(); expect(migrationFiles).toHaveLength(migrations.length); @@ -150,9 +146,8 @@ CREATE TABLE junior_schema_migrations ( it("does not adopt an unknown memory legacy checksum", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrationCount = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }).length; + const migrationCount = (await resolveMigrations(memoryMigrationsDir())) + .length; const [baselineFile] = memoryMigrationFiles(); expect(baselineFile).toBeDefined(); @@ -187,21 +182,24 @@ CREATE TABLE junior_schema_migrations ( }); it("applies packaged migrations through plugin discovery", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); NEON.sql = fixture.sql; try { - const migrationCount = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }).length; + const migrationCount = (await resolveMigrations(memoryMigrationsDir())) + .length; await expect( - migratePluginsToSql({ + migratePluginJournals({ + io: { info: () => {} }, pluginSet: defineJuniorPlugins([createMemoryPlugin()]), - sqlExecutor: fixture.sql, + stateAdapter, }), ).resolves.toEqual({ existing: 0, migrated: migrationCount, + missing: 0, scanned: migrationCount, }); @@ -216,52 +214,7 @@ WHERE table_name = 'junior_memory_memories' ).resolves.toEqual([{ table_name: "junior_memory_memories" }]); } finally { NEON.sql = undefined; - await fixture.close(); - } - }, 15_000); - - it("reports core and nonempty plugin migration journals", async () => { - const fixture = await createLocalJuniorSqlFixture(); - NEON.sql = fixture.sql; - - try { - const coreMigrationCount = readMigrationFiles({ - migrationsFolder: path.resolve(process.cwd(), "migrations"), - }).length; - const memoryMigrationCount = readMigrationFiles({ - migrationsFolder: memoryMigrationsDir(), - }).length; - const totalMigrationCount = coreMigrationCount + memoryMigrationCount; - const lines: string[] = []; - const pluginSet = defineJuniorPlugins([ - createMemoryPlugin(), - defineJuniorPlugin({ - manifest: { - description: "Plugin without SQL migrations", - displayName: "Empty", - name: "empty", - }, - }), - ]); - - await runUpgrade({ info: (line) => lines.push(line) }, { pluginSet }); - expect(lines).toEqual([ - "Checking database migrations...", - ` junior: applied ${coreMigrationCount} migrations (${coreMigrationCount} total)`, - ` junior-memory: applied ${memoryMigrationCount} migrations (${memoryMigrationCount} total)`, - `Applied ${totalMigrationCount} migrations (${totalMigrationCount} total).`, - ]); - - lines.length = 0; - await runUpgrade({ info: (line) => lines.push(line) }, { pluginSet }); - expect(lines).toEqual([ - "Checking database migrations...", - ` junior: up to date (${coreMigrationCount} migrations)`, - ` junior-memory: up to date (${memoryMigrationCount} migrations)`, - `Database is up to date (${totalMigrationCount} migrations).`, - ]); - } finally { - NEON.sql = undefined; + await stateAdapter.disconnect(); await fixture.close(); } }, 15_000); diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index df9a50247..8a015530e 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -1,19 +1,53 @@ import path from "node:path"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { readMigrationFiles } from "drizzle-orm/migrator"; -import { describe, expect, it, vi } from "vitest"; +import { createMemoryState } from "@chat-adapter/state-memory"; +import { resolveMigrations } from "@sentry/junior-migrations"; +import { defineJuniorPlugin } from "@sentry/junior-plugin-api"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { createSchedulerSqlStore, schedulerPlugin, type SchedulerDb, type ScheduledTask, } from "@sentry/junior-scheduler"; +import { createSchedulerStore } from "../../../junior-scheduler/src/store"; import { defineJuniorPlugins } from "@/plugins"; import { migratePluginSchemas } from "@/chat/plugins/migrations"; -import { migratePluginsToSql } from "@/cli/upgrade/migrations/plugin-sql"; +import { createPluginState } from "@/chat/plugins/state"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; +const NEON = vi.hoisted(() => ({ + originalDatabaseUrl: process.env.DATABASE_URL, + sql: undefined as + | Awaited>["sql"] + | undefined, +})); + +vi.hoisted(() => { + process.env.DATABASE_URL = "postgres://configured.example.test/neon"; + process.env.JUNIOR_STATE_ADAPTER = "memory"; +}); + +vi.mock("@/db/executor", () => ({ + createJuniorSqlExecutor: vi.fn(() => { + if (!NEON.sql) { + throw new Error("Missing test SQL executor"); + } + return { + db: NEON.sql.db.bind(NEON.sql), + execute: NEON.sql.execute.bind(NEON.sql), + query: NEON.sql.query.bind(NEON.sql), + transaction: NEON.sql.transaction.bind(NEON.sql), + withLock: NEON.sql.withLock.bind(NEON.sql), + withMigrationLock: NEON.sql.withMigrationLock.bind(NEON.sql), + close: async () => {}, + }; + }), +})); + const TEST_RUN_AT_MS = Date.parse("2026-05-26T12:00:00.000Z"); const TEST_NOW_MS = Date.parse("2026-05-26T12:05:00.000Z"); const LEGACY_SCHEDULER_MIGRATION_CHECKSUM = @@ -38,6 +72,22 @@ async function migrateSchedulerSchema( ]); } +async function runSchedulerMigrationTask(args: { + db: SchedulerDb; + stateAdapter: ReturnType; +}) { + const plugin = schedulerPlugin(); + const task = plugin.migrationTasks?.["scheduler-state-to-sql-v1"]; + if (!task) { + throw new Error("Missing scheduler migration task"); + } + return await task({ + db: args.db, + log: { error: () => {}, info: () => {}, warn: () => {} }, + state: createPluginState("scheduler", args.stateAdapter), + }); +} + function createTask(overrides: Partial = {}): ScheduledTask { return { id: "sched_sql_1", @@ -65,6 +115,19 @@ function createTask(overrides: Partial = {}): ScheduledTask { } describe("scheduler SQL plugin storage", () => { + afterAll(() => { + if (NEON.originalDatabaseUrl === undefined) { + delete process.env.DATABASE_URL; + } else { + process.env.DATABASE_URL = NEON.originalDatabaseUrl; + } + }); + + afterEach(async () => { + NEON.sql = undefined; + await disconnectStateAdapter(); + }); + it("adopts deployed scheduler schema state into its Drizzle journal", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -131,10 +194,13 @@ INSERT INTO junior_scheduler_tasks ( pluginName: "scheduler", }, ]), - ).resolves.toEqual({ existing: 1, migrated: 1, scanned: 2 }); - const migrations = readMigrationFiles({ - migrationsFolder: schedulerMigrationsDir(), + ).resolves.toEqual({ + existing: 1, + migrated: 1, + scanned: 3, + skipped: 1, }); + const migrations = await resolveMigrations(schedulerMigrationsDir()); const migrationRows = await fixture.sql.query<{ createdAt: string; hash: string; @@ -144,10 +210,12 @@ FROM drizzle.${migrationTable!.tablename} ORDER BY created_at `); expect(migrationRows).toEqual( - migrations.map((migration) => ({ - createdAt: String(migration.folderMillis), - hash: migration.hash, - })), + migrations + .filter((migration) => migration.kind === "sql") + .map((migration) => ({ + createdAt: String(migration.when), + hash: migration.hash, + })), ); const [migratedTask] = await fixture.sql.query<{ record: unknown }>( `SELECT record FROM junior_scheduler_tasks WHERE id = $1`, @@ -164,7 +232,12 @@ ORDER BY created_at pluginName: "scheduler", }, ]), - ).resolves.toEqual({ existing: 2, migrated: 0, scanned: 2 }); + ).resolves.toEqual({ + existing: 2, + migrated: 0, + scanned: 3, + skipped: 1, + }); await expect( fixture.sql.query<{ createdAt: string; @@ -186,17 +259,21 @@ ORDER BY created_at { dir: schedulerMigrationsDir(), pluginName: "scheduler" }, { dir: memoryMigrationsDir(), pluginName: "memory" }, ]; - const migrationCount = roots.reduce( - (count, root) => - count + readMigrationFiles({ migrationsFolder: root.dir }).length, - 0, - ); - + const migrations = ( + await Promise.all( + roots.map(async (root) => await resolveMigrations(root.dir)), + ) + ).flat(); + const migrationCount = migrations.length; + const sqlMigrationCount = migrations.filter( + (migration) => migration.kind === "sql", + ).length; try { await expect(migratePluginSchemas(fixture.sql, roots)).resolves.toEqual({ existing: 0, - migrated: migrationCount, + migrated: sqlMigrationCount, scanned: migrationCount, + skipped: 1, }); const migrationTables = await fixture.sql.query<{ tablename: string }>(` SELECT tablename @@ -206,19 +283,14 @@ WHERE schemaname = 'drizzle' ORDER BY tablename `); expect(migrationTables).toHaveLength(2); - const migrationLock = vi.spyOn(fixture.sql, "withMigrationLock"); - const summaries: string[] = []; await expect( - migratePluginSchemas(fixture.sql, [...roots].reverse(), { - onPluginMigration: ({ pluginName }) => summaries.push(pluginName), - }), + migratePluginSchemas(fixture.sql, [...roots].reverse()), ).resolves.toEqual({ - existing: migrationCount, + existing: sqlMigrationCount, migrated: 0, scanned: migrationCount, + skipped: 1, }); - expect(migrationLock).not.toHaveBeenCalled(); - expect(summaries).toEqual(["memory", "scheduler"]); } finally { await fixture.close(); } @@ -456,38 +528,173 @@ ORDER BY tablename } }, 15_000); + it("migrates existing scheduler plugin state into SQL idempotently", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchedulerSchema(fixture); + const db = fixture.sql.db() as unknown as SchedulerDb; + const stateStore = createSchedulerStore( + createPluginState("scheduler", stateAdapter), + ); + const task = createTask({ id: "sched_state_sql" }); + await stateStore.saveTask(task); + const run = await stateStore.claimDueRun({ nowMs: TEST_NOW_MS }); + expect(run).toBeDefined(); + + await expect( + runSchedulerMigrationTask({ db, stateAdapter }), + ).resolves.toEqual({ + existing: 0, + migrated: 2, + missing: 0, + scanned: 2, + }); + await expect( + runSchedulerMigrationTask({ db, stateAdapter }), + ).resolves.toEqual({ + existing: 2, + migrated: 0, + missing: 0, + scanned: 2, + }); + + const sqlStore = createSchedulerSqlStore(db); + await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ + id: task.id, + }); + await expect(sqlStore.getRun(run!.id)).resolves.toMatchObject({ + id: run!.id, + taskId: task.id, + }); + } finally { + await stateAdapter.disconnect(); + await fixture.close(); + } + }, 15_000); + + it("skips malformed scheduler state records during SQL storage migration", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + + try { + await migrateSchedulerSchema(fixture); + const db = fixture.sql.db() as unknown as SchedulerDb; + const state = createPluginState("scheduler", stateAdapter); + const task = createTask({ id: "sched_state_sql_valid_after_bad" }); + const { credentialMode: _credentialMode, ...legacyTask } = task; + const badRunId = `${task.id}:${TEST_RUN_AT_MS}`; + await state.set( + "junior:scheduler:tasks", + ["sched_state_sql_bad", task.id], + 5 * 60 * 1000, + ); + await state.set( + `junior:scheduler:task:${task.id}`, + { + ...legacyTask, + credentialSubject: { + type: "user", + userId: "U123", + allowedWhen: "private-direct-conversation", + }, + }, + 5 * 60 * 1000, + ); + await state.set( + "junior:scheduler:task:sched_state_sql_bad", + { + ...task, + id: "sched_state_sql_bad", + task: { text: 123 }, + }, + 5 * 60 * 1000, + ); + await state.set( + `junior:scheduler:active:${task.id}`, + { + claimedAtMs: TEST_NOW_MS, + runId: badRunId, + scheduledForMs: TEST_RUN_AT_MS, + }, + 5 * 60 * 1000, + ); + await state.set( + `junior:scheduler:run:${badRunId}`, + { id: badRunId }, + 5 * 60 * 1000, + ); + + await expect( + runSchedulerMigrationTask({ db, stateAdapter }), + ).resolves.toEqual({ + existing: 0, + migrated: 1, + missing: 1, + scanned: 2, + }); + + const sqlStore = createSchedulerSqlStore(db); + await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ + credentialMode: "system", + id: task.id, + }); + await expect(sqlStore.getTask("sched_state_sql_bad")).resolves.toBe( + undefined, + ); + await expect(sqlStore.getRun(badRunId)).resolves.toBe(undefined); + } finally { + await stateAdapter.disconnect(); + await fixture.close(); + } + }, 15_000); + it("does not apply scheduler SQL migrations from package-only config", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; try { await expect( - migratePluginsToSql({ + migratePluginJournals({ + io: { info: () => {} }, pluginCatalogConfig: { packages: ["@sentry/junior-scheduler"] }, - sqlExecutor: fixture.sql, + stateAdapter, }), ).resolves.toEqual({ existing: 0, migrated: 0, + missing: 0, scanned: 0, }); } finally { + await stateAdapter.disconnect(); await fixture.close(); } }); it("applies scheduler SQL migrations from registration-only config", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; try { await expect( - migratePluginsToSql({ + migratePluginJournals({ + io: { info: () => {} }, pluginSet: defineJuniorPlugins([schedulerPlugin()]), - sqlExecutor: fixture.sql, + stateAdapter, }), ).resolves.toEqual({ existing: 0, - migrated: 2, - scanned: 2, + migrated: 3, + missing: 0, + scanned: 3, }); const db = fixture.sql.db() as unknown as SchedulerDb; @@ -498,28 +705,66 @@ ORDER BY tablename id: task.id, }); } finally { + await stateAdapter.disconnect(); + await fixture.close(); + } + }); + + it("runs a migration-only plugin registration", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; + const scheduler = schedulerPlugin(); + const migrationOnly = defineJuniorPlugin({ + manifest: scheduler.manifest, + migrationTasks: scheduler.migrationTasks, + packageName: scheduler.packageName, + }); + + try { + await expect( + migratePluginJournals({ + io: { info: () => {} }, + pluginSet: defineJuniorPlugins([migrationOnly]), + stateAdapter, + }), + ).resolves.toEqual({ + existing: 0, + migrated: 3, + missing: 0, + scanned: 3, + }); + } finally { + await stateAdapter.disconnect(); await fixture.close(); } }); it("does not duplicate scheduler SQL migrations for explicit registrations", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; try { await expect( - migratePluginsToSql({ + migratePluginJournals({ + io: { info: () => {} }, pluginSet: defineJuniorPlugins([ "@sentry/junior-scheduler", schedulerPlugin(), ]), - sqlExecutor: fixture.sql, + stateAdapter, }), ).resolves.toEqual({ existing: 0, - migrated: 2, - scanned: 2, + migrated: 3, + missing: 0, + scanned: 3, }); } finally { + await stateAdapter.disconnect(); await fixture.close(); } }); diff --git a/packages/junior/tests/fixtures/postgres/executor.ts b/packages/junior/tests/fixtures/postgres/executor.ts index 2e90d9ad0..3757405c3 100644 --- a/packages/junior/tests/fixtures/postgres/executor.ts +++ b/packages/junior/tests/fixtures/postgres/executor.ts @@ -1,7 +1,5 @@ import { type PoolClient, type QueryResultRow } from "pg"; import { drizzle } from "drizzle-orm/node-postgres"; -import { migrate } from "drizzle-orm/node-postgres/migrator"; -import type { MigrationConfig } from "drizzle-orm/migrator"; import type { JuniorDatabase, JuniorSqlExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; import { juniorSqlSchema } from "@/db/schema"; @@ -37,10 +35,6 @@ class ClientJuniorSqlExecutor implements JuniorSqlExecutor { return result.rows as T[]; } - async migrate(config: MigrationConfig): Promise { - await migrate(drizzle(this.client, { schema: juniorSqlSchema }), config); - } - async transaction(callback: () => Promise): Promise { const savepoint = `junior_test_savepoint_${++this.savepointId}`; await this.client.query(`SAVEPOINT ${savepoint}`); diff --git a/packages/junior/tests/fixtures/sql.ts b/packages/junior/tests/fixtures/sql.ts index c729dda3c..6c0855477 100644 --- a/packages/junior/tests/fixtures/sql.ts +++ b/packages/junior/tests/fixtures/sql.ts @@ -5,7 +5,6 @@ import { createLocalPgliteFixture, type LocalPgliteFixture, } from "@sentry/junior-testing/pglite"; -import { migrate } from "drizzle-orm/pglite/migrator"; import type { PgliteDatabase } from "drizzle-orm/pglite"; import { createEmptyJuniorSqlFixture, @@ -43,7 +42,6 @@ export async function createLocalJuniorSqlFixture(): Promise fixture.close(), db: () => fixture.db() as unknown as JuniorDatabase, execute: (statement, params) => fixture.execute(statement, params), - migrate: (config) => migrate(fixture.db(), config), query: (statement: string, params?: readonly unknown[]) => fixture.query(statement, params), transaction: (callback) => fixture.transaction(callback), diff --git a/packages/junior/tests/unit/sql/executor.test.ts b/packages/junior/tests/unit/sql/executor.test.ts index 4950889e4..329ca8e0f 100644 --- a/packages/junior/tests/unit/sql/executor.test.ts +++ b/packages/junior/tests/unit/sql/executor.test.ts @@ -13,7 +13,6 @@ function executor(name: string): JuniorSqlExecutor { throw new Error(`${name} test executor does not expose Drizzle`); }), execute: vi.fn(), - migrate: vi.fn(), query: vi.fn(), transaction: vi.fn(async (callback) => await callback()), withLock: vi.fn(async (_lockName, callback) => await callback()), diff --git a/packages/junior/tsup.config.ts b/packages/junior/tsup.config.ts index 56630a8bf..9eb35b75a 100644 --- a/packages/junior/tsup.config.ts +++ b/packages/junior/tsup.config.ts @@ -36,6 +36,7 @@ export default defineConfig({ "@chat-adapter/state-redis", "@earendil-works/pi-agent-core", "@earendil-works/pi-ai", + "@sentry/junior-migrations", "@sinclair/typebox", "@slack/web-api", "@vercel/functions", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c2274f93..59fc59c23 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,9 @@ importers: '@neondatabase/serverless': specifier: ^1.1.0 version: 1.1.0 + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@sentry/junior-plugin-api': specifier: workspace:* version: link:../junior-plugin-api @@ -475,6 +478,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@sentry/junior-testing': specifier: workspace:* version: file:packages/junior-testing @@ -497,6 +503,30 @@ importers: specifier: ^4.1.7 version: 4.1.7(@types/node@25.9.1)(tsx@4.22.3) + packages/junior-migrations: + devDependencies: + '@types/node': + specifier: ^25.9.1 + version: 25.9.1 + drizzle-kit: + specifier: 'catalog:' + version: 0.31.10 + drizzle-orm: + specifier: 'catalog:' + version: 0.45.2 + oxlint: + specifier: ^1.66.0 + version: 1.66.0 + tsup: + specifier: ^8.5.1 + version: 8.5.1(tsx@4.22.3)(typescript@6.0.3) + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.7 + version: 4.1.7(@types/node@25.9.1)(tsx@4.22.3) + packages/junior-notion: {} packages/junior-plugin-api: @@ -530,6 +560,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@sentry/junior-migrations': + specifier: workspace:* + version: link:../junior-migrations '@types/node': specifier: ^25.9.1 version: 25.9.1 @@ -3199,6 +3232,10 @@ packages: '@sentry/junior-memory@file:packages/junior-memory': resolution: {directory: packages/junior-memory, type: directory} + '@sentry/junior-migrations@file:packages/junior-migrations': + resolution: {directory: packages/junior-migrations, type: directory} + hasBin: true + '@sentry/junior-plugin-api@file:packages/junior-plugin-api': resolution: {directory: packages/junior-plugin-api, type: directory} @@ -6914,8 +6951,8 @@ packages: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} - string-width@8.2.1: - resolution: {integrity: sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==} + string-width@8.2.2: + resolution: {integrity: sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==} engines: {node: '>=20'} string_decoder@1.3.0: @@ -10215,6 +10252,8 @@ snapshots: - sql.js - sqlite3 + '@sentry/junior-migrations@file:packages/junior-migrations': {} + '@sentry/junior-plugin-api@file:packages/junior-plugin-api': dependencies: commander: 14.0.3 @@ -10375,6 +10414,7 @@ snapshots: '@logtape/logtape': 2.1.1 '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) '@neondatabase/serverless': 1.1.0 + '@sentry/junior-migrations': file:packages/junior-migrations '@sentry/junior-plugin-api': file:packages/junior-plugin-api '@sentry/node': 10.53.1 '@sinclair/typebox': 0.34.49 @@ -11826,7 +11866,7 @@ snapshots: cli-truncate@5.2.0: dependencies: slice-ansi: 8.0.0 - string-width: 8.2.1 + string-width: 8.2.2 cli-width@4.1.0: {} @@ -15264,7 +15304,7 @@ snapshots: get-east-asian-width: 1.6.0 strip-ansi: 7.2.0 - string-width@8.2.1: + string-width@8.2.2: dependencies: get-east-asian-width: 1.6.0 strip-ansi: 7.2.0 @@ -16127,7 +16167,7 @@ snapshots: wrap-ansi@10.0.0: dependencies: ansi-styles: 6.2.3 - string-width: 8.2.1 + string-width: 8.2.2 strip-ansi: 7.2.0 wrap-ansi@7.0.0: diff --git a/scripts/bump-release-versions.mjs b/scripts/bump-release-versions.mjs index 07646a8d5..d37091931 100644 --- a/scripts/bump-release-versions.mjs +++ b/scripts/bump-release-versions.mjs @@ -10,6 +10,7 @@ if (!newVersion) { const files = [ "packages/junior/package.json", + "packages/junior-migrations/package.json", "packages/junior-plugin-api/package.json", "packages/junior-agent-browser/package.json", "packages/junior-amplitude/package.json", From dfa55de3310c6e3dd57f4c58a98fb3673b3804ae Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 13 Jul 2026 16:12:39 -0700 Subject: [PATCH 02/15] fix(migrations): Embed durable migration implementations Remove the migration task registry and keep historical data migration code in each journal entry. This prevents future runtime refactors or deletions from breaking pending upgrades. Expose versioned migration capabilities, move scheduler and core backfills into their migration files, and update migration tests and documentation. Co-Authored-By: GPT-5.6 Codex --- packages/junior-migrations/README.md | 16 +++--- packages/junior-migrations/src/index.ts | 3 +- packages/junior-migrations/src/journal.ts | 15 ++++-- packages/junior-migrations/src/types.ts | 27 +++++++--- .../junior-migrations/tests/journal.test.ts | 2 +- .../junior-migrations/tests/runner.test.ts | 51 ++++++++----------- packages/junior-plugin-api/README.md | 11 ++-- .../junior-plugin-api/src/registration.ts | 3 -- packages/junior-scheduler/README.md | 4 +- packages/junior/migrations/README.md | 6 +-- .../src/chat/conversations/sql/migrations.ts | 27 ++++++++-- .../junior/src/chat/plugins/migrations.ts | 26 ++++------ .../cli/upgrade/migrations/core-journal.ts | 27 +--------- .../cli/upgrade/migrations/plugin-journal.ts | 27 +--------- .../tests/component/cli/upgrade-cli.test.ts | 44 ++++++++-------- .../component/scheduler-sql-plugin.test.ts | 29 +++++------ 16 files changed, 148 insertions(+), 170 deletions(-) diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index c78ddc14c..ca4a3e1dd 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -6,11 +6,9 @@ migrations. Every journal entry resolves to exactly one `.sql` or the Junior migration runner owns execution and exact per-entry tracking. TypeScript migrations target a versioned capability API and must not import -Junior runtime internals. New parsers and transforms belong in the migration -file so application refactors cannot break pending upgrades. A journal can -also invoke a versioned host task when adopting legacy migration code that -already shipped before the mixed journal existed; the task name then becomes -part of the permanent migration ABI. +Junior runtime internals. Every parser, transform, and migration-specific +helper belongs permanently in the migration file so application refactors or +deletions cannot break pending upgrades. ## Authoring @@ -34,6 +32,8 @@ Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock requires every entry to have a SQL file. Call `runMigrationJournal` instead and provide the SQL, state, loader, and locking capabilities owned by the host. -The runner validates that each TypeScript migration has no runtime imports. -Only a type-only import from `@sentry/junior-migrations` is accepted. Add a new -API version rather than changing an existing migration capability contract. +The runner rejects runtime imports of application source, relative modules, +and `@sentry/junior`. External package imports are allowed when the migration +needs a stable library dependency; migration-specific implementation must +still remain in the migration file. Add a new API version rather than changing +an existing migration capability contract. diff --git a/packages/junior-migrations/src/index.ts b/packages/junior-migrations/src/index.ts index 44e810595..0723969b1 100644 --- a/packages/junior-migrations/src/index.ts +++ b/packages/junior-migrations/src/index.ts @@ -7,11 +7,12 @@ export type { MigrationContextV1, MigrationJsonValue, MigrationJournalEntry, + MigrationLockV1, MigrationProgressV1, + MigrationRedisV1, MigrationRunResult, MigrationSqlExecutor, MigrationStateV1, - MigrationTasksV1, MigrationV1, ResolvedMigration, TypeScriptMigrationLoader, diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index 14bd1dc8b..397326e30 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -57,18 +57,25 @@ function validateTypeScriptSource(tag: string, source: string): void { for (const match of imports) { const clause = match[1]?.trim(); const specifier = match[2]; + if (clause?.startsWith("type ")) { + continue; + } if ( - !clause?.startsWith("type ") || - specifier !== "@sentry/junior-migrations" + !specifier || + specifier.startsWith(".") || + specifier.startsWith("/") || + specifier.startsWith("@/") || + specifier === "@sentry/junior" || + specifier.startsWith("@sentry/junior/") ) { throw new Error( - `TypeScript migration ${tag} may only import migration types`, + `TypeScript migration ${tag} cannot import application runtime code`, ); } } if ( /^\s*import\s*["']/m.test(source) || - /^\s*export\s+.+\s+from\s+["']/m.test(source) || + /^\s*export\s+.+\s+from\s+["'][./]/m.test(source) || /\b(?:import\s*\(|require\s*\()/.test(source) ) { throw new Error(`TypeScript migration ${tag} cannot load runtime modules`); diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts index 20a45206d..f5f9eb4a4 100644 --- a/packages/junior-migrations/src/types.ts +++ b/packages/junior-migrations/src/types.ts @@ -12,17 +12,33 @@ export interface MigrationSqlExecutor { ): Promise; } +/** Stable state-store capabilities exposed to v1 TypeScript migrations. */ +export interface MigrationLockV1 { + expiresAt: number; + threadId: string; + token: string; +} + /** Stable state-store capabilities exposed to v1 TypeScript migrations. */ export interface MigrationStateV1 { + acquireLock(threadId: string, ttlMs: number): Promise; appendToList( key: string, value: unknown, options?: { maxLength?: number; ttlMs?: number }, ): Promise; + connect(): Promise; delete(key: string): Promise; get(key: string): Promise; getList(key: string): Promise; + releaseLock(lock: MigrationLockV1): Promise; set(key: string, value: unknown, ttlMs?: number): Promise; + setIfNotExists(key: string, value: unknown, ttlMs?: number): Promise; +} + +/** Optional raw Redis capability for migrations preserving Redis indexes. */ +export interface MigrationRedisV1 { + sendCommand(args: readonly string[]): Promise; } /** Resumable progress storage scoped to one TypeScript migration. */ @@ -31,11 +47,6 @@ export interface MigrationProgressV1 { save(value: MigrationJsonValue): Promise; } -/** Versioned host tasks available to migrations that predate the public ABI. */ -export interface MigrationTasksV1 { - run(name: string): Promise; -} - /** JSON-compatible value persisted in migration progress and result columns. */ export type MigrationJsonValue = | boolean @@ -49,9 +60,11 @@ export type MigrationJsonValue = export interface MigrationContextV1 { log(message: string): void; progress: MigrationProgressV1; - sql: Pick; + redis?: MigrationRedisV1; + sql: Pick & { + db(): unknown; + }; state: MigrationStateV1; - tasks: MigrationTasksV1; } /** Isolated TypeScript data migration targeting the v1 ABI. */ diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index 005586efd..56100fb34 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -58,7 +58,7 @@ describe("resolveMigrations", () => { ); await expect(resolveMigrations(folder)).rejects.toThrow( - "may only import migration types", + "cannot import application runtime code", ); }); diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index 0c1c21575..ca3a51415 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -20,6 +20,10 @@ class FakeExecutor implements MigrationSqlExecutor { readonly rows = new Map(); readonly statements: string[] = []; + db(): undefined { + return undefined; + } + async execute(statement: string, parameters: readonly unknown[] = []) { const normalized = statement.trim(); if ( @@ -87,6 +91,20 @@ class FakeExecutor implements MigrationSqlExecutor { } } +function fakeMigrationState(): MigrationContextV1["state"] { + return { + acquireLock: async () => null, + appendToList: async () => {}, + connect: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + releaseLock: async () => {}, + set: async () => {}, + setIfNotExists: async () => true, + }; +} + const temporaryDirectories: string[] = []; async function mixedFolder(): Promise { @@ -149,16 +167,7 @@ describe("runMigrationJournal", () => { log: () => {}, progress, sql: executor, - state: { - appendToList: async () => {}, - delete: async () => {}, - get: async () => undefined, - getList: async () => [], - set: async () => {}, - }, - tasks: { - run: async () => undefined, - }, + state: fakeMigrationState(), }), }), ).resolves.toEqual({ existing: 0, migrated: 3, scanned: 3, skipped: 0 }); @@ -178,16 +187,7 @@ describe("runMigrationJournal", () => { log: () => {}, progress, sql: executor, - state: { - appendToList: async () => {}, - delete: async () => {}, - get: async () => undefined, - getList: async () => [], - set: async () => {}, - }, - tasks: { - run: async () => undefined, - }, + state: fakeMigrationState(), }), }), ).resolves.toEqual({ existing: 3, migrated: 0, scanned: 3, skipped: 0 }); @@ -237,16 +237,7 @@ describe("runMigrationJournal", () => { log: () => {}, progress, sql: executor, - state: { - appendToList: async () => {}, - delete: async () => {}, - get: async () => undefined, - getList: async () => [], - set: async () => {}, - }, - tasks: { - run: async () => undefined, - }, + state: fakeMigrationState(), }), }; diff --git a/packages/junior-plugin-api/README.md b/packages/junior-plugin-api/README.md index 3eeae30d0..407d7925b 100644 --- a/packages/junior-plugin-api/README.md +++ b/packages/junior-plugin-api/README.md @@ -5,14 +5,13 @@ exported TypeScript types and runtime validators are authoritative. ## Registration -Use `defineJuniorPlugin({ manifest, hooks, tasks, migrationTasks, cli, model })`. +Use `defineJuniorPlugin({ manifest, hooks, tasks, cli, model })`. A plugin name is a lowercase identifier and is unique within the enabled app plugin set. A plugin may instead be a declarative `plugin.yaml` package when it has no -host-executed hooks, tasks, migration tasks, CLI, or model implementation. Do -not combine an inline manifest with a second YAML definition for the same -plugin. +host-executed hooks, tasks, CLI, or model implementation. Do not combine an +inline manifest with a second YAML definition for the same plugin. ## Manifest @@ -60,8 +59,8 @@ reports, and other typed hook surfaces exported by this package. - Packaged migrations create plugin-owned tables through the host migration runner. -- TypeScript journal entries may invoke explicitly versioned `migrationTasks`; - tasks run only when their owning journal entry is pending. +- TypeScript journal entries contain their complete durable implementation and + execute through the versioned migration capability API. - Generate migration artifacts from the package schema; do not hand-maintain a second schema contract. - Runtime hooks and CLI actions use host-provided `ctx.db`. diff --git a/packages/junior-plugin-api/src/registration.ts b/packages/junior-plugin-api/src/registration.ts index 17c2b282e..1800108ef 100644 --- a/packages/junior-plugin-api/src/registration.ts +++ b/packages/junior-plugin-api/src/registration.ts @@ -1,7 +1,6 @@ import type { PluginCliDefinition } from "./cli"; import type { PluginHooks } from "./hooks"; import type { PluginManifest } from "./manifest"; -import type { PluginMigrationTask } from "./operations"; import type { PluginTasks } from "./tasks"; export interface PluginModelConfig { @@ -15,8 +14,6 @@ export type PluginRegistrationInput = { cli?: PluginCliDefinition; hooks?: PluginHooks; manifest: PluginManifest; - /** Permanently versioned data tasks referenced by this plugin's migration journal. */ - migrationTasks?: Record; model?: PluginModelConfig; packageName?: string; tasks?: PluginTasks; diff --git a/packages/junior-scheduler/README.md b/packages/junior-scheduler/README.md index 18b08c2ae..fcb9914a7 100644 --- a/packages/junior-scheduler/README.md +++ b/packages/junior-scheduler/README.md @@ -57,8 +57,8 @@ operational reporting. Generate schema changes with `pnpm --filter @sentry/junior-scheduler db:generate`. Use `pnpm --filter @sentry/junior-scheduler db:generate:data --name ` for a TypeScript data migration in the same journal. New migrations should use -the stable migration capabilities directly; the existing scheduler backfill -uses a versioned task to preserve its pre-journal implementation. +the stable migration capabilities directly and keep their complete +implementation in the migration file. Follow `../../policies/serverless-background-work.md`, `../../policies/context-bound-systems.md`, and diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index f72a90dab..0ff62d250 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -29,9 +29,9 @@ after the schema migration they require. For a data migration, run `pnpm --filter @sentry/junior db:generate:data --name `. TypeScript migrations target the versioned migration capability API and must -not import Junior runtime internals or current feature schemas. Entries that -adopt pre-journal upgrade code call a versioned host task; new data migrations -should use the stable SQL, state, and progress capabilities directly. +not import Junior runtime internals or current feature schemas. Their complete +implementation remains in the migration file and uses only the stable SQL, +state, Redis, and progress capabilities supplied by the runner. The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index aa25310d6..99fefca9a 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -10,6 +10,7 @@ import { type TypeScriptMigrationLoader, } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { juniorSqlSchema as schema } from "@/db/schema"; @@ -185,17 +186,27 @@ CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { return { + acquireLock: async (threadId, ttlMs) => + await stateAdapter.acquireLock(threadId, ttlMs), appendToList: async (key, value, options) => { await stateAdapter.appendToList(key, value, options); }, + connect: async () => { + await stateAdapter.connect(); + }, delete: async (key) => { await stateAdapter.delete(key); }, get: async (key) => await stateAdapter.get(key), getList: async (key) => await stateAdapter.getList(key), + releaseLock: async (lock) => { + await stateAdapter.releaseLock(lock); + }, set: async (key, value, ttlMs) => { await stateAdapter.set(key, value, ttlMs); }, + setIfNotExists: async (key, value, ttlMs) => + await stateAdapter.setIfNotExists(key, value, ttlMs), }; } @@ -204,6 +215,7 @@ function migrationSql( executor: JuniorSqlMigrationExecutor, ): MigrationContextV1["sql"] { return { + db: executor.db.bind(executor), execute: executor.execute.bind(executor), query: executor.query.bind(executor), transaction: executor.transaction.bind(executor), @@ -216,7 +228,7 @@ export type MigrateSchemaOptions = loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; mode: "all"; - runTask: MigrationContextV1["tasks"]["run"]; + redisStateAdapter?: RedisStateAdapter; stateAdapter: StateAdapter; }; @@ -245,11 +257,18 @@ export async function migrateSchema( createContext: ({ progress }): MigrationContextV1 => ({ log: options.log ?? (() => {}), progress, + ...(options.redisStateAdapter + ? { + redis: { + sendCommand: async (args: readonly string[]) => + await options + .redisStateAdapter!.getClient() + .sendCommand([...args]), + }, + } + : {}), sql: migrationSql(executor), state: migrationState(options.stateAdapter), - tasks: { - run: options.runTask, - }, }), loadTypeScript: options.loadTypeScript, mode: "all", diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index 8a6dfe442..da16629d3 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -3,7 +3,6 @@ import { resolveMigrations, runMigrationJournal, type MigrationContextV1, - type MigrationJsonValue, type MigrationStateV1, type ResolvedMigration, type TypeScriptMigrationLoader, @@ -30,10 +29,6 @@ type PluginMigrationOptions = loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; mode: "all"; - runTask?: ( - pluginName: string, - taskName: string, - ) => Promise; stateAdapter: StateAdapter; }; @@ -138,17 +133,27 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { return { + acquireLock: async (threadId, ttlMs) => + await stateAdapter.acquireLock(threadId, ttlMs), appendToList: async (key, value, options) => { await stateAdapter.appendToList(key, value, options); }, + connect: async () => { + await stateAdapter.connect(); + }, delete: async (key) => { await stateAdapter.delete(key); }, get: async (key) => await stateAdapter.get(key), getList: async (key) => await stateAdapter.getList(key), + releaseLock: async (lock) => { + await stateAdapter.releaseLock(lock); + }, set: async (key, value, ttlMs) => { await stateAdapter.set(key, value, ttlMs); }, + setIfNotExists: async (key, value, ttlMs) => + await stateAdapter.setIfNotExists(key, value, ttlMs), }; } @@ -157,6 +162,7 @@ function migrationSql( executor: JuniorSqlMigrationExecutor, ): MigrationContextV1["sql"] { return { + db: executor.db.bind(executor), execute: executor.execute.bind(executor), query: executor.query.bind(executor), transaction: executor.transaction.bind(executor), @@ -202,16 +208,6 @@ export async function migratePluginSchemas( progress, sql: migrationSql(executor), state: migrationState(options.stateAdapter), - tasks: { - run: async (taskName) => { - if (!options.runTask) { - throw new Error( - `Unsupported migration task for plugin ${root.pluginName}: ${taskName}`, - ); - } - return await options.runTask(root.pluginName, taskName); - }, - }, }), loadTypeScript: options.loadTypeScript, mode: "all", diff --git a/packages/junior/src/cli/upgrade/migrations/core-journal.ts b/packages/junior/src/cli/upgrade/migrations/core-journal.ts index d753825bb..7951c77d8 100644 --- a/packages/junior/src/cli/upgrade/migrations/core-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/core-journal.ts @@ -1,35 +1,12 @@ import { getChatConfig } from "@/chat/config"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createJuniorSqlExecutor } from "@/db/executor"; -import type { MigrationJsonValue } from "@sentry/junior-migrations"; import { createJiti } from "jiti"; -import { migrateAgentTurnSessionActor } from "./agent-turn-session-actor"; -import { migrateConversationHistoryToSql } from "./conversations-history-sql"; -import { migrateConversationsToSql } from "./conversations-sql"; -import { migrateRedisConversationState } from "./redis-conversation-state"; import type { MigrationContext, MigrationResult } from "../types"; const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); -async function runCoreMigrationTask( - context: MigrationContext, - name: string, -): Promise { - switch (name) { - case "agent-turn-session-actor-v1": - return await migrateAgentTurnSessionActor(context); - case "redis-conversation-state-v1": - return await migrateRedisConversationState(context); - case "conversations-to-sql-v1": - return await migrateConversationsToSql(context); - case "conversation-history-to-sql-v1": - return await migrateConversationHistoryToSql(context); - default: - throw new Error(`Unknown core migration task: ${name}`); - } -} - -/** Apply core schema migrations and versioned legacy data tasks in journal order. */ +/** Apply core schema and self-contained data migrations in journal order. */ export async function migrateCoreJournal( context: MigrationContext, ): Promise { @@ -44,7 +21,7 @@ export async function migrateCoreJournal( await migrationLoader.import>(path), log: context.io.info, mode: "all", - runTask: async (name) => await runCoreMigrationTask(context, name), + redisStateAdapter: context.redisStateAdapter, stateAdapter: context.stateAdapter, }); return { diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts index 4ab03e05f..d03c89249 100644 --- a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -1,8 +1,6 @@ import { getChatConfig } from "@/chat/config"; import { migratePluginSchemas } from "@/chat/plugins/migrations"; import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; -import { createPluginLogger } from "@/chat/plugins/logging"; -import { createPluginState } from "@/chat/plugins/state"; import { createJuniorSqlExecutor } from "@/db/executor"; import { createJiti } from "jiti"; import { resolveUpgradePlugins } from "./upgrade-plugins"; @@ -15,22 +13,13 @@ export async function migratePluginJournals( context: MigrationContext, ): Promise { const { sql } = getChatConfig(); - const { pluginCatalogConfig, pluginSet } = - await resolveUpgradePlugins(context); + const { pluginCatalogConfig } = await resolveUpgradePlugins(context); const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig); const executor = createJuniorSqlExecutor({ connectionString: sql.databaseUrl, driver: sql.driver, }); try { - const registrations = new Map( - pluginSet - ? pluginSet.registrations.map((plugin) => [ - plugin.manifest.name, - plugin, - ]) - : [], - ); const result = await migratePluginSchemas( executor, pluginCatalogRuntime.getMigrationRoots(), @@ -39,20 +28,6 @@ export async function migratePluginJournals( await migrationLoader.import>(path), log: context.io.info, mode: "all", - runTask: async (pluginName, taskName) => { - const task = - registrations.get(pluginName)?.migrationTasks?.[taskName]; - if (!task) { - throw new Error( - `Plugin ${pluginName} does not provide migration task ${taskName}`, - ); - } - return await task({ - db: context.db ?? executor.db(), - log: createPluginLogger(pluginName), - state: createPluginState(pluginName, context.stateAdapter), - }); - }, stateAdapter: context.stateAdapter, }, ); diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 021b386d1..5136b0ed6 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -20,10 +20,10 @@ import type { PiMessage } from "@/chat/pi/messages"; import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { recordAgentTurnSessionSummary } from "@/chat/state/turn-session"; import { resolveUpgradePluginSet } from "@/cli/upgrade"; -import { migrateAgentTurnSessionActor } from "@/cli/upgrade/migrations/agent-turn-session-actor"; import { repairConversationUsage } from "@/cli/upgrade/migrations/conversation-usage"; -import { migrateConversationsToSql } from "@/cli/upgrade/migrations/conversations-sql"; -import { migrateRedisConversationState } from "@/cli/upgrade/migrations/redis-conversation-state"; +import { migrateAgentTurnSessionActor } from "../../../migrations/0005_agent_turn_session_actor"; +import { migrateRedisConversationState } from "../../../migrations/0006_redis_conversation_state"; +import { migrateConversationsToSql } from "../../../migrations/0007_conversations_to_sql"; import { CONVERSATION_ID, SLACK_DESTINATION, @@ -295,10 +295,21 @@ export const plugins = { const stateAdapter = getStateAdapter(); await stateAdapter.connect(); const fixture = await createLocalJuniorSqlFixture(); - const actor = { platform: "api", userId: "migration-user" }; + const actor = { + platform: "slack", + teamId: "T123", + userId: "migration-user", + } as const; const summary = { + version: 1, conversationId: CONVERSATION_ID, + cumulativeDurationMs: 0, + lastProgressAtMs: 2, sessionId: "session-one", + sliceId: 1, + startedAtMs: 1, + state: "completed", + updatedAtMs: 3, requester: actor, }; await stateAdapter.appendToList("junior:agent_turn_session:index", summary); @@ -319,14 +330,6 @@ export const plugins = { migrationPath, ), mode: "all", - runTask: async (name) => { - if (name === "agent-turn-session-actor-v1") { - return await migrateAgentTurnSessionActor({ - io: { info: () => {} }, - stateAdapter, - }); - } - }, stateAdapter, }), ).resolves.toEqual({ @@ -338,21 +341,23 @@ export const plugins = { await expect( stateAdapter.getList("junior:agent_turn_session:index"), ).resolves.toEqual([ - { + expect.objectContaining({ actor, conversationId: CONVERSATION_ID, sessionId: "session-one", - }, + }), ]); await expect( stateAdapter.get( `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, ), - ).resolves.toEqual({ - actor, - conversationId: CONVERSATION_ID, - sessionId: "session-one", - }); + ).resolves.toEqual( + expect.objectContaining({ + actor, + conversationId: CONVERSATION_ID, + sessionId: "session-one", + }), + ); await expect( migrateSchema(fixture.sql, { loadTypeScript: async (migrationPath) => @@ -360,7 +365,6 @@ export const plugins = { migrationPath, ), mode: "all", - runTask: async () => undefined, stateAdapter, }), ).resolves.toEqual({ diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index 8a015530e..b16c18e64 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -11,6 +11,7 @@ import { type SchedulerDb, type ScheduledTask, } from "@sentry/junior-scheduler"; +import schedulerStateToSqlMigration from "../../../junior-scheduler/migrations/0002_scheduler_state_to_sql"; import { createSchedulerStore } from "../../../junior-scheduler/src/store"; import { defineJuniorPlugins } from "@/plugins"; import { migratePluginSchemas } from "@/chat/plugins/migrations"; @@ -72,19 +73,18 @@ async function migrateSchedulerSchema( ]); } -async function runSchedulerMigrationTask(args: { - db: SchedulerDb; +async function runSchedulerStateMigration(args: { + fixture: Awaited>; stateAdapter: ReturnType; }) { - const plugin = schedulerPlugin(); - const task = plugin.migrationTasks?.["scheduler-state-to-sql-v1"]; - if (!task) { - throw new Error("Missing scheduler migration task"); - } - return await task({ - db: args.db, - log: { error: () => {}, info: () => {}, warn: () => {} }, - state: createPluginState("scheduler", args.stateAdapter), + return await schedulerStateToSqlMigration.up({ + log: () => {}, + progress: { + load: async () => undefined, + save: async () => {}, + }, + sql: args.fixture.sql, + state: args.stateAdapter, }); } @@ -545,7 +545,7 @@ ORDER BY tablename expect(run).toBeDefined(); await expect( - runSchedulerMigrationTask({ db, stateAdapter }), + runSchedulerStateMigration({ fixture, stateAdapter }), ).resolves.toEqual({ existing: 0, migrated: 2, @@ -553,7 +553,7 @@ ORDER BY tablename scanned: 2, }); await expect( - runSchedulerMigrationTask({ db, stateAdapter }), + runSchedulerStateMigration({ fixture, stateAdapter }), ).resolves.toEqual({ existing: 2, migrated: 0, @@ -629,7 +629,7 @@ ORDER BY tablename ); await expect( - runSchedulerMigrationTask({ db, stateAdapter }), + runSchedulerStateMigration({ fixture, stateAdapter }), ).resolves.toEqual({ existing: 0, migrated: 1, @@ -718,7 +718,6 @@ ORDER BY tablename const scheduler = schedulerPlugin(); const migrationOnly = defineJuniorPlugin({ manifest: scheduler.manifest, - migrationTasks: scheduler.migrationTasks, packageName: scheduler.packageName, }); From 532ff36b30c7dc1d7917965e2920fac909aea8b9 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 13 Jul 2026 16:23:56 -0700 Subject: [PATCH 03/15] ref(migrations): Inject the database adapter Expose the exact host database adapter through MigrationContextV1 instead of reconstructing a partial SQL capability. Rebuild the SQL-heavy frozen migrations without connection setup and driver implementations so migrations retain transformation logic without duplicating database infrastructure. Co-Authored-By: GPT-5.6 Codex --- packages/junior-migrations/README.md | 4 +++- packages/junior-migrations/src/index.ts | 2 +- packages/junior-migrations/src/runner.ts | 14 +++++++------- packages/junior-migrations/src/types.ts | 10 +++++----- packages/junior-migrations/tests/runner.test.ts | 14 +++++++++----- packages/junior/migrations/README.md | 6 ++++-- .../src/chat/conversations/sql/migrations.ts | 14 +------------- packages/junior/src/chat/plugins/migrations.ts | 14 +------------- .../tests/component/scheduler-sql-plugin.test.ts | 2 +- 9 files changed, 32 insertions(+), 48 deletions(-) diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index ca4a3e1dd..653329708 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -30,7 +30,9 @@ then replaces the empty SQL file with a `MigrationV1` TypeScript scaffold. Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock `migrate()` function is not a supported executor for mixed journals because it requires every entry to have a SQL file. Call `runMigrationJournal` instead and -provide the SQL, state, loader, and locking capabilities owned by the host. +provide the host database adapter, state adapter, and TypeScript loader. The +same database adapter drives the journal ledger and is exposed as +`context.database`, so migration files never own connection or driver setup. The runner rejects runtime imports of application source, relative modules, and `@sentry/junior`. External package imports are allowed when the migration diff --git a/packages/junior-migrations/src/index.ts b/packages/junior-migrations/src/index.ts index 0723969b1..6c07a55ad 100644 --- a/packages/junior-migrations/src/index.ts +++ b/packages/junior-migrations/src/index.ts @@ -5,13 +5,13 @@ export type { GenerateTypeScriptMigrationOptions } from "./generate"; export type { RunMigrationJournalOptions } from "./runner"; export type { MigrationContextV1, + MigrationDatabaseAdapter, MigrationJsonValue, MigrationJournalEntry, MigrationLockV1, MigrationProgressV1, MigrationRedisV1, MigrationRunResult, - MigrationSqlExecutor, MigrationStateV1, MigrationV1, ResolvedMigration, diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index 48912d40b..d0d043e93 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -1,7 +1,7 @@ import type { MigrationContextV1, + MigrationDatabaseAdapter, MigrationRunResult, - MigrationSqlExecutor, MigrationV1, ResolvedMigration, TypeScriptMigrationLoader, @@ -17,7 +17,7 @@ interface MigrationRow { interface RunMigrationJournalBaseOptions { beforeRun?: () => Promise; - executor: MigrationSqlExecutor; + executor: MigrationDatabaseAdapter; migrationsFolder: string; migrationsTable: string; } @@ -54,7 +54,7 @@ function qualifiedTable(table: string): string { } async function ensureMigrationTable( - executor: MigrationSqlExecutor, + executor: MigrationDatabaseAdapter, table: string, ): Promise { const qualified = qualifiedTable(table); @@ -90,7 +90,7 @@ CREATE TABLE IF NOT EXISTS ${qualified} ( } async function migrationRows( - executor: MigrationSqlExecutor, + executor: MigrationDatabaseAdapter, table: string, ): Promise> { const rows = await executor.query(` @@ -107,7 +107,7 @@ ORDER BY created_at ASC, id ASC } async function adoptLegacySqlPrefix(args: { - executor: MigrationSqlExecutor; + executor: MigrationDatabaseAdapter; migrations: readonly ResolvedMigration[]; rows: Map; table: string; @@ -160,7 +160,7 @@ function migrationV1(value: unknown, tag: string): MigrationV1 { } async function runSqlMigration(args: { - executor: MigrationSqlExecutor; + executor: MigrationDatabaseAdapter; migration: ResolvedMigration; table: string; }): Promise { @@ -179,7 +179,7 @@ async function runSqlMigration(args: { async function runTypeScriptMigration(args: { createContext: RunAllMigrationJournalOptions["createContext"]; - executor: MigrationSqlExecutor; + executor: MigrationDatabaseAdapter; loadTypeScript: TypeScriptMigrationLoader; migration: ResolvedMigration; row: MigrationRow | undefined; diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts index f5f9eb4a4..b49b3ac6f 100644 --- a/packages/junior-migrations/src/types.ts +++ b/packages/junior-migrations/src/types.ts @@ -1,11 +1,13 @@ -/** SQL capabilities available to the mixed migration runner. */ -export interface MigrationSqlExecutor { +/** Database adapter used by the mixed migration runner and TypeScript entries. */ +export interface MigrationDatabaseAdapter { + db(): unknown; execute(statement: string, parameters?: readonly unknown[]): Promise; query( statement: string, parameters?: readonly unknown[], ): Promise; transaction(callback: () => Promise): Promise; + withLock(lockName: string, callback: () => Promise): Promise; withMigrationLock( migrationTable: string, callback: () => Promise, @@ -58,12 +60,10 @@ export type MigrationJsonValue = /** Permanent capability contract for apiVersion 1 migrations. */ export interface MigrationContextV1 { + database: MigrationDatabaseAdapter; log(message: string): void; progress: MigrationProgressV1; redis?: MigrationRedisV1; - sql: Pick & { - db(): unknown; - }; state: MigrationStateV1; } diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index ca3a51415..c072487b5 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { runMigrationJournal } from "../src/runner"; import type { MigrationContextV1, - MigrationSqlExecutor, + MigrationDatabaseAdapter, MigrationV1, } from "../src/types"; @@ -16,7 +16,7 @@ interface StoredRow { status: string | null; } -class FakeExecutor implements MigrationSqlExecutor { +class FakeExecutor implements MigrationDatabaseAdapter { readonly rows = new Map(); readonly statements: string[] = []; @@ -89,6 +89,10 @@ class FakeExecutor implements MigrationSqlExecutor { ): Promise { return await callback(); } + + async withLock(_lockName: string, callback: () => Promise): Promise { + return await callback(); + } } function fakeMigrationState(): MigrationContextV1["state"] { @@ -166,7 +170,7 @@ describe("runMigrationJournal", () => { createContext: ({ progress }) => ({ log: () => {}, progress, - sql: executor, + database: executor, state: fakeMigrationState(), }), }), @@ -186,7 +190,7 @@ describe("runMigrationJournal", () => { createContext: ({ progress }) => ({ log: () => {}, progress, - sql: executor, + database: executor, state: fakeMigrationState(), }), }), @@ -236,7 +240,7 @@ describe("runMigrationJournal", () => { }) => ({ log: () => {}, progress, - sql: executor, + database: executor, state: fakeMigrationState(), }), }; diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index 0ff62d250..b01116fc6 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -30,8 +30,10 @@ For a data migration, run `pnpm --filter @sentry/junior db:generate:data --name `. TypeScript migrations target the versioned migration capability API and must not import Junior runtime internals or current feature schemas. Their complete -implementation remains in the migration file and uses only the stable SQL, -state, Redis, and progress capabilities supplied by the runner. +implementation remains in the migration file and uses only the stable +database, state, Redis, and progress capabilities supplied by the runner. The +host database adapter owns connections, drivers, transactions, and locks so +those infrastructure modules are not frozen into every migration. The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index 99fefca9a..de433c6fe 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -210,18 +210,6 @@ function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { }; } -/** Project the executor onto the permanent SQL migration capability. */ -function migrationSql( - executor: JuniorSqlMigrationExecutor, -): MigrationContextV1["sql"] { - return { - db: executor.db.bind(executor), - execute: executor.execute.bind(executor), - query: executor.query.bind(executor), - transaction: executor.transaction.bind(executor), - }; -} - export type MigrateSchemaOptions = | { mode?: "sql" } | { @@ -255,6 +243,7 @@ export async function migrateSchema( return await runMigrationJournal({ ...baseOptions, createContext: ({ progress }): MigrationContextV1 => ({ + database: executor, log: options.log ?? (() => {}), progress, ...(options.redisStateAdapter @@ -267,7 +256,6 @@ export async function migrateSchema( }, } : {}), - sql: migrationSql(executor), state: migrationState(options.stateAdapter), }), loadTypeScript: options.loadTypeScript, diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index da16629d3..5583b60b8 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -157,18 +157,6 @@ function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { }; } -/** Project the executor onto the permanent SQL migration capability. */ -function migrationSql( - executor: JuniorSqlMigrationExecutor, -): MigrationContextV1["sql"] { - return { - db: executor.db.bind(executor), - execute: executor.execute.bind(executor), - query: executor.query.bind(executor), - transaction: executor.transaction.bind(executor), - }; -} - /** Apply enabled plugins' mixed migrations in plugin-name and journal order. */ export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, @@ -204,9 +192,9 @@ export async function migratePluginSchemas( ? await runMigrationJournal({ ...baseOptions, createContext: ({ progress }): MigrationContextV1 => ({ + database: executor, log: options.log ?? (() => {}), progress, - sql: migrationSql(executor), state: migrationState(options.stateAdapter), }), loadTypeScript: options.loadTypeScript, diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index b16c18e64..b2ce0516d 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -78,12 +78,12 @@ async function runSchedulerStateMigration(args: { stateAdapter: ReturnType; }) { return await schedulerStateToSqlMigration.up({ + database: args.fixture.sql, log: () => {}, progress: { load: async () => undefined, save: async () => {}, }, - sql: args.fixture.sql, state: args.stateAdapter, }); } From bfe439f93c052964da7374b9274a59e9fef46ba9 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 13 Jul 2026 17:11:42 -0700 Subject: [PATCH 04/15] ref(migrations): Add a versioned helper boundary Allow migrations to import append-only infrastructure helpers from @sentry/junior/migration-helpers/v1 while continuing to reject unversioned Junior runtime imports. Keep one-off Redis transformations and backfill decisions in their journal entries, replace generated capsules for migrations 0004 through 0006 with readable source, and publish explicit helper types without private module references. Co-Authored-By: GPT-5.6 Codex --- packages/junior-migrations/README.md | 9 +- packages/junior-migrations/src/journal.ts | 8 +- packages/junior-migrations/src/types.ts | 4 +- .../junior-migrations/tests/journal.test.ts | 12 + packages/junior/migrations/README.md | 6 + packages/junior/package.json | 4 + .../src/chat/conversations/sql/migrations.ts | 5 +- .../junior/src/chat/plugins/migrations.ts | 5 +- packages/junior/src/cli/upgrade/types.ts | 1 - .../junior/src/migration-helpers/README.md | 13 + packages/junior/src/migration-helpers/v1.ts | 234 ++++++++++++++++++ .../component/scheduler-sql-plugin.test.ts | 7 +- packages/junior/tsconfig.build.json | 1 + packages/junior/tsup.config.ts | 1 + 14 files changed, 295 insertions(+), 15 deletions(-) create mode 100644 packages/junior/src/migration-helpers/README.md create mode 100644 packages/junior/src/migration-helpers/v1.ts diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index 653329708..1abc59712 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -35,7 +35,8 @@ same database adapter drives the journal ledger and is exposed as `context.database`, so migration files never own connection or driver setup. The runner rejects runtime imports of application source, relative modules, -and `@sentry/junior`. External package imports are allowed when the migration -needs a stable library dependency; migration-specific implementation must -still remain in the migration file. Add a new API version rather than changing -an existing migration capability contract. +and unversioned `@sentry/junior` modules. Migrations may import the append-only +`@sentry/junior/migration-helpers/v1` surface for parsing primitives, adapters, +stores, and key resolution. One-off migration decisions and data transforms +must still remain in the journal entry. Add a new helper or capability version +rather than changing an existing contract. diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index 397326e30..826ffef26 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -51,6 +51,9 @@ async function optionalFile(path: string): Promise { } function validateTypeScriptSource(tag: string, source: string): void { + const allowedRuntimeImports = new Set([ + "@sentry/junior/migration-helpers/v1", + ]); const imports = source.matchAll( /^\s*import\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, ); @@ -65,8 +68,9 @@ function validateTypeScriptSource(tag: string, source: string): void { specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("@/") || - specifier === "@sentry/junior" || - specifier.startsWith("@sentry/junior/") + ((specifier === "@sentry/junior" || + specifier.startsWith("@sentry/junior/")) && + !allowedRuntimeImports.has(specifier)) ) { throw new Error( `TypeScript migration ${tag} cannot import application runtime code`, diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts index b49b3ac6f..dd2aa6813 100644 --- a/packages/junior-migrations/src/types.ts +++ b/packages/junior-migrations/src/types.ts @@ -31,8 +31,8 @@ export interface MigrationStateV1 { ): Promise; connect(): Promise; delete(key: string): Promise; - get(key: string): Promise; - getList(key: string): Promise; + get(key: string): Promise; + getList(key: string): Promise; releaseLock(lock: MigrationLockV1): Promise; set(key: string, value: unknown, ttlMs?: number): Promise; setIfNotExists(key: string, value: unknown, ttlMs?: number): Promise; diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index 56100fb34..0acef14cf 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -62,6 +62,18 @@ describe("resolveMigrations", () => { ); }); + it("allows versioned Junior migration helpers", async () => { + const folder = await migrationFolder(["0000_helpers"]); + await writeFile( + join(folder, "0000_helpers.ts"), + 'import { isRecord } from "@sentry/junior/migration-helpers/v1";\nexport default { apiVersion: 1, async up() { isRecord({}); } };\n', + ); + + await expect(resolveMigrations(folder)).resolves.toMatchObject([ + { kind: "typescript", tag: "0000_helpers" }, + ]); + }); + it("rejects ambiguous migration files", async () => { const folder = await migrationFolder(["0000_ambiguous"]); await writeFile(join(folder, "0000_ambiguous.sql"), "SELECT 1;"); diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index b01116fc6..fccbe54ac 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -35,6 +35,12 @@ database, state, Redis, and progress capabilities supplied by the runner. The host database adapter owns connections, drivers, transactions, and locks so those infrastructure modules are not frozen into every migration. +Reusable infrastructure belongs in the append-only +`@sentry/junior/migration-helpers/v1` export. It may provide stable parsers, +stores, adapter projections, and key resolution, but must not implement a +specific data migration. The journal entry remains the only owner of one-off +record transformations. + The `0000_initial.sql` baseline represents the schema already deployed by the pre-Drizzle Junior migration runner. During upgrade, existing installations adopt that baseline once; new installations execute it normally. All later diff --git a/packages/junior/package.json b/packages/junior/package.json index 8ca163d44..159348247 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -24,6 +24,10 @@ "types": "./dist/instrumentation.d.ts", "default": "./dist/instrumentation.js" }, + "./migration-helpers/v1": { + "types": "./dist/migration-helpers/v1.d.ts", + "default": "./dist/migration-helpers/v1.js" + }, "./nitro": { "types": "./dist/nitro.d.ts", "default": "./dist/nitro.js" diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index de433c6fe..bdadcf182 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -197,8 +197,9 @@ function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { delete: async (key) => { await stateAdapter.delete(key); }, - get: async (key) => await stateAdapter.get(key), - getList: async (key) => await stateAdapter.getList(key), + get: async (key: string) => + (await stateAdapter.get(key)) ?? undefined, + getList: async (key: string) => await stateAdapter.getList(key), releaseLock: async (lock) => { await stateAdapter.releaseLock(lock); }, diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index 5583b60b8..a0321beea 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -144,8 +144,9 @@ function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { delete: async (key) => { await stateAdapter.delete(key); }, - get: async (key) => await stateAdapter.get(key), - getList: async (key) => await stateAdapter.getList(key), + get: async (key: string) => + (await stateAdapter.get(key)) ?? undefined, + getList: async (key: string) => await stateAdapter.getList(key), releaseLock: async (lock) => { await stateAdapter.releaseLock(lock); }, diff --git a/packages/junior/src/cli/upgrade/types.ts b/packages/junior/src/cli/upgrade/types.ts index 218ce83bf..913af8b3b 100644 --- a/packages/junior/src/cli/upgrade/types.ts +++ b/packages/junior/src/cli/upgrade/types.ts @@ -8,7 +8,6 @@ export interface UpgradeIo { } export interface MigrationContext { - db?: unknown; io: UpgradeIo; pluginCatalogConfig?: PluginCatalogConfig; pluginSet?: JuniorPluginSet; diff --git a/packages/junior/src/migration-helpers/README.md b/packages/junior/src/migration-helpers/README.md new file mode 100644 index 000000000..9f2f3637f --- /dev/null +++ b/packages/junior/src/migration-helpers/README.md @@ -0,0 +1,13 @@ +# Migration Helpers + +This directory owns versioned, append-only infrastructure helpers available to +packaged TypeScript migrations through `@sentry/junior/migration-helpers/v1`. + +Helpers may expose stable parsing primitives, state/database adapters, stores, +and key resolution. They must not contain one-off migration decisions or data +transforms. Logic such as mapping one retired record shape into another belongs +only in the journal entry that performs that migration. + +Never change the behavior or signature of an existing version in a way that can +break a pending migration. Add a new versioned entrypoint when the helper +contract must change. diff --git a/packages/junior/src/migration-helpers/v1.ts b/packages/junior/src/migration-helpers/v1.ts new file mode 100644 index 000000000..f6c922991 --- /dev/null +++ b/packages/junior/src/migration-helpers/v1.ts @@ -0,0 +1,234 @@ +import type { Destination } from "@sentry/junior-plugin-api"; +import type { + MigrationDatabaseAdapter, + MigrationStateV1, +} from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import { + isRecord as runtimeIsRecord, + toOptionalNumber as runtimeToOptionalNumber, + toOptionalString as runtimeToOptionalString, +} from "@/chat/coerce"; +import { getChatConfig } from "@/chat/config"; +import { createStateConversationStore } from "@/chat/conversations/state"; +import { createSqlStore } from "@/chat/conversations/sql/store"; +import { + parseDestination as runtimeParseDestination, + sameDestination as runtimeSameDestination, +} from "@/chat/destination"; +import { coerceThreadConversationState as runtimeCoerceThreadConversationState } from "@/chat/state/conversation"; +import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; +import { + getConversation, + requestConversationWork, +} from "@/chat/task-execution/state"; +import { addAgentTurnUsage as runtimeAddAgentTurnUsage } from "@/chat/usage"; +import type { JuniorSqlDatabase } from "@/db/db"; + +export const JUNIOR_THREAD_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1_000; + +export type MigrationSourceV1 = + | "api" + | "internal" + | "local" + | "plugin" + | "resource_event" + | "scheduler" + | "slack"; + +export type MigrationExecutionStatusV1 = + | "awaiting_resume" + | "failed" + | "idle" + | "pending" + | "running"; + +export interface MigrationInboundMessageV1 { + attemptCount?: number; + conversationId: string; + createdAtMs: number; + destination: Destination; + inboundMessageId: string; + injectedAtMs?: number; + input: { + attachments?: unknown[]; + authorId?: string; + metadata?: Record; + text: string; + }; + receivedAtMs: number; + source: MigrationSourceV1; +} + +export interface MigrationLeaseV1 { + acquiredAtMs: number; + expiresAtMs: number; + lastCheckInAtMs: number; + token: string; +} + +export interface MigrationConversationV1 { + actor?: unknown; + channelName?: string; + conversationId: string; + createdAtMs: number; + destination?: Destination; + execution: { + inboundMessageIds?: string[]; + lastCheckpointAtMs?: number; + lastEnqueuedAtMs?: number; + lease?: MigrationLeaseV1; + pendingCount?: number; + pendingMessages?: MigrationInboundMessageV1[]; + runId?: string; + status: MigrationExecutionStatusV1; + updatedAtMs?: number; + }; + lastActivityAtMs: number; + schemaVersion: 1; + source?: MigrationSourceV1; + title?: string; + updatedAtMs: number; +} + +export interface MigrationRetainedConversationV1 extends MigrationConversationV1 { + execution: MigrationConversationV1["execution"] & { + inboundMessageIds: string[]; + pendingCount: number; + pendingMessages: MigrationInboundMessageV1[]; + }; +} + +export interface MigrationThreadConversationStateV1 { + processing: { activeTurnId?: string }; +} + +export interface MigrationTurnSessionSummaryV1 { + cumulativeDurationMs: number; + cumulativeUsage?: Record; + sessionId: string; +} + +export interface MigrationStateConversationStoreV1 { + listByActivity(args: { limit: number }): Promise; +} + +export interface MigrationSqlConversationStoreV1 { + backfillConversation( + conversation: MigrationConversationV1, + metrics?: { + durationMs: number; + executionDurationMs: number; + executionUsage?: Record; + usage?: Record; + }, + ): Promise; + listByActivity(args: { limit: number }): Promise; +} + +/** Return whether a value is a non-null object record. */ +export function isRecord(value: unknown): value is Record { + return runtimeIsRecord(value); +} + +/** Return a finite number or undefined. */ +export function toOptionalNumber(value: unknown): number | undefined { + return runtimeToOptionalNumber(value); +} + +/** Return a non-empty string or undefined. */ +export function toOptionalString(value: unknown): string | undefined { + return runtimeToOptionalString(value); +} + +/** Parse one persisted destination through the stable destination schema. */ +export function parseDestination(value: unknown): Destination | undefined { + return runtimeParseDestination(value); +} + +/** Compare two persisted destinations by routing identity. */ +export function sameDestination( + left: Destination, + right: Destination, +): boolean { + return runtimeSameDestination(left, right); +} + +/** Coerce one legacy thread-state value into the retained conversation shape. */ +export function coerceThreadConversationState( + value: unknown, +): MigrationThreadConversationStateV1 { + return runtimeCoerceThreadConversationState( + value, + ) as unknown as MigrationThreadConversationStateV1; +} + +/** Merge retained turn usage records. */ +export function addAgentTurnUsage( + ...values: Array | undefined> +): Record | undefined { + return runtimeAddAgentTurnUsage(...values) as + | Record + | undefined; +} + +/** Resolve a logical state key to the configured physical Redis key. */ +export function migrationRedisKey(key: string): string { + const prefix = getChatConfig().state.keyPrefix; + return [...(prefix ? [prefix] : []), key].join(":"); +} + +/** Create the retained state-backed conversation store for a v1 migration. */ +export function createMigrationStateConversationStore( + state: MigrationStateV1, +): MigrationStateConversationStoreV1 { + return createStateConversationStore( + state as StateAdapter, + ) as unknown as MigrationStateConversationStoreV1; +} + +/** Create the SQL conversation store for a v1 migration database adapter. */ +export function createMigrationSqlStore( + database: MigrationDatabaseAdapter, +): MigrationSqlConversationStoreV1 { + return createSqlStore( + database as unknown as JuniorSqlDatabase, + ) as unknown as MigrationSqlConversationStoreV1; +} + +/** Read one retained conversation through the v1 migration state adapter. */ +export async function getMigrationConversation(args: { + conversationId: string; + state: MigrationStateV1; +}): Promise { + return (await getConversation({ + conversationId: args.conversationId, + state: args.state as StateAdapter, + })) as unknown as MigrationRetainedConversationV1 | undefined; +} + +/** Mark one retained conversation runnable through the v1 migration state adapter. */ +export async function requestMigrationConversationWork(args: { + conversationId: string; + destination: Destination; + nowMs?: number; + state: MigrationStateV1; +}): Promise { + await requestConversationWork({ + conversationId: args.conversationId, + destination: args.destination, + ...(args.nowMs === undefined ? {} : { nowMs: args.nowMs }), + state: args.state as StateAdapter, + }); +} + +/** Read retained turn summaries through the v1 migration state adapter. */ +export async function listMigrationTurnSessionSummaries( + state: MigrationStateV1, + conversationIds: string[], +): Promise> { + return (await listAgentTurnSessionSummariesForConversations( + state as StateAdapter, + conversationIds, + )) as Map; +} diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index b2ce0516d..507cadf64 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -2,7 +2,10 @@ import path from "node:path"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { createMemoryState } from "@chat-adapter/state-memory"; -import { resolveMigrations } from "@sentry/junior-migrations"; +import { + resolveMigrations, + type MigrationContextV1, +} from "@sentry/junior-migrations"; import { defineJuniorPlugin } from "@sentry/junior-plugin-api"; import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { @@ -84,7 +87,7 @@ async function runSchedulerStateMigration(args: { load: async () => undefined, save: async () => {}, }, - state: args.stateAdapter, + state: args.stateAdapter as unknown as MigrationContextV1["state"], }); } diff --git a/packages/junior/tsconfig.build.json b/packages/junior/tsconfig.build.json index b94167f70..bbec21a3d 100644 --- a/packages/junior/tsconfig.build.json +++ b/packages/junior/tsconfig.build.json @@ -13,6 +13,7 @@ "src/app.ts", "src/handlers/**/*.ts", "src/instrumentation.ts", + "src/migration-helpers/**/*.ts", "src/nitro.ts", "src/reporting.ts", "src/vercel.ts", diff --git a/packages/junior/tsup.config.ts b/packages/junior/tsup.config.ts index 9eb35b75a..ca9dbf69a 100644 --- a/packages/junior/tsup.config.ts +++ b/packages/junior/tsup.config.ts @@ -15,6 +15,7 @@ export default defineConfig({ api: "src/api.ts", "api/schema": "src/api/schema.ts", instrumentation: "src/instrumentation.ts", + "migration-helpers/v1": "src/migration-helpers/v1.ts", nitro: "src/nitro.ts", vercel: "src/vercel.ts", version: "src/version.ts", From 068a5ff5ea67777377e3b6dcdf3056818ba9fdcf Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 16 Jul 2026 12:08:21 -0700 Subject: [PATCH 05/15] ref(migrations): Reduce mixed journal bloat Rebase the mixed journal onto the latest schema, move the new conversation usage repair into the journal, and preserve its focused tests. Remove unchanged Drizzle snapshots for data entries, rebuild the legacy history migration around versioned infrastructure helpers, and reject runtime re-export bypasses. Co-Authored-By: GPT-5.6 Codex --- packages/docs/src/content/docs/cli/upgrade.md | 4 +- packages/junior-migrations/README.md | 6 +- packages/junior-migrations/src/generate.ts | 8 ++- packages/junior-migrations/src/journal.ts | 32 ++++++--- .../junior-migrations/tests/generate.test.ts | 3 + .../junior-migrations/tests/journal.test.ts | 12 ++++ packages/junior/migrations/README.md | 7 +- packages/junior/src/cli/upgrade.ts | 2 - packages/junior/src/migration-helpers/v1.ts | 66 +++++++++++++++++++ .../tests/component/cli/upgrade-cli.test.ts | 10 +-- 10 files changed, 124 insertions(+), 26 deletions(-) diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index 7052383cd..426c71559 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -80,9 +80,7 @@ Typical logs look like this: ```text Running Junior upgrade migrations... Running migration core-migrations... -Finished migration core-migrations: scanned=8 migrated=4 existing=4 missing=0 skipped=0 -Running migration repair-conversation-usage... -Finished migration repair-conversation-usage: scanned=2 migrated=1 existing=1 missing=0 +Finished migration core-migrations: scanned=10 migrated=6 existing=4 missing=0 skipped=0 Running migration plugin-migrations... Finished migration plugin-migrations: scanned=8 migrated=8 existing=0 missing=0 skipped=0 Junior upgrade complete. diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index 1abc59712..9772335d7 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -22,8 +22,10 @@ junior-migrations generate \ --name backfill_actor ``` -The wrapper asks Drizzle Kit to create a custom journal entry and snapshot, -then replaces the empty SQL file with a `MigrationV1` TypeScript scaffold. +The wrapper asks Drizzle Kit to create a custom journal entry, replaces the +empty SQL file with a `MigrationV1` TypeScript scaffold, and removes the +unchanged custom snapshot. Schema generation continues from the latest real +schema snapshot while preserving the mixed journal order. ## Compatibility diff --git a/packages/junior-migrations/src/generate.ts b/packages/junior-migrations/src/generate.ts index 3c78c79a3..20e703fe8 100644 --- a/packages/junior-migrations/src/generate.ts +++ b/packages/junior-migrations/src/generate.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile, rename, rm, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { readMigrationJournal } from "./journal"; @@ -62,7 +62,13 @@ export async function generateTypeScriptMigration( throw new Error(`Refusing to replace non-custom migration ${entry.tag}`); } const typescriptPath = resolve(folder, `${entry.tag}.ts`); + const snapshotPath = resolve( + folder, + "meta", + `${String(entry.index).padStart(4, "0")}_snapshot.json`, + ); await rename(sqlPath, typescriptPath); + await rm(snapshotPath, { force: true }); await writeFile(typescriptPath, scaffold(), "utf8"); return typescriptPath; } diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index 826ffef26..685533b7e 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -54,15 +54,7 @@ function validateTypeScriptSource(tag: string, source: string): void { const allowedRuntimeImports = new Set([ "@sentry/junior/migration-helpers/v1", ]); - const imports = source.matchAll( - /^\s*import\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, - ); - for (const match of imports) { - const clause = match[1]?.trim(); - const specifier = match[2]; - if (clause?.startsWith("type ")) { - continue; - } + const validateRuntimeSpecifier = (specifier: string | undefined): void => { if ( !specifier || specifier.startsWith(".") || @@ -76,10 +68,30 @@ function validateTypeScriptSource(tag: string, source: string): void { `TypeScript migration ${tag} cannot import application runtime code`, ); } + }; + const imports = source.matchAll( + /^\s*import\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, + ); + for (const match of imports) { + const clause = match[1]?.trim(); + const specifier = match[2]; + if (clause?.startsWith("type ")) { + continue; + } + validateRuntimeSpecifier(specifier); + } + const exports = source.matchAll( + /^\s*export\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, + ); + for (const match of exports) { + const clause = match[1]?.trim(); + if (clause?.startsWith("type ")) { + continue; + } + validateRuntimeSpecifier(match[2]); } if ( /^\s*import\s*["']/m.test(source) || - /^\s*export\s+.+\s+from\s+["'][./]/m.test(source) || /\b(?:import\s*\(|require\s*\()/.test(source) ) { throw new Error(`TypeScript migration ${tag} cannot load runtime modules`); diff --git a/packages/junior-migrations/tests/generate.test.ts b/packages/junior-migrations/tests/generate.test.ts index 399d368a8..1513a6312 100644 --- a/packages/junior-migrations/tests/generate.test.ts +++ b/packages/junior-migrations/tests/generate.test.ts @@ -70,6 +70,9 @@ it("keeps Drizzle schema generation working across a TypeScript entry", async () await expect( access(join(root, "migrations", "0001_backfill.sql")), ).rejects.toThrow("ENOENT"); + await expect( + access(join(root, "migrations", "meta", "0001_snapshot.json")), + ).rejects.toThrow("ENOENT"); await writeFile( join(root, "schema.ts"), diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index 0acef14cf..4309b58e6 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -74,6 +74,18 @@ describe("resolveMigrations", () => { ]); }); + it("rejects runtime re-exports of Junior internals", async () => { + const folder = await migrationFolder(["0000_reexport"]); + await writeFile( + join(folder, "0000_reexport.ts"), + 'export { getChatConfig } from "@sentry/junior/internal";\nexport default { apiVersion: 1, async up() {} };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot import application runtime code", + ); + }); + it("rejects ambiguous migration files", async () => { const folder = await migrationFolder(["0000_ambiguous"]); await writeFile(join(folder, "0000_ambiguous.sql"), "SELECT 1;"); diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index fccbe54ac..097fd194f 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -3,9 +3,10 @@ `src/db/schema.ts` is the Drizzle schema entrypoint. This directory is the append-only history used to bring an existing database to that schema. It extends Drizzle Kit's journal with self-contained -TypeScript data migrations. Drizzle Kit owns numbering, snapshots, and journal -entries; `junior upgrade` executes each entry as either `.sql` or -`.ts` through `@sentry/junior-migrations`. +TypeScript data migrations. Drizzle Kit owns numbering, schema snapshots, and +journal entries; `junior upgrade` executes each entry as either `.sql` or +`.ts` through `@sentry/junior-migrations`. Data entries do not retain an +unchanged snapshot. `junior upgrade` runs the ordered migration list in `src/cli/upgrade.ts`. Core SQL runs under a migration lock and is recorded in diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 225a651c6..b5ff82c17 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -6,7 +6,6 @@ import { createJiti } from "jiti"; import { loadAppPluginSet } from "@/plugin-module"; import { migrateCoreJournal } from "./upgrade/migrations/core-journal"; import { migratePluginJournals } from "./upgrade/migrations/plugin-journal"; -import { conversationUsageRepairMigration } from "./upgrade/migrations/conversation-usage"; import { resolveUpgradePlugins } from "./upgrade/migrations/upgrade-plugins"; import type { MigrationContext, @@ -85,7 +84,6 @@ export async function runUpgradeMigrations( results.push(result); }; await run("core-migrations", migrateCoreJournal); - await run("repair-conversation-usage", conversationUsageRepairMigration.run); await run("plugin-migrations", migratePluginJournals); return results; } diff --git a/packages/junior/src/migration-helpers/v1.ts b/packages/junior/src/migration-helpers/v1.ts index f6c922991..1c4561ead 100644 --- a/packages/junior/src/migration-helpers/v1.ts +++ b/packages/junior/src/migration-helpers/v1.ts @@ -10,22 +10,47 @@ import { toOptionalString as runtimeToOptionalString, } from "@/chat/coerce"; import { getChatConfig } from "@/chat/config"; +import { agentStepEntrySchema } from "@/chat/conversations/history"; +import { createLegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; +import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { createStateConversationStore } from "@/chat/conversations/state"; import { createSqlStore } from "@/chat/conversations/sql/store"; +import { toStoredConversationMessage } from "@/chat/conversations/visible-messages"; import { parseDestination as runtimeParseDestination, sameDestination as runtimeSameDestination, } from "@/chat/destination"; import { coerceThreadConversationState as runtimeCoerceThreadConversationState } from "@/chat/state/conversation"; import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; +import { + contextProvenance, + decodeSessionLogEntry, + legacyActorProvenance, + piMessageProvenanceSchema, +} from "@/chat/state/session-log"; import { getConversation, requestConversationWork, } from "@/chat/task-execution/state"; import { addAgentTurnUsage as runtimeAddAgentTurnUsage } from "@/chat/usage"; +import { unescapeXml } from "@/chat/xml"; import type { JuniorSqlDatabase } from "@/db/db"; +import { + juniorAgentSteps, + juniorConversationMessages, + juniorConversations, +} from "@/db/schema"; export const JUNIOR_THREAD_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1_000; +export const migrationAgentStepEntrySchema: unknown = agentStepEntrySchema; +export const migrationContextProvenance: unknown = contextProvenance; +export const migrationJuniorAgentSteps: unknown = juniorAgentSteps; +export const migrationJuniorConversationMessages: unknown = + juniorConversationMessages; +export const migrationJuniorConversations: unknown = juniorConversations; +export const migrationPiMessageProvenanceSchema: unknown = + piMessageProvenanceSchema; export type MigrationSourceV1 = | "api" @@ -196,6 +221,47 @@ export function createMigrationSqlStore( ) as unknown as MigrationSqlConversationStoreV1; } +/** Create the SQL agent-step store used by legacy history migrations. */ +export function createMigrationSqlAgentStepStore( + database: MigrationDatabaseAdapter, +): unknown { + return createSqlAgentStepStore(database as unknown as JuniorSqlDatabase); +} + +/** Create the SQL message store used by legacy history migrations. */ +export function createMigrationSqlConversationMessageStore( + database: MigrationDatabaseAdapter, +): unknown { + return createSqlConversationMessageStore( + database as unknown as JuniorSqlDatabase, + ); +} + +/** Create the legacy advisor reader used by history migrations. */ +export function createMigrationLegacyAdvisorSessionReader(): unknown { + return createLegacyAdvisorSessionReader(); +} + +/** Decode one retained session-log value. */ +export function decodeMigrationSessionLogEntry(value: unknown): unknown { + return decodeSessionLogEntry(value); +} + +/** Recover provenance from one legacy actor value. */ +export function migrationLegacyActorProvenance(value: unknown): unknown { + return legacyActorProvenance(value as never); +} + +/** Convert one legacy visible message into its SQL projection. */ +export function toMigrationStoredConversationMessage(value: unknown): unknown { + return toStoredConversationMessage(value as never); +} + +/** Unescape one legacy XML text fragment. */ +export function migrationUnescapeXml(value: string): string { + return unescapeXml(value); +} + /** Read one retained conversation through the v1 migration state adapter. */ export async function getMigrationConversation(args: { conversationId: string; diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 5136b0ed6..fdb0c42c0 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -20,10 +20,10 @@ import type { PiMessage } from "@/chat/pi/messages"; import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { recordAgentTurnSessionSummary } from "@/chat/state/turn-session"; import { resolveUpgradePluginSet } from "@/cli/upgrade"; -import { repairConversationUsage } from "@/cli/upgrade/migrations/conversation-usage"; import { migrateAgentTurnSessionActor } from "../../../migrations/0005_agent_turn_session_actor"; import { migrateRedisConversationState } from "../../../migrations/0006_redis_conversation_state"; import { migrateConversationsToSql } from "../../../migrations/0007_conversations_to_sql"; +import { repairConversationUsage } from "../../../migrations/0009_conversation_usage"; import { CONVERSATION_ID, SLACK_DESTINATION, @@ -334,8 +334,8 @@ export const plugins = { }), ).resolves.toEqual({ existing: 0, - migrated: 8, - scanned: 8, + migrated: 10, + scanned: 10, skipped: 0, }); await expect( @@ -368,9 +368,9 @@ export const plugins = { stateAdapter, }), ).resolves.toEqual({ - existing: 8, + existing: 10, migrated: 0, - scanned: 8, + scanned: 10, skipped: 0, }); } finally { From 1c7c24489d3f38df5b8f5308b448e87ced97e930 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 16 Jul 2026 12:52:22 -0700 Subject: [PATCH 06/15] fix(migrations): Address adversarial review findings Freeze complete scheduler migration schemas, repair malformed existing scheduler rows, and add focused regression coverage. Harden migration path and JSON boundaries, narrow the runner-only locking interface, remove dead helper surface, deduplicate state projection, and preserve original migration errors. Co-Authored-By: GPT-5.6 Codex --- packages/docs/src/content/docs/cli/upgrade.md | 2 +- packages/junior-migrations/README.md | 6 +- packages/junior-migrations/package.json | 2 +- packages/junior-migrations/src/generate.ts | 15 ++- packages/junior-migrations/src/index.ts | 1 + packages/junior-migrations/src/journal.ts | 29 ++++-- packages/junior-migrations/src/runner.ts | 91 ++++++++++++++++--- packages/junior-migrations/src/types.ts | 6 +- .../junior-migrations/tests/generate.test.ts | 20 ++++ .../junior-migrations/tests/journal.test.ts | 24 ++++- .../junior-migrations/tests/runner.test.ts | 27 ++++++ .../src/chat/conversations/sql/migrations.ts | 31 +------ .../junior/src/chat/migrations/state-v1.ts | 32 +++++++ .../junior/src/chat/plugins/migrations.ts | 31 +------ packages/junior/src/migration-helpers/v1.ts | 18 ++-- .../component/scheduler-sql-plugin.test.ts | 46 +++++++++- .../tests/fixtures/postgres/executor.ts | 4 +- packages/junior/tests/fixtures/sql.ts | 3 +- 18 files changed, 287 insertions(+), 101 deletions(-) create mode 100644 packages/junior/src/chat/migrations/state-v1.ts diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index 426c71559..9ef322d35 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -34,7 +34,7 @@ SQL schema migrations or TypeScript data migrations targeting a versioned host capability API. Current upgrade work includes: -- Apply core and enabled-plugin SQL schema migrations. +- Apply core and enabled-plugin schema and data journal entries. - Rewrite retained turn-session records from legacy storage shapes before the new runtime reads them. - Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed. diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index 9772335d7..abf230771 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -6,9 +6,9 @@ migrations. Every journal entry resolves to exactly one `.sql` or the Junior migration runner owns execution and exact per-entry tracking. TypeScript migrations target a versioned capability API and must not import -Junior runtime internals. Every parser, transform, and migration-specific -helper belongs permanently in the migration file so application refactors or -deletions cannot break pending upgrades. +Junior runtime internals. Migration-specific parsers and transforms belong +permanently in the migration file so application refactors or deletions cannot +break pending upgrades. ## Authoring diff --git a/packages/junior-migrations/package.json b/packages/junior-migrations/package.json index 391e14d0f..36af03dbf 100644 --- a/packages/junior-migrations/package.json +++ b/packages/junior-migrations/package.json @@ -1,6 +1,6 @@ { "name": "@sentry/junior-migrations", - "version": "0.101.0", + "version": "0.104.2", "private": false, "publishConfig": { "access": "public" diff --git a/packages/junior-migrations/src/generate.ts b/packages/junior-migrations/src/generate.ts index 20e703fe8..97beca0c7 100644 --- a/packages/junior-migrations/src/generate.ts +++ b/packages/junior-migrations/src/generate.ts @@ -29,13 +29,26 @@ function scaffold(): string { return `import type { MigrationV1 } from "@sentry/junior-migrations";\n\nconst migration = {\n apiVersion: 1,\n async up(context) {\n void context;\n },\n} satisfies MigrationV1;\n\nexport default migration;\n`; } +function isMissingJournal(error: unknown): boolean { + return ( + error instanceof Error && + (error.cause as NodeJS.ErrnoException | undefined)?.code === "ENOENT" + ); +} + /** Create a journaled TypeScript migration through Drizzle Kit's custom generator. */ export async function generateTypeScriptMigration( options: GenerateTypeScriptMigrationOptions, ): Promise { const cwd = resolve(options.cwd ?? process.cwd()); const folder = resolve(cwd, options.migrationsFolder); - const before = await readMigrationJournal(folder).catch(() => []); + let before; + try { + before = await readMigrationJournal(folder); + } catch (error) { + if (!isMissingJournal(error)) throw error; + before = []; + } await run( "drizzle-kit", [ diff --git a/packages/junior-migrations/src/index.ts b/packages/junior-migrations/src/index.ts index 6c07a55ad..1770006de 100644 --- a/packages/junior-migrations/src/index.ts +++ b/packages/junior-migrations/src/index.ts @@ -8,6 +8,7 @@ export type { MigrationDatabaseAdapter, MigrationJsonValue, MigrationJournalEntry, + MigrationJournalExecutor, MigrationLockV1, MigrationProgressV1, MigrationRedisV1, diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index 685533b7e..56ab5c969 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { isAbsolute, join, relative, sep } from "node:path"; import type { MigrationJournalEntry, ResolvedMigration } from "./types"; interface DrizzleJournalEntry { @@ -26,7 +26,7 @@ function parseJournalEntry( typeof value.when !== "number" || !Number.isFinite(value.when) || typeof value.tag !== "string" || - !value.tag || + !/^[a-z0-9][a-z0-9_-]*$/.test(value.tag) || typeof value.breakpoints !== "boolean" ) { throw new Error(`Invalid Drizzle journal entry at index ${position}`); @@ -39,8 +39,24 @@ function parseJournalEntry( }; } -async function optionalFile(path: string): Promise { +async function optionalFile( + path: string, + migrationsRoot: string, +): Promise { try { + const stat = await lstat(path); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`Migration source must be a regular file: ${path}`); + } + const canonical = await realpath(path); + const fromRoot = relative(migrationsRoot, canonical); + if ( + fromRoot === ".." || + fromRoot.startsWith(`..${sep}`) || + isAbsolute(fromRoot) + ) { + throw new Error(`Migration source escapes its journal root: ${path}`); + } return await readFile(path, "utf8"); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { @@ -133,14 +149,15 @@ export async function readMigrationJournal( export async function resolveMigrations( migrationsFolder: string, ): Promise { + const migrationsRoot = await realpath(migrationsFolder); const entries = await readMigrationJournal(migrationsFolder); return await Promise.all( entries.map(async (entry) => { const sqlPath = join(migrationsFolder, `${entry.tag}.sql`); const typescriptPath = join(migrationsFolder, `${entry.tag}.ts`); const [sqlSource, typescriptSource] = await Promise.all([ - optionalFile(sqlPath), - optionalFile(typescriptPath), + optionalFile(sqlPath, migrationsRoot), + optionalFile(typescriptPath, migrationsRoot), ]); if ((sqlSource === undefined) === (typescriptSource === undefined)) { throw new Error( diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index d0d043e93..7eb99035a 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -1,6 +1,10 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; import type { MigrationContextV1, MigrationDatabaseAdapter, + MigrationJournalExecutor, + MigrationJsonValue, MigrationRunResult, MigrationV1, ResolvedMigration, @@ -15,9 +19,51 @@ interface MigrationRow { status: string | null; } +function migrationJsonValue( + value: unknown, + label: string, + seen = new WeakSet(), +): MigrationJsonValue { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (Number.isFinite(value)) return value; + throw new Error(`${label} must contain only finite JSON numbers`); + } + if (typeof value !== "object") { + throw new Error(`${label} must be JSON-compatible`); + } + if (seen.has(value)) { + throw new Error(`${label} must not contain circular references`); + } + seen.add(value); + if (Array.isArray(value)) { + const result = value.map((item) => migrationJsonValue(item, label, seen)); + seen.delete(value); + return result; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${label} must contain only plain JSON objects`); + } + const result = Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + migrationJsonValue(item, label, seen), + ]), + ); + seen.delete(value); + return result; +} + interface RunMigrationJournalBaseOptions { beforeRun?: () => Promise; - executor: MigrationDatabaseAdapter; + executor: MigrationJournalExecutor; migrationsFolder: string; migrationsTable: string; } @@ -54,7 +100,7 @@ function qualifiedTable(table: string): string { } async function ensureMigrationTable( - executor: MigrationDatabaseAdapter, + executor: MigrationJournalExecutor, table: string, ): Promise { const qualified = qualifiedTable(table); @@ -90,7 +136,7 @@ CREATE TABLE IF NOT EXISTS ${qualified} ( } async function migrationRows( - executor: MigrationDatabaseAdapter, + executor: MigrationJournalExecutor, table: string, ): Promise> { const rows = await executor.query(` @@ -107,7 +153,7 @@ ORDER BY created_at ASC, id ASC } async function adoptLegacySqlPrefix(args: { - executor: MigrationDatabaseAdapter; + executor: MigrationJournalExecutor; migrations: readonly ResolvedMigration[]; rows: Map; table: string; @@ -160,7 +206,7 @@ function migrationV1(value: unknown, tag: string): MigrationV1 { } async function runSqlMigration(args: { - executor: MigrationDatabaseAdapter; + executor: MigrationJournalExecutor; migration: ResolvedMigration; table: string; }): Promise { @@ -211,33 +257,54 @@ async function runTypeScriptMigration(args: { }>(`SELECT progress FROM ${qualified} WHERE created_at = $1 LIMIT 1`, [ args.migration.when, ]); - return current?.progress ?? undefined; + return current?.progress == null + ? undefined + : migrationJsonValue(current.progress, "Migration progress"); }, async save(value) { + const progressValue = migrationJsonValue(value, "Migration progress"); await args.executor.execute( `UPDATE ${qualified} SET progress = $1, status = 'running' WHERE created_at = $2`, - [value, args.migration.when], + [progressValue, args.migration.when], ); }, }; try { const context = args.createContext({ migration: args.migration, progress }); + const currentSource = await readFile(args.migration.path, "utf8"); + const currentHash = createHash("sha256") + .update(currentSource) + .digest("hex"); + if (currentHash !== args.migration.hash) { + throw new Error(`Migration ${args.migration.tag} changed before loading`); + } const migration = migrationV1( await args.loadTypeScript(args.migration.path), args.migration.tag, ); const result = await migration.up(context); + const resultValue = + result === undefined + ? null + : migrationJsonValue(result, "Migration result"); await args.executor.execute( `UPDATE ${qualified} SET status = 'completed', result = $1, completed_at = NOW() WHERE created_at = $2`, - [result ?? null, args.migration.when], + [resultValue, args.migration.when], ); } catch (error) { - await args.executor.execute( - `UPDATE ${qualified} SET status = 'failed' WHERE created_at = $1`, - [args.migration.when], - ); + try { + await args.executor.execute( + `UPDATE ${qualified} SET status = 'failed' WHERE created_at = $1`, + [args.migration.when], + ); + } catch (statusError) { + throw new AggregateError( + [error, statusError], + `Migration ${args.migration.tag} failed and its failure status could not be persisted`, + ); + } throw error; } } diff --git a/packages/junior-migrations/src/types.ts b/packages/junior-migrations/src/types.ts index dd2aa6813..e3b3b2528 100644 --- a/packages/junior-migrations/src/types.ts +++ b/packages/junior-migrations/src/types.ts @@ -1,4 +1,4 @@ -/** Database adapter used by the mixed migration runner and TypeScript entries. */ +/** Database capabilities exposed to v1 TypeScript migrations. */ export interface MigrationDatabaseAdapter { db(): unknown; execute(statement: string, parameters?: readonly unknown[]): Promise; @@ -8,6 +8,10 @@ export interface MigrationDatabaseAdapter { ): Promise; transaction(callback: () => Promise): Promise; withLock(lockName: string, callback: () => Promise): Promise; +} + +/** Host executor used to serialize and persist one migration journal. */ +export interface MigrationJournalExecutor extends MigrationDatabaseAdapter { withMigrationLock( migrationTable: string, callback: () => Promise, diff --git a/packages/junior-migrations/tests/generate.test.ts b/packages/junior-migrations/tests/generate.test.ts index 1513a6312..d594a1a67 100644 --- a/packages/junior-migrations/tests/generate.test.ts +++ b/packages/junior-migrations/tests/generate.test.ts @@ -100,3 +100,23 @@ it("keeps Drizzle schema generation working across a TypeScript entry", async () readFile(join(root, "migrations", "0002_add_name.sql"), "utf8"), ).resolves.toContain('ADD COLUMN "name" text'); }); + +it("does not invoke Drizzle when an existing journal is invalid", async () => { + const relativeRoot = `.tmp-invalid-migrations-${Date.now()}`; + const root = join(process.cwd(), relativeRoot); + temporaryDirectories.push(root); + await mkdir(join(root, "migrations", "meta"), { recursive: true }); + await writeFile( + join(root, "migrations", "meta", "_journal.json"), + JSON.stringify({ dialect: "sqlite", entries: [] }), + ); + + await expect( + generateTypeScriptMigration({ + configPath: `${relativeRoot}/drizzle.config.ts`, + cwd: process.cwd(), + migrationsFolder: `${relativeRoot}/migrations`, + name: "backfill", + }), + ).rejects.toThrow("Unsupported Drizzle journal"); +}); diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index 4309b58e6..a25336a8b 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -98,4 +98,26 @@ describe("resolveMigrations", () => { "exactly one .sql or .ts file", ); }); + + it("rejects journal tags that escape the migration directory", async () => { + const folder = await migrationFolder(["../outside"]); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "Invalid Drizzle journal entry", + ); + }); + + it("rejects symlinked migration sources", async () => { + const folder = await migrationFolder(["0000_symlink"]); + const outside = join(folder, "..", "outside-migration.ts"); + await writeFile( + outside, + "export default { apiVersion: 1, async up() {} };\n", + ); + await symlink(outside, join(folder, "0000_symlink.ts")); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "Migration source must be a regular file", + ); + }); }); diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index c072487b5..6779e4127 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -268,4 +268,31 @@ describe("runMigrationJournal", () => { "SELECT 'schema-two';", ]); }); + + it("rejects non-JSON migration results before completing the ledger row", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + const migration = { + apiVersion: 1, + async up() { + return Number.NaN as never; + }, + } satisfies MigrationV1; + + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }), + ).rejects.toThrow("Migration result must contain only finite JSON numbers"); + expect(executor.rows.get(2_001)?.status).toBe("failed"); + }); }); diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index bdadcf182..4b50ed911 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -6,11 +6,11 @@ import { runMigrationJournal, type MigrationContextV1, type MigrationRunResult, - type MigrationStateV1, type TypeScriptMigrationLoader, } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import { createMigrationStateV1 } from "@/chat/migrations/state-v1"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { juniorSqlSchema as schema } from "@/db/schema"; @@ -184,33 +184,6 @@ CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( }); } -function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { - return { - acquireLock: async (threadId, ttlMs) => - await stateAdapter.acquireLock(threadId, ttlMs), - appendToList: async (key, value, options) => { - await stateAdapter.appendToList(key, value, options); - }, - connect: async () => { - await stateAdapter.connect(); - }, - delete: async (key) => { - await stateAdapter.delete(key); - }, - get: async (key: string) => - (await stateAdapter.get(key)) ?? undefined, - getList: async (key: string) => await stateAdapter.getList(key), - releaseLock: async (lock) => { - await stateAdapter.releaseLock(lock); - }, - set: async (key, value, ttlMs) => { - await stateAdapter.set(key, value, ttlMs); - }, - setIfNotExists: async (key, value, ttlMs) => - await stateAdapter.setIfNotExists(key, value, ttlMs), - }; -} - export type MigrateSchemaOptions = | { mode?: "sql" } | { @@ -257,7 +230,7 @@ export async function migrateSchema( }, } : {}), - state: migrationState(options.stateAdapter), + state: createMigrationStateV1(options.stateAdapter), }), loadTypeScript: options.loadTypeScript, mode: "all", diff --git a/packages/junior/src/chat/migrations/state-v1.ts b/packages/junior/src/chat/migrations/state-v1.ts new file mode 100644 index 000000000..dd93a9ab3 --- /dev/null +++ b/packages/junior/src/chat/migrations/state-v1.ts @@ -0,0 +1,32 @@ +import type { MigrationStateV1 } from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; + +/** Project the host state adapter onto the permanent v1 migration capability. */ +export function createMigrationStateV1( + stateAdapter: StateAdapter, +): MigrationStateV1 { + return { + acquireLock: async (threadId, ttlMs) => + await stateAdapter.acquireLock(threadId, ttlMs), + appendToList: async (key, value, options) => { + await stateAdapter.appendToList(key, value, options); + }, + connect: async () => { + await stateAdapter.connect(); + }, + delete: async (key) => { + await stateAdapter.delete(key); + }, + get: async (key: string) => + (await stateAdapter.get(key)) ?? undefined, + getList: async (key: string) => await stateAdapter.getList(key), + releaseLock: async (lock) => { + await stateAdapter.releaseLock(lock); + }, + set: async (key, value, ttlMs) => { + await stateAdapter.set(key, value, ttlMs); + }, + setIfNotExists: async (key, value, ttlMs) => + await stateAdapter.setIfNotExists(key, value, ttlMs), + }; +} diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index a0321beea..59fc2e406 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -3,11 +3,11 @@ import { resolveMigrations, runMigrationJournal, type MigrationContextV1, - type MigrationStateV1, type ResolvedMigration, type TypeScriptMigrationLoader, } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; +import { createMigrationStateV1 } from "@/chat/migrations/state-v1"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; interface PluginMigrationRoot { @@ -131,33 +131,6 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( }); } -function migrationState(stateAdapter: StateAdapter): MigrationStateV1 { - return { - acquireLock: async (threadId, ttlMs) => - await stateAdapter.acquireLock(threadId, ttlMs), - appendToList: async (key, value, options) => { - await stateAdapter.appendToList(key, value, options); - }, - connect: async () => { - await stateAdapter.connect(); - }, - delete: async (key) => { - await stateAdapter.delete(key); - }, - get: async (key: string) => - (await stateAdapter.get(key)) ?? undefined, - getList: async (key: string) => await stateAdapter.getList(key), - releaseLock: async (lock) => { - await stateAdapter.releaseLock(lock); - }, - set: async (key, value, ttlMs) => { - await stateAdapter.set(key, value, ttlMs); - }, - setIfNotExists: async (key, value, ttlMs) => - await stateAdapter.setIfNotExists(key, value, ttlMs), - }; -} - /** Apply enabled plugins' mixed migrations in plugin-name and journal order. */ export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, @@ -196,7 +169,7 @@ export async function migratePluginSchemas( database: executor, log: options.log ?? (() => {}), progress, - state: migrationState(options.stateAdapter), + state: createMigrationStateV1(options.stateAdapter), }), loadTypeScript: options.loadTypeScript, mode: "all", diff --git a/packages/junior/src/migration-helpers/v1.ts b/packages/junior/src/migration-helpers/v1.ts index 1c4561ead..15665d5fd 100644 --- a/packages/junior/src/migration-helpers/v1.ts +++ b/packages/junior/src/migration-helpers/v1.ts @@ -10,9 +10,7 @@ import { toOptionalString as runtimeToOptionalString, } from "@/chat/coerce"; import { getChatConfig } from "@/chat/config"; -import { agentStepEntrySchema } from "@/chat/conversations/history"; import { createLegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; -import { createSqlAgentStepStore } from "@/chat/conversations/sql/history"; import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; import { createStateConversationStore } from "@/chat/conversations/state"; import { createSqlStore } from "@/chat/conversations/sql/store"; @@ -43,7 +41,6 @@ import { } from "@/db/schema"; export const JUNIOR_THREAD_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1_000; -export const migrationAgentStepEntrySchema: unknown = agentStepEntrySchema; export const migrationContextProvenance: unknown = contextProvenance; export const migrationJuniorAgentSteps: unknown = juniorAgentSteps; export const migrationJuniorConversationMessages: unknown = @@ -68,6 +65,7 @@ export type MigrationExecutionStatusV1 = | "pending" | "running"; +/** Frozen retained inbound-message shape used by v1 data migrations. */ export interface MigrationInboundMessageV1 { attemptCount?: number; conversationId: string; @@ -85,6 +83,7 @@ export interface MigrationInboundMessageV1 { source: MigrationSourceV1; } +/** Frozen retained execution-lease shape used by v1 data migrations. */ export interface MigrationLeaseV1 { acquiredAtMs: number; expiresAtMs: number; @@ -92,6 +91,7 @@ export interface MigrationLeaseV1 { token: string; } +/** Frozen conversation projection shared by v1 state and SQL helpers. */ export interface MigrationConversationV1 { actor?: unknown; channelName?: string; @@ -116,6 +116,7 @@ export interface MigrationConversationV1 { updatedAtMs: number; } +/** State-backed conversation projection with retained mailbox fields. */ export interface MigrationRetainedConversationV1 extends MigrationConversationV1 { execution: MigrationConversationV1["execution"] & { inboundMessageIds: string[]; @@ -124,20 +125,24 @@ export interface MigrationRetainedConversationV1 extends MigrationConversationV1 }; } +/** Legacy thread-state projection required by the v1 continuation migration. */ export interface MigrationThreadConversationStateV1 { processing: { activeTurnId?: string }; } +/** Frozen turn-summary projection used by the conversation SQL backfill. */ export interface MigrationTurnSessionSummaryV1 { cumulativeDurationMs: number; cumulativeUsage?: Record; sessionId: string; } +/** Narrow state conversation store exposed to v1 migrations. */ export interface MigrationStateConversationStoreV1 { listByActivity(args: { limit: number }): Promise; } +/** Narrow SQL conversation store exposed to v1 migrations. */ export interface MigrationSqlConversationStoreV1 { backfillConversation( conversation: MigrationConversationV1, @@ -221,13 +226,6 @@ export function createMigrationSqlStore( ) as unknown as MigrationSqlConversationStoreV1; } -/** Create the SQL agent-step store used by legacy history migrations. */ -export function createMigrationSqlAgentStepStore( - database: MigrationDatabaseAdapter, -): unknown { - return createSqlAgentStepStore(database as unknown as JuniorSqlDatabase); -} - /** Create the SQL message store used by legacy history migrations. */ export function createMigrationSqlConversationMessageStore( database: MigrationDatabaseAdapter, diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index 507cadf64..e2023ba8d 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -564,6 +564,30 @@ ORDER BY tablename scanned: 2, }); + const { credentialMode: _credentialMode, ...legacySqlTask } = task; + await fixture.sql.execute( + "UPDATE junior_scheduler_tasks SET record = $2::jsonb WHERE id = $1", + [ + task.id, + JSON.stringify({ + ...legacySqlTask, + credentialSubject: { + allowedWhen: "private-direct-conversation", + type: "user", + userId: "U123", + }, + }), + ], + ); + await expect( + runSchedulerStateMigration({ fixture, stateAdapter }), + ).resolves.toEqual({ + existing: 1, + migrated: 1, + missing: 0, + scanned: 2, + }); + const sqlStore = createSchedulerSqlStore(db); await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ id: task.id, @@ -592,7 +616,7 @@ ORDER BY tablename const badRunId = `${task.id}:${TEST_RUN_AT_MS}`; await state.set( "junior:scheduler:tasks", - ["sched_state_sql_bad", task.id], + ["sched_state_sql_bad", "sched_state_sql_missing_required", task.id], 5 * 60 * 1000, ); await state.set( @@ -616,6 +640,12 @@ ORDER BY tablename }, 5 * 60 * 1000, ); + const { schedule: _schedule, ...missingSchedule } = task; + await state.set( + "junior:scheduler:task:sched_state_sql_missing_required", + { ...missingSchedule, id: "sched_state_sql_missing_required" }, + 5 * 60 * 1000, + ); await state.set( `junior:scheduler:active:${task.id}`, { @@ -627,7 +657,12 @@ ORDER BY tablename ); await state.set( `junior:scheduler:run:${badRunId}`, - { id: badRunId }, + { + id: badRunId, + scheduledForMs: TEST_RUN_AT_MS, + status: "pending", + taskId: task.id, + }, 5 * 60 * 1000, ); @@ -636,8 +671,8 @@ ORDER BY tablename ).resolves.toEqual({ existing: 0, migrated: 1, - missing: 1, - scanned: 2, + missing: 2, + scanned: 3, }); const sqlStore = createSchedulerSqlStore(db); @@ -648,6 +683,9 @@ ORDER BY tablename await expect(sqlStore.getTask("sched_state_sql_bad")).resolves.toBe( undefined, ); + await expect( + sqlStore.getTask("sched_state_sql_missing_required"), + ).resolves.toBe(undefined); await expect(sqlStore.getRun(badRunId)).resolves.toBe(undefined); } finally { await stateAdapter.disconnect(); diff --git a/packages/junior/tests/fixtures/postgres/executor.ts b/packages/junior/tests/fixtures/postgres/executor.ts index 3757405c3..420d0f3f4 100644 --- a/packages/junior/tests/fixtures/postgres/executor.ts +++ b/packages/junior/tests/fixtures/postgres/executor.ts @@ -60,10 +60,10 @@ class ClientJuniorSqlExecutor implements JuniorSqlExecutor { } async withMigrationLock( - _migrationTable: string, + migrationTable: string, callback: () => Promise, ): Promise { - return await callback(); + return await this.withLock(`junior:migrate:${migrationTable}`, callback); } async close(): Promise { diff --git a/packages/junior/tests/fixtures/sql.ts b/packages/junior/tests/fixtures/sql.ts index 6c0855477..b7bd6a235 100644 --- a/packages/junior/tests/fixtures/sql.ts +++ b/packages/junior/tests/fixtures/sql.ts @@ -46,7 +46,8 @@ export async function createLocalJuniorSqlFixture(): Promise(statement, params), transaction: (callback) => fixture.transaction(callback), withLock: (lockName, callback) => fixture.withLock(lockName, callback), - withMigrationLock: (_migrationTable, callback) => callback(), + withMigrationLock: (migrationTable, callback) => + fixture.withLock(`junior:migrate:${migrationTable}`, callback), }; return { From d5538a28f146f2db51842cc032521c1357c83ff4 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 16 Jul 2026 14:48:59 -0700 Subject: [PATCH 07/15] ref(migrations): Make bootstrap and plugin state explicit Require explicit schema-bootstrap execution, scope plugin migration state by default while preserving historical plugin keys, and document permanent helper-version compatibility. Add real PostgreSQL progress, resume, and lock serialization contracts plus host-wiring isolation coverage. Co-Authored-By: GPT-5.6 Codex --- .../junior-evals/postgres-global-setup.ts | 6 +- packages/junior-migrations/README.md | 9 + packages/junior-migrations/src/runner.ts | 8 +- .../junior-migrations/tests/runner.test.ts | 33 +- .../src/chat/conversations/sql/migrations.ts | 11 +- .../src/chat/plugins/migration-state.ts | 49 +++ .../junior/src/chat/plugins/migrations.ts | 23 +- packages/junior/src/chat/plugins/state.ts | 6 +- .../junior/src/migration-helpers/README.md | 7 +- .../tests/component/cli/upgrade-cli.test.ts | 14 +- .../component/conversation-search.test.ts | 2 +- .../component/conversations/retention.test.ts | 2 +- .../component/memory-plugin-storage.test.ts | 8 +- .../migrations/mixed-runner-database.test.ts | 307 ++++++++++++++++++ .../component/scheduler-sql-plugin.test.ts | 155 ++++++++- .../tests/fixtures/postgres/global-setup.ts | 6 +- .../api/conversations/list.test.ts | 2 +- .../api/conversations/stats.test.ts | 4 +- .../tests/integration/api/locations.test.ts | 2 +- .../tests/integration/api/people/fixture.ts | 2 +- .../integration/api/people/profile.test.ts | 2 +- .../slack-conversation-search.test.ts | 2 +- .../integration/slack-schedule-tools.test.ts | 4 +- 23 files changed, 602 insertions(+), 62 deletions(-) create mode 100644 packages/junior/src/chat/plugins/migration-state.ts create mode 100644 packages/junior/tests/component/migrations/mixed-runner-database.test.ts diff --git a/packages/junior-evals/postgres-global-setup.ts b/packages/junior-evals/postgres-global-setup.ts index 743c3d1a7..0c29d35c9 100644 --- a/packages/junior-evals/postgres-global-setup.ts +++ b/packages/junior-evals/postgres-global-setup.ts @@ -6,7 +6,7 @@ import { type PostgresHarnessConfig, } from "@sentry/junior-testing/postgres"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; const workspaceRoot = path.resolve( @@ -43,8 +43,8 @@ export default async function setup( migrateTemplate: async (connectionString) => { const executor = createPostgresJuniorSqlExecutor({ connectionString }); try { - await migrateSchema(executor); - await migratePluginSchemas(executor, [ + await migrateSchema(executor, { mode: "schema-bootstrap" }); + await bootstrapPluginSchemas(executor, [ { dir: path.resolve( workspaceRoot, diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index abf230771..5b652b25d 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -36,9 +36,18 @@ provide the host database adapter, state adapter, and TypeScript loader. The same database adapter drives the journal ledger and is exposed as `context.database`, so migration files never own connection or driver setup. +Normal upgrades run with `mode: "all"` and execute every SQL and TypeScript +entry in journal order. `mode: "schema-bootstrap"` is reserved for constructing +an empty database at the latest schema in tests or bootstrap tooling. It skips +TypeScript data migrations while executing SQL entries across the full journal, +so it must not be used to upgrade an existing installation. + The runner rejects runtime imports of application source, relative modules, and unversioned `@sentry/junior` modules. Migrations may import the append-only `@sentry/junior/migration-helpers/v1` surface for parsing primitives, adapters, stores, and key resolution. One-off migration decisions and data transforms must still remain in the journal entry. Add a new helper or capability version rather than changing an existing contract. + +This source validation is an authoring guard for trusted packaged migration +code, not a security sandbox for untrusted scripts. diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index 7eb99035a..dd9ae0a83 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -77,16 +77,16 @@ interface RunAllMigrationJournalOptions extends RunMigrationJournalBaseOptions { mode?: "all"; } -interface RunSqlMigrationJournalOptions extends RunMigrationJournalBaseOptions { +interface RunSchemaBootstrapMigrationJournalOptions extends RunMigrationJournalBaseOptions { createContext?: never; loadTypeScript?: never; - mode: "sql"; + mode: "schema-bootstrap"; } /** Host capabilities and execution mode for one mixed migration journal. */ export type RunMigrationJournalOptions = | RunAllMigrationJournalOptions - | RunSqlMigrationJournalOptions; + | RunSchemaBootstrapMigrationJournalOptions; function identifier(value: string): string { if (!/^[a-z_][a-z0-9_]*$/.test(value)) { @@ -345,7 +345,7 @@ export async function runMigrationJournal( result.existing += 1; continue; } - if (mode === "sql" && migration.kind === "typescript") { + if (mode === "schema-bootstrap" && migration.kind === "typescript") { result.skipped += 1; continue; } diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index 6779e4127..7c1baa0cc 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -197,7 +197,7 @@ describe("runMigrationJournal", () => { ).resolves.toEqual({ existing: 3, migrated: 0, scanned: 3, skipped: 0 }); }); - it("allows schema-only execution to leave a TypeScript gap pending", async () => { + it("bootstraps the latest schema while leaving TypeScript entries pending", async () => { const folder = await mixedFolder(); const executor = new FakeExecutor(); @@ -206,10 +206,39 @@ describe("runMigrationJournal", () => { executor, migrationsFolder: folder, migrationsTable: "__drizzle_test", - mode: "sql", + mode: "schema-bootstrap", }), ).resolves.toEqual({ existing: 0, migrated: 2, scanned: 3, skipped: 1 }); expect([...executor.rows.keys()].sort()).toEqual([2_000, 2_002]); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); + + const migration: MigrationV1 = { + apiVersion: 1, + async up() { + return { backfilled: true }; + }, + }; + await expect( + runMigrationJournal({ + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ progress }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }), + ).resolves.toEqual({ existing: 2, migrated: 1, scanned: 3, skipped: 0 }); + expect(executor.statements).toEqual([ + "SELECT 'schema-zero';", + "SELECT 'schema-two';", + ]); }); it("resumes a failed TypeScript migration from saved progress", async () => { diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index 4b50ed911..a99e2704c 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( } export type MigrateSchemaOptions = - | { mode?: "sql" } + | { mode: "schema-bootstrap" } | { loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; @@ -196,10 +196,10 @@ export type MigrateSchemaOptions = export { schema }; -/** Apply the packaged mixed migration journal, or SQL entries alone in tests. */ +/** Apply all migrations, or bootstrap an empty test database to the latest schema. */ export async function migrateSchema( executor: JuniorSqlMigrationExecutor, - options: MigrateSchemaOptions = { mode: "sql" }, + options: MigrateSchemaOptions, ): Promise { const migrationsFolder = migrationFolder(); const runAll = options.mode === "all"; @@ -212,7 +212,10 @@ export async function migrateSchema( migrationsTable: MIGRATIONS_TABLE, }; if (!runAll) { - return await runMigrationJournal({ ...baseOptions, mode: "sql" }); + return await runMigrationJournal({ + ...baseOptions, + mode: "schema-bootstrap", + }); } return await runMigrationJournal({ ...baseOptions, diff --git a/packages/junior/src/chat/plugins/migration-state.ts b/packages/junior/src/chat/plugins/migration-state.ts new file mode 100644 index 000000000..cea06e997 --- /dev/null +++ b/packages/junior/src/chat/plugins/migration-state.ts @@ -0,0 +1,49 @@ +import type { MigrationStateV1 } from "@sentry/junior-migrations"; +import type { StateAdapter } from "chat"; +import { + createPluginState, + pluginStateKey, + validatePluginStateKey, +} from "@/chat/plugins/state"; + +/** Project host state onto the permanent v1 migration API for one plugin. */ +export function createPluginMigrationStateV1( + plugin: string, + stateAdapter: StateAdapter, +): MigrationStateV1 { + const pluginState = createPluginState(plugin, stateAdapter); + const stateKey = (key: string): string => { + validatePluginStateKey(key); + return pluginStateKey(plugin, key); + }; + + return { + acquireLock: async (key, ttlMs) => { + await stateAdapter.connect(); + return await stateAdapter.acquireLock(stateKey(key), ttlMs); + }, + appendToList: async (key, value, options) => { + await stateAdapter.connect(); + await stateAdapter.appendToList(stateKey(key), value, options); + }, + connect: async () => { + await stateAdapter.connect(); + }, + delete: async (key) => { + await pluginState.delete(key); + }, + get: async (key: string) => await pluginState.get(key), + getList: async (key: string) => { + await stateAdapter.connect(); + return await stateAdapter.getList(stateKey(key)); + }, + releaseLock: async (lock) => { + await stateAdapter.releaseLock(lock); + }, + set: async (key, value, ttlMs) => { + await pluginState.set(key, value, ttlMs); + }, + setIfNotExists: async (key, value, ttlMs) => + await pluginState.setIfNotExists(key, value, ttlMs), + }; +} diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index 59fc2e406..e3c620489 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -7,7 +7,7 @@ import { type TypeScriptMigrationLoader, } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; -import { createMigrationStateV1 } from "@/chat/migrations/state-v1"; +import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; interface PluginMigrationRoot { @@ -24,7 +24,7 @@ type PluginMigrationResult = { }; type PluginMigrationOptions = - | { mode?: "sql" } + | { mode: "schema-bootstrap" } | { loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; @@ -135,7 +135,7 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, roots: readonly PluginMigrationRoot[], - options: PluginMigrationOptions = { mode: "sql" }, + options: PluginMigrationOptions, ): Promise { const result: PluginMigrationResult = { existing: 0, @@ -169,12 +169,15 @@ export async function migratePluginSchemas( database: executor, log: options.log ?? (() => {}), progress, - state: createMigrationStateV1(options.stateAdapter), + state: createPluginMigrationStateV1( + root.pluginName, + options.stateAdapter, + ), }), loadTypeScript: options.loadTypeScript, mode: "all", }) - : await runMigrationJournal({ ...baseOptions, mode: "sql" }); + : await runMigrationJournal({ ...baseOptions, mode: "schema-bootstrap" }); result.scanned += pluginResult.scanned; result.existing += pluginResult.existing; result.migrated += pluginResult.migrated; @@ -184,3 +187,13 @@ export async function migratePluginSchemas( } return result; } + +/** Construct enabled plugins' latest SQL schemas without running data entries. */ +export async function bootstrapPluginSchemas( + executor: JuniorSqlMigrationExecutor, + roots: readonly PluginMigrationRoot[], +): Promise { + return await migratePluginSchemas(executor, roots, { + mode: "schema-bootstrap", + }); +} diff --git a/packages/junior/src/chat/plugins/state.ts b/packages/junior/src/chat/plugins/state.ts index 4812176f3..ccd090b2f 100644 --- a/packages/junior/src/chat/plugins/state.ts +++ b/packages/junior/src/chat/plugins/state.ts @@ -9,7 +9,8 @@ function hashKeyPart(value: string): string { return createHash("sha256").update(value).digest("hex").slice(0, 32); } -function pluginStateKey(plugin: string, key: string): string { +/** Resolve one logical plugin key to its isolated host-state key. */ +export function pluginStateKey(plugin: string, key: string): string { const pluginPrefix = `junior:${plugin}`; if (key === pluginPrefix || key.startsWith(`${pluginPrefix}:`)) { return key; @@ -17,7 +18,8 @@ function pluginStateKey(plugin: string, key: string): string { return `junior:plugin_state:${hashKeyPart(plugin)}:${hashKeyPart(key)}`; } -function validatePluginStateKey(key: string): void { +/** Validate one logical plugin state key before namespacing it. */ +export function validatePluginStateKey(key: string): void { if (!key.trim()) { throw new Error("Plugin state key is required"); } diff --git a/packages/junior/src/migration-helpers/README.md b/packages/junior/src/migration-helpers/README.md index 9f2f3637f..f2550f515 100644 --- a/packages/junior/src/migration-helpers/README.md +++ b/packages/junior/src/migration-helpers/README.md @@ -8,6 +8,7 @@ and key resolution. They must not contain one-off migration decisions or data transforms. Logic such as mapping one retired record shape into another belongs only in the journal entry that performs that migration. -Never change the behavior or signature of an existing version in a way that can -break a pending migration. Add a new versioned entrypoint when the helper -contract must change. +The behavior and signature of an existing version are permanent compatibility +contracts. Add a new versioned entrypoint when that contract must change. +Migration files may be updated while still unreleased, but shipped migration +sources are hash-verified ledger entries and must remain immutable. diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index fdb0c42c0..657a49cef 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -398,7 +398,7 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const context = { io: { info: () => {} }, stateAdapter, @@ -455,7 +455,7 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); for (let index = 0; index < 3; index++) { const conversationId = `slack:C123:page-${index}`; await appendInboundMessage({ @@ -496,7 +496,7 @@ export const plugins = { const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const seedMs = Date.now() - 1_000; await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, @@ -691,7 +691,7 @@ WHERE conversation_id = $1 } as PiMessage; try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -869,7 +869,7 @@ WHERE conversation_id = $1 const conversationIds = ["local:usage-batch-a", "local:usage-batch-b"]; try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); for (const [index, conversationId] of conversationIds.entries()) { await sqlStore.recordExecution({ conversationId, @@ -1004,7 +1004,7 @@ WHERE conversation_id = $1 const eventStore = createSqlConversationEventStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await sqlStore.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -1300,7 +1300,7 @@ WHERE conversation_id = $1 const sqlStore = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const context = { io: { info: () => {} }, stateAdapter, diff --git a/packages/junior/tests/component/conversation-search.test.ts b/packages/junior/tests/component/conversation-search.test.ts index 18c04687b..5315a76aa 100644 --- a/packages/junior/tests/component/conversation-search.test.ts +++ b/packages/junior/tests/component/conversation-search.test.ts @@ -12,7 +12,7 @@ describe("conversation search", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const conversations = createSqlStore(fixture.sql); const events = createSqlConversationEventStore(fixture.sql); const search = createSqlConversationSearchStore(fixture.sql); diff --git a/packages/junior/tests/component/conversations/retention.test.ts b/packages/junior/tests/component/conversations/retention.test.ts index 9802dbeb2..592b7884b 100644 --- a/packages/junior/tests/component/conversations/retention.test.ts +++ b/packages/junior/tests/component/conversations/retention.test.ts @@ -132,7 +132,7 @@ describe("retention purge job", () => { beforeEach(async () => { fixture = await createLocalJuniorSqlFixture(); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); }); afterEach(async () => { diff --git a/packages/junior/tests/component/memory-plugin-storage.test.ts b/packages/junior/tests/component/memory-plugin-storage.test.ts index d9600d282..02d219d6d 100644 --- a/packages/junior/tests/component/memory-plugin-storage.test.ts +++ b/packages/junior/tests/component/memory-plugin-storage.test.ts @@ -14,7 +14,7 @@ import { } from "@sentry/junior-plugin-api"; import { defineJuniorPlugins } from "@/plugins"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import { closeDb } from "@/chat/db"; import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; @@ -86,7 +86,7 @@ function memoryMigrationFiles(): string[] { async function migrateMemorySchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -128,7 +128,7 @@ CREATE TABLE junior_schema_migrations ( } await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -165,7 +165,7 @@ CREATE TABLE junior_schema_migrations ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", diff --git a/packages/junior/tests/component/migrations/mixed-runner-database.test.ts b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts new file mode 100644 index 000000000..a972b989d --- /dev/null +++ b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts @@ -0,0 +1,307 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + runMigrationJournal, + type MigrationContextV1, + type MigrationStateV1, +} from "@sentry/junior-migrations"; +import { createJiti } from "jiti"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createLocalJuniorSqlFixture, + type LocalJuniorSqlFixture, +} from "../../fixtures/sql"; +import { hasJuniorPostgresTestDatabase } from "../../fixtures/postgres/fixture"; + +const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); +const temporaryDirectories: string[] = []; + +const unusedState: MigrationStateV1 = { + acquireLock: async () => null, + appendToList: async () => {}, + connect: async () => {}, + delete: async () => {}, + get: async () => undefined, + getList: async () => [], + releaseLock: async () => {}, + set: async () => {}, + setIfNotExists: async () => false, +}; + +async function createMigrationFolder(args: { + source: string; + tag: string; + when: number; +}): Promise { + const folder = await mkdtemp(join(tmpdir(), "junior-mixed-runner-")); + temporaryDirectories.push(folder); + await mkdir(join(folder, "meta")); + await writeFile( + join(folder, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: [ + { + idx: 0, + version: "7", + when: args.when, + tag: args.tag, + breakpoints: true, + }, + ], + }), + ); + await writeFile(join(folder, `${args.tag}.ts`), args.source); + return folder; +} + +function migrationOptions(args: { + executor?: LocalJuniorSqlFixture["sql"]; + fixture: LocalJuniorSqlFixture; + folder: string; + table: string; +}) { + const executor = args.executor ?? args.fixture.sql; + return { + executor, + migrationsFolder: args.folder, + migrationsTable: args.table, + loadTypeScript: async (migrationPath: string) => + await migrationLoader.import>(migrationPath), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }): MigrationContextV1 => ({ + database: executor, + log: () => {}, + progress, + state: unusedState, + }), + }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("mixed migration runner database contract", () => { + it("persists failed progress and resumes a TypeScript migration", async () => { + const fixture = await createLocalJuniorSqlFixture(); + const folder = await createMigrationFolder({ + source: ` +export default { + apiVersion: 1, + async up(context) { + const progress = await context.progress.load(); + if (progress === undefined) { + await context.database.execute( + "INSERT INTO mixed_runner_events (stage) VALUES ($1)", + ["before-failure"], + ); + await context.progress.save({ cursor: 1 }); + throw new Error("intentional migration interruption"); + } + await context.database.execute( + "INSERT INTO mixed_runner_events (stage) VALUES ($1)", + ["after-resume"], + ); + return { resumedFrom: progress.cursor }; + }, +}; +`, + tag: "0000_resumable", + when: 2_026_071_600_001, + }); + const table = "__junior_mixed_runner_resume"; + const options = migrationOptions({ fixture, folder, table }); + + try { + await fixture.sql.execute(` +CREATE TABLE mixed_runner_events ( + id SERIAL PRIMARY KEY, + stage TEXT NOT NULL +) +`); + + await expect(runMigrationJournal(options)).rejects.toThrow( + "intentional migration interruption", + ); + + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_events ORDER BY id ASC`, + ), + ).resolves.toEqual([{ stage: "before-failure" }]); + await expect( + fixture.sql.query(` +SELECT + name, + kind, + status, + progress, + result, + completed_at IS NOT NULL AS "completed" +FROM drizzle.${table} +`), + ).resolves.toEqual([ + { + completed: false, + kind: "typescript", + name: "0000_resumable", + progress: { cursor: 1 }, + result: null, + status: "failed", + }, + ]); + + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 0, + migrated: 1, + scanned: 1, + skipped: 0, + }); + + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_events ORDER BY id ASC`, + ), + ).resolves.toEqual([ + { stage: "before-failure" }, + { stage: "after-resume" }, + ]); + await expect( + fixture.sql.query(` +SELECT + status, + progress, + result, + completed_at IS NOT NULL AS "completed" +FROM drizzle.${table} +`), + ).resolves.toEqual([ + { + completed: true, + progress: { cursor: 1 }, + result: { resumedFrom: 1 }, + status: "completed", + }, + ]); + } finally { + await fixture.close(); + } + }); + + it.skipIf(!hasJuniorPostgresTestDatabase())( + "serializes concurrent runs and executes a TypeScript migration once", + async () => { + const fixture = await createLocalJuniorSqlFixture(); + const folder = await createMigrationFolder({ + source: ` +export default { + apiVersion: 1, + async up(context) { + await context.database.execute( + "INSERT INTO mixed_runner_lock_events (stage) VALUES ($1)", + ["body"], + ); + while (true) { + const [barrier] = await context.database.query( + "SELECT released FROM mixed_runner_lock_barrier WHERE id = 1", + ); + if (barrier?.released === true) break; + await context.database.query("SELECT pg_sleep(0.01)"); + } + return { executed: true }; + }, +}; +`, + tag: "0000_locked", + when: 2_026_071_600_002, + }); + const table = "__junior_mixed_runner_lock"; + let lockAttempts = 0; + const observedExecutor = new Proxy(fixture.sql, { + get(target, key, receiver) { + if (key === "withMigrationLock") { + return async ( + migrationTable: string, + callback: () => Promise, + ) => { + lockAttempts += 1; + return await target.withMigrationLock(migrationTable, callback); + }; + } + const value = Reflect.get(target, key, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const options = migrationOptions({ + executor: observedExecutor, + fixture, + folder, + table, + }); + + try { + await fixture.sql.execute(` +CREATE TABLE mixed_runner_lock_events ( + id SERIAL PRIMARY KEY, + stage TEXT NOT NULL +); +CREATE TABLE mixed_runner_lock_barrier ( + id INTEGER PRIMARY KEY, + released BOOLEAN NOT NULL +); +INSERT INTO mixed_runner_lock_barrier (id, released) VALUES (1, false) +`); + + const first = runMigrationJournal(options); + await vi.waitFor(async () => { + await expect( + fixture.sql.query( + "SELECT count(*)::integer AS count FROM mixed_runner_lock_events", + ), + ).resolves.toEqual([{ count: 1 }]); + }); + const second = runMigrationJournal(options); + await vi.waitFor(() => expect(lockAttempts).toBe(2)); + await expect( + fixture.sql.query( + "SELECT count(*)::integer AS count FROM mixed_runner_lock_events", + ), + ).resolves.toEqual([{ count: 1 }]); + await fixture.sql.execute( + "UPDATE mixed_runner_lock_barrier SET released = true WHERE id = 1", + ); + const results = await Promise.all([first, second]); + + expect(results).toEqual( + expect.arrayContaining([ + { existing: 0, migrated: 1, scanned: 1, skipped: 0 }, + { existing: 1, migrated: 0, scanned: 1, skipped: 0 }, + ]), + ); + await expect( + fixture.sql.query( + `SELECT stage FROM mixed_runner_lock_events ORDER BY id ASC`, + ), + ).resolves.toEqual([{ stage: "body" }]); + await expect( + fixture.sql.query(`SELECT status, result FROM drizzle.${table}`), + ).resolves.toEqual([ + { result: { executed: true }, status: "completed" }, + ]); + } finally { + await fixture.close(); + } + }, + 15_000, + ); +}); diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index e2023ba8d..14ef04386 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -17,7 +17,11 @@ import { import schedulerStateToSqlMigration from "../../../junior-scheduler/migrations/0002_scheduler_state_to_sql"; import { createSchedulerStore } from "../../../junior-scheduler/src/store"; import { defineJuniorPlugins } from "@/plugins"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; +import { + bootstrapPluginSchemas, + migratePluginSchemas, +} from "@/chat/plugins/migrations"; import { createPluginState } from "@/chat/plugins/state"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; @@ -68,7 +72,7 @@ function memoryMigrationsDir(): string { async function migrateSchedulerSchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -87,7 +91,7 @@ async function runSchedulerStateMigration(args: { load: async () => undefined, save: async () => {}, }, - state: args.stateAdapter as unknown as MigrationContextV1["state"], + state: createPluginMigrationStateV1("scheduler", args.stateAdapter), }); } @@ -131,6 +135,127 @@ describe("scheduler SQL plugin storage", () => { await disconnectStateAdapter(); }); + it("scopes migration state while preserving plugin legacy keys", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const state = createPluginMigrationStateV1("scheduler", stateAdapter); + + try { + await stateAdapter.set("global-key", "global-value"); + await stateAdapter.set("junior:memory:secret", "memory-value"); + await state.set("global-key", "scheduler-value"); + await state.set("junior:scheduler:legacy", "legacy-value"); + + await expect(state.get("global-key")).resolves.toBe("scheduler-value"); + await expect(stateAdapter.get("global-key")).resolves.toBe( + "global-value", + ); + await expect(state.get("junior:memory:secret")).resolves.toBeUndefined(); + await expect(state.get("junior:scheduler:legacy")).resolves.toBe( + "legacy-value", + ); + await expect(stateAdapter.get("junior:scheduler:legacy")).resolves.toBe( + "legacy-value", + ); + + await stateAdapter.appendToList("global-list", "global-item"); + await state.appendToList("global-list", "scheduler-item"); + await expect(state.getList("global-list")).resolves.toEqual([ + "scheduler-item", + ]); + await expect(stateAdapter.getList("global-list")).resolves.toEqual([ + "global-item", + ]); + + const scopedLock = await state.acquireLock("migration-lock", 1_000); + const globalLock = await stateAdapter.acquireLock( + "migration-lock", + 1_000, + ); + expect(scopedLock).not.toBeNull(); + expect(globalLock).not.toBeNull(); + if (scopedLock) { + await state.releaseLock(scopedLock); + } + if (globalLock) { + await stateAdapter.releaseLock(globalLock); + } + } finally { + await stateAdapter.disconnect(); + } + }); + + it("passes scoped state through the plugin migration host", async () => { + const stateAdapter = createMemoryState(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const migrationsDir = mkdtempSync( + path.join(tmpdir(), "junior-plugin-state-migration-"), + ); + mkdirSync(path.join(migrationsDir, "meta")); + writeFileSync( + path.join(migrationsDir, "meta", "_journal.json"), + JSON.stringify({ + version: "7", + dialect: "postgresql", + entries: [ + { + idx: 0, + version: "7", + when: 2_026_071_600_003, + tag: "0000_state_scope", + breakpoints: true, + }, + ], + }), + ); + writeFileSync( + path.join(migrationsDir, "0000_state_scope.ts"), + "export default { apiVersion: 1, async up() {} };\n", + ); + const observed: unknown[] = []; + + try { + await stateAdapter.set("global-key", "global-value"); + await stateAdapter.set("junior:other:secret", "other-value"); + await migratePluginSchemas( + fixture.sql, + [{ dir: migrationsDir, pluginName: "isolation" }], + { + mode: "all", + stateAdapter, + loadTypeScript: async () => ({ + default: { + apiVersion: 1, + async up(context: MigrationContextV1) { + observed.push( + await context.state.get("global-key"), + await context.state.get("junior:other:secret"), + ); + await context.state.set("global-key", "scoped-value"); + return { scoped: true }; + }, + }, + }), + }, + ); + + expect(observed).toEqual([undefined, undefined]); + await expect(stateAdapter.get("global-key")).resolves.toBe( + "global-value", + ); + await expect( + createPluginMigrationStateV1("isolation", stateAdapter).get( + "global-key", + ), + ).resolves.toBe("scoped-value"); + } finally { + rmSync(migrationsDir, { force: true, recursive: true }); + await stateAdapter.disconnect(); + await fixture.close(); + } + }); + it("adopts deployed scheduler schema state into its Drizzle journal", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -191,7 +316,7 @@ INSERT INTO junior_scheduler_tasks ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -229,7 +354,7 @@ ORDER BY created_at }); expect(migratedTask?.record).not.toHaveProperty("credentialSubject"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -272,12 +397,14 @@ ORDER BY created_at (migration) => migration.kind === "sql", ).length; try { - await expect(migratePluginSchemas(fixture.sql, roots)).resolves.toEqual({ - existing: 0, - migrated: sqlMigrationCount, - scanned: migrationCount, - skipped: 1, - }); + await expect(bootstrapPluginSchemas(fixture.sql, roots)).resolves.toEqual( + { + existing: 0, + migrated: sqlMigrationCount, + scanned: migrationCount, + skipped: 1, + }, + ); const migrationTables = await fixture.sql.query<{ tablename: string }>(` SELECT tablename FROM pg_tables @@ -287,7 +414,7 @@ ORDER BY tablename `); expect(migrationTables).toHaveLength(2); await expect( - migratePluginSchemas(fixture.sql, [...roots].reverse()), + bootstrapPluginSchemas(fixture.sql, [...roots].reverse()), ).resolves.toEqual({ existing: sqlMigrationCount, migrated: 0, @@ -312,12 +439,12 @@ ORDER BY tablename try { await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: missingJournal, pluginName: "missing" }, ]), ).rejects.toThrow("Can't find meta/_journal.json file"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: invalidJournal, pluginName: "invalid" }, ]), ).rejects.toThrow("Expected property name"); diff --git a/packages/junior/tests/fixtures/postgres/global-setup.ts b/packages/junior/tests/fixtures/postgres/global-setup.ts index 8fa39271b..59a007636 100644 --- a/packages/junior/tests/fixtures/postgres/global-setup.ts +++ b/packages/junior/tests/fixtures/postgres/global-setup.ts @@ -6,7 +6,7 @@ import { type PostgresHarnessConfig, } from "@sentry/junior-testing/postgres"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; @@ -47,8 +47,8 @@ export async function setupJuniorPostgresHarness( migrateTemplate: async (connectionString) => { const executor = createPostgresJuniorSqlExecutor({ connectionString }); try { - await migrateSchema(executor); - await migratePluginSchemas(executor, [ + await migrateSchema(executor, { mode: "schema-bootstrap" }); + await bootstrapPluginSchemas(executor, [ { dir: path.resolve(process.cwd(), "../junior-scheduler/migrations"), pluginName: "scheduler", diff --git a/packages/junior/tests/integration/api/conversations/list.test.ts b/packages/junior/tests/integration/api/conversations/list.test.ts index 290a9c5a1..26d45a6ca 100644 --- a/packages/junior/tests/integration/api/conversations/list.test.ts +++ b/packages/junior/tests/integration/api/conversations/list.test.ts @@ -126,7 +126,7 @@ describe("conversation list API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ actor: { email: "other@example.com", diff --git a/packages/junior/tests/integration/api/conversations/stats.test.ts b/packages/junior/tests/integration/api/conversations/stats.test.ts index 5c9c9a83c..24cbb1a05 100644 --- a/packages/junior/tests/integration/api/conversations/stats.test.ts +++ b/packages/junior/tests/integration/api/conversations/stats.test.ts @@ -16,7 +16,7 @@ describe("conversation stats API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:recent", channelName: "proj-alpha", @@ -203,7 +203,7 @@ describe("conversation stats API", () => { const fixture = createConfiguredJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const now = new Date("2026-06-15T11:00:00.000Z"); await fixture.sql .db() diff --git a/packages/junior/tests/integration/api/locations.test.ts b/packages/junior/tests/integration/api/locations.test.ts index db7bd6425..bae7ecb26 100644 --- a/packages/junior/tests/integration/api/locations.test.ts +++ b/packages/junior/tests/integration/api/locations.test.ts @@ -19,7 +19,7 @@ describe("locations API", () => { const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const nowMs = Date.parse("2026-06-15T11:00:00.000Z"); await store.recordActivity({ conversationId: "slack:C1:seed", diff --git a/packages/junior/tests/integration/api/people/fixture.ts b/packages/junior/tests/integration/api/people/fixture.ts index c35dc0ee1..99356d1ad 100644 --- a/packages/junior/tests/integration/api/people/fixture.ts +++ b/packages/junior/tests/integration/api/people/fixture.ts @@ -6,7 +6,7 @@ import type { LocalJuniorSqlFixture } from "../../../fixtures/sql"; /** Seed representative verified and untrusted people rows for people API tests. */ export async function seedPeople(fixture: LocalJuniorSqlFixture) { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:123", diff --git a/packages/junior/tests/integration/api/people/profile.test.ts b/packages/junior/tests/integration/api/people/profile.test.ts index a8138560b..5afad1fa6 100644 --- a/packages/junior/tests/integration/api/people/profile.test.ts +++ b/packages/junior/tests/integration/api/people/profile.test.ts @@ -329,7 +329,7 @@ describe("people profile API", () => { const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const nowMs = Date.parse("2026-06-15T11:00:00.000Z"); await store.recordActivity({ conversationId: "slack:C1:seed", diff --git a/packages/junior/tests/integration/slack-conversation-search.test.ts b/packages/junior/tests/integration/slack-conversation-search.test.ts index a2e4aa432..7200e9e16 100644 --- a/packages/junior/tests/integration/slack-conversation-search.test.ts +++ b/packages/junior/tests/integration/slack-conversation-search.test.ts @@ -25,7 +25,7 @@ describe("searchConversationHistory", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const conversations = createSqlStore(fixture.sql); const events = createSqlConversationEventStore(fixture.sql); const search = createSqlConversationSearchStore(fixture.sql); diff --git a/packages/junior/tests/integration/slack-schedule-tools.test.ts b/packages/junior/tests/integration/slack-schedule-tools.test.ts index 39cdbbcfb..30b39685e 100644 --- a/packages/junior/tests/integration/slack-schedule-tools.test.ts +++ b/packages/junior/tests/integration/slack-schedule-tools.test.ts @@ -18,7 +18,7 @@ import { type SchedulerDb, type SchedulerToolContext, } from "@sentry/junior-scheduler"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import * as dbModule from "@/chat/db"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; import { disconnectStateAdapter } from "@/chat/state/adapter"; @@ -43,7 +43,7 @@ function schedulerMigrationsDir(): string { async function useSchedulerSqlPlugin() { const fixture = await createLocalJuniorSqlFixture(); - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", From e9341ea2324b3b762336a2eee703b9383ab7a15d Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 16 Jul 2026 15:42:23 -0700 Subject: [PATCH 08/15] fix(migrations): Preserve host state adapter identity Pass the real core state adapter into TypeScript migrations so Redis-native index and lock detection continues to work. Add multiline import guard coverage and remove the stale generated temp-path comment from the history migration. Co-Authored-By: GPT-5.6 Codex --- packages/junior-migrations/src/journal.ts | 4 +-- .../junior-migrations/tests/journal.test.ts | 12 +++++++ .../src/chat/conversations/sql/migrations.ts | 3 +- .../junior/src/chat/migrations/state-v1.ts | 32 ------------------- .../tests/component/cli/upgrade-cli.test.ts | 30 +++++++++++++++++ 5 files changed, 45 insertions(+), 36 deletions(-) delete mode 100644 packages/junior/src/chat/migrations/state-v1.ts diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index 56ab5c969..ca1b92847 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -86,7 +86,7 @@ function validateTypeScriptSource(tag: string, source: string): void { } }; const imports = source.matchAll( - /^\s*import\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, + /^\s*import\s+([\s\S]*?)\s+from\s+["']([^"']+)["'];?/gm, ); for (const match of imports) { const clause = match[1]?.trim(); @@ -97,7 +97,7 @@ function validateTypeScriptSource(tag: string, source: string): void { validateRuntimeSpecifier(specifier); } const exports = source.matchAll( - /^\s*export\s+([^;]+?)\s+from\s+["']([^"']+)["'];?/gm, + /^\s*export\s+([\s\S]*?)\s+from\s+["']([^"']+)["'];?/gm, ); for (const match of exports) { const clause = match[1]?.trim(); diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index a25336a8b..38a2c9875 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -62,6 +62,18 @@ describe("resolveMigrations", () => { ); }); + it("rejects multiline imports of Junior runtime modules", async () => { + const folder = await migrationFolder(["0000_multiline"]); + await writeFile( + join(folder, "0000_multiline.ts"), + 'import {\n getChatConfig,\n getDb,\n} from "@sentry/junior/internal";\nexport default { apiVersion: 1, async up() {} };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot import application runtime code", + ); + }); + it("allows versioned Junior migration helpers", async () => { const folder = await migrationFolder(["0000_helpers"]); await writeFile( diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index a99e2704c..a10f2948a 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -10,7 +10,6 @@ import { } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; -import { createMigrationStateV1 } from "@/chat/migrations/state-v1"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { juniorSqlSchema as schema } from "@/db/schema"; @@ -233,7 +232,7 @@ export async function migrateSchema( }, } : {}), - state: createMigrationStateV1(options.stateAdapter), + state: options.stateAdapter, }), loadTypeScript: options.loadTypeScript, mode: "all", diff --git a/packages/junior/src/chat/migrations/state-v1.ts b/packages/junior/src/chat/migrations/state-v1.ts deleted file mode 100644 index dd93a9ab3..000000000 --- a/packages/junior/src/chat/migrations/state-v1.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { MigrationStateV1 } from "@sentry/junior-migrations"; -import type { StateAdapter } from "chat"; - -/** Project the host state adapter onto the permanent v1 migration capability. */ -export function createMigrationStateV1( - stateAdapter: StateAdapter, -): MigrationStateV1 { - return { - acquireLock: async (threadId, ttlMs) => - await stateAdapter.acquireLock(threadId, ttlMs), - appendToList: async (key, value, options) => { - await stateAdapter.appendToList(key, value, options); - }, - connect: async () => { - await stateAdapter.connect(); - }, - delete: async (key) => { - await stateAdapter.delete(key); - }, - get: async (key: string) => - (await stateAdapter.get(key)) ?? undefined, - getList: async (key: string) => await stateAdapter.getList(key), - releaseLock: async (lock) => { - await stateAdapter.releaseLock(lock); - }, - set: async (key, value, ttlMs) => { - await stateAdapter.set(key, value, ttlMs); - }, - setIfNotExists: async (key, value, ttlMs) => - await stateAdapter.setIfNotExists(key, value, ttlMs), - }; -} diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index 657a49cef..b0b5918ad 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -2,6 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import type { MigrationContextV1 } from "@sentry/junior-migrations"; import { createJiti } from "jiti"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getChatConfig } from "@/chat/config"; @@ -378,6 +379,35 @@ export const plugins = { } }, 15_000); + it("passes the host state adapter through without wrapping its identity", async () => { + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const fixture = await createLocalJuniorSqlFixture(); + const observedStates: unknown[] = []; + + try { + await migrateSchema(fixture.sql, { + loadTypeScript: async () => ({ + default: { + apiVersion: 1, + async up(context: MigrationContextV1) { + observedStates.push(context.state); + }, + }, + }), + mode: "all", + stateAdapter, + }); + + expect(observedStates).toHaveLength(5); + expect(observedStates.every((state) => state === stateAdapter)).toBe( + true, + ); + } finally { + await fixture.close(); + } + }); + it("migrates legacy conversation work before SQL conversation backfill", async () => { const stateAdapter = getStateAdapter(); await stateAdapter.connect(); From f0df3331e9fae4bddb21e883166590cbf3288fb0 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 22 Jul 2026 22:17:39 -0700 Subject: [PATCH 09/15] ref(migrations): Align journals with SQL cutover Drop the retired core and plugin data migration capsules after the bridge-release cutover. Keep mixed journal support for future entries while preserving the current core SQL ledger and upgrade guard. --- packages/docs/src/content/docs/cli/upgrade.md | 109 +- packages/junior-migrations/README.md | 7 +- packages/junior/migrations/README.md | 29 +- .../src/chat/conversations/sql/migrations.ts | 190 +-- .../junior/src/chat/plugins/migrations.ts | 2 +- packages/junior/src/cli/upgrade.ts | 9 +- .../cli/upgrade/migrations/plugin-journal.ts | 4 +- .../cli/upgrade/migrations/upgrade-plugins.ts | 8 +- .../junior/src/migration-helpers/README.md | 8 +- packages/junior/src/migration-helpers/v1.ts | 256 +--- .../tests/component/cli/upgrade-cli.test.ts | 1342 +---------------- .../component/memory-plugin-storage.test.ts | 86 +- .../component/scheduler-sql-plugin.test.ts | 547 +------ .../integration/conversation-sql.test.ts | 8 +- 14 files changed, 199 insertions(+), 2406 deletions(-) diff --git a/packages/docs/src/content/docs/cli/upgrade.md b/packages/docs/src/content/docs/cli/upgrade.md index 9ef322d35..29f537608 100644 --- a/packages/docs/src/content/docs/cli/upgrade.md +++ b/packages/docs/src/content/docs/cli/upgrade.md @@ -1,8 +1,8 @@ --- title: "junior upgrade" -description: "Apply Junior schema and data migrations." +description: "Apply Junior and plugin schema or data migrations." type: reference -summary: Move configured SQL schemas and persisted state forward after upgrades. +summary: Bring the configured Junior SQL database up to the installed schema. prerequisites: - /start-here/quickstart/ related: @@ -11,100 +11,65 @@ related: - /cli/snapshot-create/ --- -Use `junior upgrade` after installing a Junior release that includes schema or -data migrations. The command mutates the configured SQL database and state -stores, so run it from the same app environment that has the production state -and SQL environment variables configured for the deployment you are upgrading. +Use `junior upgrade` after installing a new Junior release. The command applies +pending SQL schema and TypeScript data entries from the ordered Drizzle journals +owned by Junior and its enabled plugins. ## Usage -Run it from a project that already has `@sentry/junior` installed: +Run the command from your Junior app: ```bash pnpm exec junior upgrade ``` -The command takes no extra arguments. +The command takes no arguments. It migrates Junior first, then enabled plugins. +Already-applied entries are recognized as up to date and are not rerun. -## What it does +## Upgrade bridge for older databases -`junior upgrade` runs migrations sequentially. Core and plugin migration -directories use Drizzle Kit's ordered journal and may contain either generated -SQL schema migrations or TypeScript data migrations targeting a versioned -host capability API. Current -upgrade work includes: +An existing Junior database without core Drizzle migration history must +complete the `0.107.1` upgrade before upgrading to a later release. Before +starting this bridge, block new ingress, drain active and resumable work, stop +all old workers and queue consumers, and keep them stopped until the later +release is ready. If Junior reports this unsupported database state: -- Apply core and enabled-plugin schema and data journal entries. -- Rewrite retained turn-session records from legacy storage shapes before the - new runtime reads them. -- Move legacy `junior:conversation-work:*` Redis state into the newer conversation record and index state used by the durable worker and dashboard feed. -- Backfill retained conversation records into the shared Junior SQL database. The upgrade requires `DATABASE_URL`. -- Apply the SQL schema cutover and rewrite legacy Pi-message rows into canonical conversation events. -- Repair legacy token and estimated-cost rollups from durable SQL conversation events in bounded batches. Conversations that are active during the repair are left unchanged and can be repaired by rerunning the command after they become idle. +1. Install `@sentry/junior@0.107.1`. +2. Run `pnpm exec junior upgrade` and confirm it completes successfully. +3. Restore the intended Junior version. +4. Run `pnpm exec junior upgrade` again. +5. Deploy the intended version, then restart workers and reopen ingress. -Completed journal entries are tracked individually, and TypeScript migrations -can checkpoint progress for safe retries. Legacy backfills remain idempotent: -rerunning them skips records that were already moved, removes stale legacy -index entries that no longer have a record, and upserts SQL conversation rows. -The conversation-history import paginates through every conversation in the -retained activity index; orphaned or expired Redis keys outside that index are -not treated as retained history. After cutover, SQL owns durable conversation -metadata and event history. - -## Hard-cutover upgrade sequence - -The canonical conversation-event cutover is not rolling-compatible. Do not run it inside a Vercel build while the previous deployment can still accept work. Use this operator sequence: - -1. Block new ingress and enqueueing while leaving the previous release's workers and continuation consumers running. -2. Let existing work drain, then verify that no turns remain running or awaiting resume. -3. Stop every old worker, queue consumer, and heartbeat. Keep the old deployment stopped for the rest of the procedure. -4. Run the upgrade from an operator environment with the production `REDIS_URL`, `JUNIOR_STATE_KEY_PREFIX`, and `DATABASE_URL`. -5. Confirm the history import and message-event seal complete with no missing rows. -6. Run `junior check`, deploy the new release, and only then reopen ingress and start the new workers. - -Run the upgrade as a separate operator command: - -```bash -pnpm exec junior upgrade -pnpm exec junior check -``` - -The checkpoint and message-event rewrites fail closed if resumable work remains. After the drain succeeds, each rewrite invalidates stale resume state before changing physical event positions. Checkpoint normalization closes deletion gaps, and the message migration resequences the streams it changes while preserving reporting summaries. - -If the command exits nonzero, leave the deployment stopped, correct the reported state, and rerun it. Do not restart workers after only part of the sequence completes. +Fresh databases without Junior tables do not require the bridge release. ## Example output -Typical logs look like this: +An already-current database reports its migrations as existing: ```text -Running Junior upgrade migrations... -Running migration core-migrations... -Finished migration core-migrations: scanned=10 migrated=6 existing=4 missing=0 skipped=0 -Running migration plugin-migrations... -Finished migration plugin-migrations: scanned=8 migrated=8 existing=0 missing=0 skipped=0 -Junior upgrade complete. +Checking database migrations... + junior: up to date (7 migrations) + junior-github: up to date (4 migrations) + junior-memory: up to date (5 migrations) + junior-scheduler: up to date (2 migrations) +Database is up to date (18 migrations). ``` ## Failure behavior -If the configured state store is unavailable or a legacy record is malformed, the CLI exits non-zero and prints the underlying error: - -```text -junior command failed: Legacy conversation work state is invalid for slack:C123:1712345.0001 -``` - -Treat that as a deploy blocker for the affected environment. Check `REDIS_URL`, `JUNIOR_STATE_KEY_PREFIX`, `DATABASE_URL`, and the reported legacy record before retrying. +The command exits nonzero when it cannot connect to SQL, encounters an +unsupported pre-Drizzle database, or a migration fails. Treat that as a deploy +blocker: correct the reported database or migration error, then rerun the +command. ## Verification -After running the command: - -1. Confirm the final log line includes `Junior upgrade complete`. -2. Confirm `backfill-conversation-events-sql` scanned the complete retained activity index and did not stop at one page. -3. Confirm `move-conversation-messages-to-events` reports `missing=0`. The runtime now uses the copied events. The legacy message table remains available so a later upgrade can recover messages written by old workers during deployment. -4. Run `pnpm exec junior check` before building or deploying the app. +Confirm that the command exits successfully and lists `junior` plus each +enabled plugin that owns migrations. The final line reports whether the +database was already current or how many migrations were applied. ## Next step -Run [junior check](/cli/check/) after the upgrade, then continue with [junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox dependencies. +Run [junior check](/cli/check/) before deploying, then continue with +[junior snapshot create](/cli/snapshot-create/) if your plugins need sandbox +dependencies. diff --git a/packages/junior-migrations/README.md b/packages/junior-migrations/README.md index 5b652b25d..1495c5c26 100644 --- a/packages/junior-migrations/README.md +++ b/packages/junior-migrations/README.md @@ -32,7 +32,8 @@ schema snapshot while preserving the mixed journal order. Drizzle Kit remains the supported authoring tool. Drizzle ORM's stock `migrate()` function is not a supported executor for mixed journals because it requires every entry to have a SQL file. Call `runMigrationJournal` instead and -provide the host database adapter, state adapter, and TypeScript loader. The +provide the host database adapter and, for TypeScript entries, a context and +loader. The same database adapter drives the journal ledger and is exposed as `context.database`, so migration files never own connection or driver setup. @@ -44,8 +45,8 @@ so it must not be used to upgrade an existing installation. The runner rejects runtime imports of application source, relative modules, and unversioned `@sentry/junior` modules. Migrations may import the append-only -`@sentry/junior/migration-helpers/v1` surface for parsing primitives, adapters, -stores, and key resolution. One-off migration decisions and data transforms +`@sentry/junior/migration-helpers/v1` surface for stable parsing primitives and +other reusable infrastructure. One-off migration decisions and data transforms must still remain in the journal entry. Add a new helper or capability version rather than changing an existing contract. diff --git a/packages/junior/migrations/README.md b/packages/junior/migrations/README.md index 097fd194f..c81d5efc5 100644 --- a/packages/junior/migrations/README.md +++ b/packages/junior/migrations/README.md @@ -1,4 +1,4 @@ -# SQL migrations +# Migrations `src/db/schema.ts` is the Drizzle schema entrypoint. This directory is the append-only history used to bring an existing database to that schema. It @@ -33,21 +33,20 @@ TypeScript migrations target the versioned migration capability API and must not import Junior runtime internals or current feature schemas. Their complete implementation remains in the migration file and uses only the stable database, state, Redis, and progress capabilities supplied by the runner. The -host database adapter owns connections, drivers, transactions, and locks so -those infrastructure modules are not frozen into every migration. +host database adapter owns connections, drivers, transactions, and locks. Reusable infrastructure belongs in the append-only -`@sentry/junior/migration-helpers/v1` export. It may provide stable parsers, -stores, adapter projections, and key resolution, but must not implement a -specific data migration. The journal entry remains the only owner of one-off -record transformations. +`@sentry/junior/migration-helpers/v1` export. It may provide stable parsing and +other reusable infrastructure, but must not implement a specific data +migration. The journal entry remains the only owner of one-off record +transformations. The `0000_initial.sql` baseline represents the schema already deployed by the -pre-Drizzle Junior migration runner. During upgrade, existing installations -adopt that baseline once; new installations execute it normally. All later -migrations are applied by Junior in journal order. Drizzle ORM's stock -`migrate()` function is not compatible with TypeScript entries in this folder. - -Migration loading, locking, and legacy baseline adoption live in -`src/chat/conversations/sql/migrations.ts`. Their integration coverage lives in -`tests/integration/conversation-sql.test.ts`. +pre-Drizzle Junior migration runner. Existing databases without core Drizzle +history must first run the documented bridge release; current upgrades do not +infer or adopt legacy state. Fresh databases execute the complete journal +normally. Drizzle ORM's stock `migrate()` function is not compatible with +TypeScript entries in this folder. + +Migration loading, locking, and the bridge-release guard live in +`src/chat/conversations/sql/migrations.ts`. diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index a10f2948a..f63cc7244 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -2,7 +2,6 @@ import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { - resolveMigrations, runMigrationJournal, type MigrationContextV1, type MigrationRunResult, @@ -11,32 +10,10 @@ import { import type { StateAdapter } from "chat"; import type { RedisStateAdapter } from "@chat-adapter/state-redis"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; +import { isPostgresErrorCode } from "@/db/postgres-error"; import { juniorSqlSchema as schema } from "@/db/schema"; -const LEGACY_CORE_MIGRATION_IDS = [ - "0001_conversation_core", - "0002_slack_destination_visibility_backfill", - "0003_user_identities", - "0004_actor_cutover", - "0005_conversation_transcripts", -] as const; -const LEGACY_METRICS_MIGRATION_ID = "0006_conversation_metrics"; -// Pinned output of the pre-Drizzle runner's SHA-256(statement + NUL) algorithm. -const LEGACY_CORE_MIGRATION_CHECKSUMS = { - "0001_conversation_core": - "78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651", - "0002_slack_destination_visibility_backfill": - "fb590a09fa51db471a748e3d7abb4137f521ee8df97f6e9ef5563121be98c394", - "0003_user_identities": - "67d9c9c26cbd76213614eb6d7a7cc7e2501fc20e92321eb5176a08ce39cd2efb", - "0004_actor_cutover": - "d41b8bfa66b8a88d69e84af38950025ba4c9be56341565cbe1411f0ca50c1dc2", - "0005_conversation_transcripts": - "add299d1b254e023f89b5993c417dd2248dc009e874efdeaf31ec0732e0d4fb4", - "0006_conversation_metrics": - "7c7ca5c9e11ed4b0e14737fd90d3348ea46e306c88fdf31199b7afb2a11c6a41", -} as const; -type LegacyCoreMigrationId = keyof typeof LEGACY_CORE_MIGRATION_CHECKSUMS; +const CORE_MIGRATION_BRIDGE_VERSION = "0.107.1"; const MIGRATIONS_TABLE = "__drizzle_junior_core"; /** Resolve the packaged Drizzle migration directory in source or built output. */ @@ -51,136 +28,55 @@ function migrationFolder(): string { return join(packageRoot, "migrations"); } -async function adoptLegacyMigrationState( - executor: JuniorSqlMigrationExecutor, - migrationsFolder: string, -): Promise { - const [tables] = await executor.query<{ - drizzleTable: string | null; - legacyTable: string | null; - }>(` -SELECT - to_regclass('drizzle.__drizzle_junior_core')::text AS "drizzleTable", - to_regclass('public.junior_schema_migrations')::text AS "legacyTable" -`); - if (!tables?.legacyTable || tables.drizzleTable) { - return; - } +interface CoreMigrationState { + appliedAt?: number; + hasJuniorTables: boolean; +} - const migrations = await resolveMigrations(migrationsFolder); - const [metrics] = await executor.query<{ columnCount: number }>(` -SELECT count(*)::integer AS "columnCount" -FROM information_schema.columns -WHERE table_schema = 'public' - AND table_name = 'junior_conversations' - AND column_name IN ( - 'duration_ms', - 'usage_json', - 'execution_duration_ms', - 'execution_usage_json' - ) +/** Read core journal progress and detect existing pre-journal Junior tables. */ +async function loadCoreMigrationState( + executor: JuniorSqlMigrationExecutor, +): Promise { + try { + const [migration] = await executor.query<{ appliedAt: string | null }>(` +SELECT created_at::text AS "appliedAt" +FROM drizzle.__drizzle_junior_core +ORDER BY created_at DESC +LIMIT 1 `); - const legacyRecords = await executor.query<{ - checksum: string; - id: string; - }>("SELECT id, checksum FROM junior_schema_migrations"); - const legacyRecordsById = new Map( - legacyRecords.map((record) => [record.id, record.checksum]), - ); - const metricColumnCount = metrics?.columnCount ?? 0; - if (metricColumnCount !== 0 && metricColumnCount !== 4) { - throw new Error( - `Cannot adopt partial legacy metrics state: found ${metricColumnCount} of 4 required columns`, - ); - } - const metricsComplete = metricColumnCount === 4; - const hasMetricsRecord = legacyRecordsById.has(LEGACY_METRICS_MIGRATION_ID); - if (metricsComplete !== hasMetricsRecord) { - throw new Error( - "Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns", - ); - } - const expectedIds: readonly LegacyCoreMigrationId[] = metricsComplete - ? [...LEGACY_CORE_MIGRATION_IDS, LEGACY_METRICS_MIGRATION_ID] - : [...LEGACY_CORE_MIGRATION_IDS]; - const missingIds = expectedIds.filter((id) => !legacyRecordsById.has(id)); - if (missingIds.length > 0) { - throw new Error( - `Cannot adopt partial legacy core migration state; missing: ${missingIds.join(", ")}`, - ); - } - const checksumMismatches = expectedIds.filter( - (id) => legacyRecordsById.get(id) !== LEGACY_CORE_MIGRATION_CHECKSUMS[id], - ); - if (checksumMismatches.length > 0) { - throw new Error( - `Cannot adopt legacy core migration state: checksum mismatch: ${checksumMismatches.join(", ")}`, - ); + if (migration?.appliedAt !== null && migration?.appliedAt !== undefined) { + return { + appliedAt: Number(migration.appliedAt), + hasJuniorTables: true, + }; + } + } catch (error) { + if (!isPostgresErrorCode(error, "42P01")) { + throw error; + } } - const [baseline] = await executor.query<{ - conversationEventsTable: string | null; - legacyAgentStepsTable: boolean; - metricRunIdColumn: boolean; - searchIndex: string | null; - }>(` -SELECT - to_regclass('public.junior_conversation_events')::text AS "conversationEventsTable", - to_regclass('public.junior_conversation_messages_search_idx')::text AS "searchIndex", - EXISTS ( + const [tables] = await executor.query<{ hasJuniorTables: boolean }>(` +SELECT EXISTS ( SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' - AND table_name = 'junior_agent_steps' - AND table_type = 'BASE TABLE' - ) AS "legacyAgentStepsTable", - EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'junior_conversations' - AND column_name = 'metric_run_id' - ) AS "metricRunIdColumn" + AND table_name LIKE 'junior\\_%' ESCAPE '\\' + ) AS "hasJuniorTables" `); - if (!baseline?.legacyAgentStepsTable || baseline.conversationEventsTable) { - throw new Error( - "Cannot adopt legacy core migration state: expected the pre-Drizzle junior_agent_steps table and no junior_conversation_events table", - ); - } - const postBaselineMarkers = [ - baseline.searchIndex - ? "junior_conversation_messages_search_idx" - : undefined, - baseline.metricRunIdColumn - ? "junior_conversations.metric_run_id" - : undefined, - ].filter((marker): marker is string => marker !== undefined); - if (postBaselineMarkers.length > 0) { - throw new Error( - `Cannot adopt legacy core migration state: post-baseline schema markers are already present: ${postBaselineMarkers.join(", ")}`, - ); - } + return { + hasJuniorTables: tables?.hasJuniorTables ?? false, + }; +} - const migration = metricsComplete ? migrations[1] : migrations[0]; - if (!migration) { - throw new Error("No core Drizzle migrations were packaged"); +/** Reject legacy databases that must first run the bridge release upgrade. */ +function assertSupportedMigrationState(state: CoreMigrationState): void { + if (!state.hasJuniorTables || state.appliedAt !== undefined) { + return; } - - await executor.transaction(async () => { - await executor.execute("CREATE SCHEMA IF NOT EXISTS drizzle"); - await executor.execute(` -CREATE TABLE IF NOT EXISTS drizzle.__drizzle_junior_core ( - id SERIAL PRIMARY KEY, - hash TEXT NOT NULL, - created_at BIGINT -) -`); - await executor.execute( - `INSERT INTO drizzle.__drizzle_junior_core (hash, created_at) - VALUES ($1, $2)`, - [migration.hash, migration.when], - ); - }); + throw new Error( + `Existing Junior SQL tables have no core Drizzle migration history. Stop old Junior workers, install @sentry/junior@${CORE_MIGRATION_BRIDGE_VERSION}, run \`junior upgrade\`, then restore this Junior version and rerun \`junior upgrade\` before restarting workers.`, + ); } export type MigrateSchemaOptions = @@ -198,13 +94,13 @@ export { schema }; /** Apply all migrations, or bootstrap an empty test database to the latest schema. */ export async function migrateSchema( executor: JuniorSqlMigrationExecutor, - options: MigrateSchemaOptions, + options: MigrateSchemaOptions = { mode: "schema-bootstrap" }, ): Promise { const migrationsFolder = migrationFolder(); const runAll = options.mode === "all"; const baseOptions = { beforeRun: async () => { - await adoptLegacyMigrationState(executor, migrationsFolder); + assertSupportedMigrationState(await loadCoreMigrationState(executor)); }, executor, migrationsFolder, diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index e3c620489..25c0b61ff 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -135,7 +135,7 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, roots: readonly PluginMigrationRoot[], - options: PluginMigrationOptions, + options: PluginMigrationOptions = { mode: "schema-bootstrap" }, ): Promise { const result: PluginMigrationResult = { existing: 0, diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index b5ff82c17..3b4884ac2 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -6,7 +6,6 @@ import { createJiti } from "jiti"; import { loadAppPluginSet } from "@/plugin-module"; import { migrateCoreJournal } from "./upgrade/migrations/core-journal"; import { migratePluginJournals } from "./upgrade/migrations/plugin-journal"; -import { resolveUpgradePlugins } from "./upgrade/migrations/upgrade-plugins"; import type { MigrationContext, MigrationResult, @@ -69,16 +68,14 @@ function formatMigrationResult(result: MigrationResult): string { export async function runUpgradeMigrations( context: MigrationContext, ): Promise { - const plugins = await resolveUpgradePlugins(context); - const migrationContext = { ...context, ...plugins }; const results: MigrationResult[] = []; const run = async ( name: string, migrate: (context: MigrationContext) => Promise, ): Promise => { - migrationContext.io.info(`Running migration ${name}...`); - const result = await migrate(migrationContext); - migrationContext.io.info( + context.io.info(`Running migration ${name}...`); + const result = await migrate(context); + context.io.info( `Finished migration ${name}: ${formatMigrationResult(result)}`, ); results.push(result); diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts index d03c89249..c5a7e988b 100644 --- a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -3,7 +3,7 @@ import { migratePluginSchemas } from "@/chat/plugins/migrations"; import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; import { createJuniorSqlExecutor } from "@/db/executor"; import { createJiti } from "jiti"; -import { resolveUpgradePlugins } from "./upgrade-plugins"; +import { resolveUpgradePluginCatalog } from "./upgrade-plugins"; import type { MigrationContext, MigrationResult } from "../types"; const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); @@ -13,7 +13,7 @@ export async function migratePluginJournals( context: MigrationContext, ): Promise { const { sql } = getChatConfig(); - const { pluginCatalogConfig } = await resolveUpgradePlugins(context); + const pluginCatalogConfig = await resolveUpgradePluginCatalog(context); const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig); const executor = createJuniorSqlExecutor({ connectionString: sql.databaseUrl, diff --git a/packages/junior/src/cli/upgrade/migrations/upgrade-plugins.ts b/packages/junior/src/cli/upgrade/migrations/upgrade-plugins.ts index 901b16a2b..7d3fc46cd 100644 --- a/packages/junior/src/cli/upgrade/migrations/upgrade-plugins.ts +++ b/packages/junior/src/cli/upgrade/migrations/upgrade-plugins.ts @@ -7,14 +7,14 @@ import { pluginCatalogConfigFromEnv, pluginCatalogConfigFromPluginSet, } from "@/plugins"; -import type { UpgradeContext } from "../types"; +import type { MigrationContext } from "../types"; function unique(values: string[]): string[] { return [...new Set(values)]; } function baseCatalogConfig( - context: UpgradeContext, + context: MigrationContext, ): PluginCatalogConfig | undefined { return ( context.pluginCatalogConfig ?? @@ -69,7 +69,7 @@ function mergeCatalogConfig( } function packageNamesFromContext( - context: UpgradeContext, + context: MigrationContext, catalog: PluginCatalogConfig | undefined, ): string[] { return unique([ @@ -80,7 +80,7 @@ function packageNamesFromContext( /** Resolve the plugin catalog used by SQL upgrade migrations. */ export async function resolveUpgradePluginCatalog( - context: UpgradeContext, + context: MigrationContext, ): Promise { const catalog = baseCatalogConfig(context); const packageNames = packageNamesFromContext(context, catalog); diff --git a/packages/junior/src/migration-helpers/README.md b/packages/junior/src/migration-helpers/README.md index f2550f515..a9bfb9b03 100644 --- a/packages/junior/src/migration-helpers/README.md +++ b/packages/junior/src/migration-helpers/README.md @@ -3,10 +3,10 @@ This directory owns versioned, append-only infrastructure helpers available to packaged TypeScript migrations through `@sentry/junior/migration-helpers/v1`. -Helpers may expose stable parsing primitives, state/database adapters, stores, -and key resolution. They must not contain one-off migration decisions or data -transforms. Logic such as mapping one retired record shape into another belongs -only in the journal entry that performs that migration. +Helpers may expose stable parsing primitives and other reusable infrastructure. +They must not contain one-off migration decisions or data transforms. Logic +such as mapping one retired record shape into another belongs only in the +journal entry that performs that migration. The behavior and signature of an existing version are permanent compatibility contracts. Add a new versioned entrypoint when that contract must change. diff --git a/packages/junior/src/migration-helpers/v1.ts b/packages/junior/src/migration-helpers/v1.ts index 15665d5fd..6f841b8bf 100644 --- a/packages/junior/src/migration-helpers/v1.ts +++ b/packages/junior/src/migration-helpers/v1.ts @@ -1,160 +1,14 @@ import type { Destination } from "@sentry/junior-plugin-api"; -import type { - MigrationDatabaseAdapter, - MigrationStateV1, -} from "@sentry/junior-migrations"; -import type { StateAdapter } from "chat"; import { isRecord as runtimeIsRecord, toOptionalNumber as runtimeToOptionalNumber, toOptionalString as runtimeToOptionalString, } from "@/chat/coerce"; -import { getChatConfig } from "@/chat/config"; -import { createLegacyAdvisorSessionReader } from "@/chat/conversations/legacy-advisor-session"; -import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; -import { createStateConversationStore } from "@/chat/conversations/state"; -import { createSqlStore } from "@/chat/conversations/sql/store"; -import { toStoredConversationMessage } from "@/chat/conversations/visible-messages"; import { parseDestination as runtimeParseDestination, sameDestination as runtimeSameDestination, } from "@/chat/destination"; -import { coerceThreadConversationState as runtimeCoerceThreadConversationState } from "@/chat/state/conversation"; -import { listAgentTurnSessionSummariesForConversations } from "@/chat/state/turn-session"; -import { - contextProvenance, - decodeSessionLogEntry, - legacyActorProvenance, - piMessageProvenanceSchema, -} from "@/chat/state/session-log"; -import { - getConversation, - requestConversationWork, -} from "@/chat/task-execution/state"; -import { addAgentTurnUsage as runtimeAddAgentTurnUsage } from "@/chat/usage"; import { unescapeXml } from "@/chat/xml"; -import type { JuniorSqlDatabase } from "@/db/db"; -import { - juniorAgentSteps, - juniorConversationMessages, - juniorConversations, -} from "@/db/schema"; - -export const JUNIOR_THREAD_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1_000; -export const migrationContextProvenance: unknown = contextProvenance; -export const migrationJuniorAgentSteps: unknown = juniorAgentSteps; -export const migrationJuniorConversationMessages: unknown = - juniorConversationMessages; -export const migrationJuniorConversations: unknown = juniorConversations; -export const migrationPiMessageProvenanceSchema: unknown = - piMessageProvenanceSchema; - -export type MigrationSourceV1 = - | "api" - | "internal" - | "local" - | "plugin" - | "resource_event" - | "scheduler" - | "slack"; - -export type MigrationExecutionStatusV1 = - | "awaiting_resume" - | "failed" - | "idle" - | "pending" - | "running"; - -/** Frozen retained inbound-message shape used by v1 data migrations. */ -export interface MigrationInboundMessageV1 { - attemptCount?: number; - conversationId: string; - createdAtMs: number; - destination: Destination; - inboundMessageId: string; - injectedAtMs?: number; - input: { - attachments?: unknown[]; - authorId?: string; - metadata?: Record; - text: string; - }; - receivedAtMs: number; - source: MigrationSourceV1; -} - -/** Frozen retained execution-lease shape used by v1 data migrations. */ -export interface MigrationLeaseV1 { - acquiredAtMs: number; - expiresAtMs: number; - lastCheckInAtMs: number; - token: string; -} - -/** Frozen conversation projection shared by v1 state and SQL helpers. */ -export interface MigrationConversationV1 { - actor?: unknown; - channelName?: string; - conversationId: string; - createdAtMs: number; - destination?: Destination; - execution: { - inboundMessageIds?: string[]; - lastCheckpointAtMs?: number; - lastEnqueuedAtMs?: number; - lease?: MigrationLeaseV1; - pendingCount?: number; - pendingMessages?: MigrationInboundMessageV1[]; - runId?: string; - status: MigrationExecutionStatusV1; - updatedAtMs?: number; - }; - lastActivityAtMs: number; - schemaVersion: 1; - source?: MigrationSourceV1; - title?: string; - updatedAtMs: number; -} - -/** State-backed conversation projection with retained mailbox fields. */ -export interface MigrationRetainedConversationV1 extends MigrationConversationV1 { - execution: MigrationConversationV1["execution"] & { - inboundMessageIds: string[]; - pendingCount: number; - pendingMessages: MigrationInboundMessageV1[]; - }; -} - -/** Legacy thread-state projection required by the v1 continuation migration. */ -export interface MigrationThreadConversationStateV1 { - processing: { activeTurnId?: string }; -} - -/** Frozen turn-summary projection used by the conversation SQL backfill. */ -export interface MigrationTurnSessionSummaryV1 { - cumulativeDurationMs: number; - cumulativeUsage?: Record; - sessionId: string; -} - -/** Narrow state conversation store exposed to v1 migrations. */ -export interface MigrationStateConversationStoreV1 { - listByActivity(args: { limit: number }): Promise; -} - -/** Narrow SQL conversation store exposed to v1 migrations. */ -export interface MigrationSqlConversationStoreV1 { - backfillConversation( - conversation: MigrationConversationV1, - metrics?: { - durationMs: number; - executionDurationMs: number; - executionUsage?: Record; - usage?: Record; - }, - ): Promise; - listByActivity(args: { limit: number }): Promise; -} /** Return whether a value is a non-null object record. */ export function isRecord(value: unknown): value is Record { @@ -184,115 +38,7 @@ export function sameDestination( return runtimeSameDestination(left, right); } -/** Coerce one legacy thread-state value into the retained conversation shape. */ -export function coerceThreadConversationState( - value: unknown, -): MigrationThreadConversationStateV1 { - return runtimeCoerceThreadConversationState( - value, - ) as unknown as MigrationThreadConversationStateV1; -} - -/** Merge retained turn usage records. */ -export function addAgentTurnUsage( - ...values: Array | undefined> -): Record | undefined { - return runtimeAddAgentTurnUsage(...values) as - | Record - | undefined; -} - -/** Resolve a logical state key to the configured physical Redis key. */ -export function migrationRedisKey(key: string): string { - const prefix = getChatConfig().state.keyPrefix; - return [...(prefix ? [prefix] : []), key].join(":"); -} - -/** Create the retained state-backed conversation store for a v1 migration. */ -export function createMigrationStateConversationStore( - state: MigrationStateV1, -): MigrationStateConversationStoreV1 { - return createStateConversationStore( - state as StateAdapter, - ) as unknown as MigrationStateConversationStoreV1; -} - -/** Create the SQL conversation store for a v1 migration database adapter. */ -export function createMigrationSqlStore( - database: MigrationDatabaseAdapter, -): MigrationSqlConversationStoreV1 { - return createSqlStore( - database as unknown as JuniorSqlDatabase, - ) as unknown as MigrationSqlConversationStoreV1; -} - -/** Create the SQL message store used by legacy history migrations. */ -export function createMigrationSqlConversationMessageStore( - database: MigrationDatabaseAdapter, -): unknown { - return createSqlConversationMessageStore( - database as unknown as JuniorSqlDatabase, - ); -} - -/** Create the legacy advisor reader used by history migrations. */ -export function createMigrationLegacyAdvisorSessionReader(): unknown { - return createLegacyAdvisorSessionReader(); -} - -/** Decode one retained session-log value. */ -export function decodeMigrationSessionLogEntry(value: unknown): unknown { - return decodeSessionLogEntry(value); -} - -/** Recover provenance from one legacy actor value. */ -export function migrationLegacyActorProvenance(value: unknown): unknown { - return legacyActorProvenance(value as never); -} - -/** Convert one legacy visible message into its SQL projection. */ -export function toMigrationStoredConversationMessage(value: unknown): unknown { - return toStoredConversationMessage(value as never); -} - -/** Unescape one legacy XML text fragment. */ +/** Unescape one persisted XML text fragment. */ export function migrationUnescapeXml(value: string): string { return unescapeXml(value); } - -/** Read one retained conversation through the v1 migration state adapter. */ -export async function getMigrationConversation(args: { - conversationId: string; - state: MigrationStateV1; -}): Promise { - return (await getConversation({ - conversationId: args.conversationId, - state: args.state as StateAdapter, - })) as unknown as MigrationRetainedConversationV1 | undefined; -} - -/** Mark one retained conversation runnable through the v1 migration state adapter. */ -export async function requestMigrationConversationWork(args: { - conversationId: string; - destination: Destination; - nowMs?: number; - state: MigrationStateV1; -}): Promise { - await requestConversationWork({ - conversationId: args.conversationId, - destination: args.destination, - ...(args.nowMs === undefined ? {} : { nowMs: args.nowMs }), - state: args.state as StateAdapter, - }); -} - -/** Read retained turn summaries through the v1 migration state adapter. */ -export async function listMigrationTurnSessionSummaries( - state: MigrationStateV1, - conversationIds: string[], -): Promise> { - return (await listAgentTurnSessionSummariesForConversations( - state as StateAdapter, - conversationIds, - )) as Map; -} diff --git a/packages/junior/tests/component/cli/upgrade-cli.test.ts b/packages/junior/tests/component/cli/upgrade-cli.test.ts index b0b5918ad..c64d13f2c 100644 --- a/packages/junior/tests/component/cli/upgrade-cli.test.ts +++ b/packages/junior/tests/component/cli/upgrade-cli.test.ts @@ -1,111 +1,17 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import type { RedisStateAdapter } from "@chat-adapter/state-redis"; -import type { MigrationContextV1 } from "@sentry/junior-migrations"; -import { createJiti } from "jiti"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { getChatConfig } from "@/chat/config"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import { - appendInboundMessage, - CONVERSATION_ACTIVE_INDEX_KEY, - CONVERSATION_BY_ACTIVITY_INDEX_KEY, - requestConversationWork, -} from "@/chat/task-execution/store"; -import { createSqlStore } from "@/chat/conversations/sql/store"; -import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import type { ConversationStore } from "@/chat/conversations/store"; -import type { PiMessage } from "@/chat/pi/messages"; -import { persistThreadStateById } from "@/chat/runtime/thread-state"; -import { recordAgentTurnSessionSummary } from "@/chat/state/turn-session"; +import { afterEach, describe, expect, it } from "vitest"; import { resolveUpgradePluginSet } from "@/cli/upgrade"; -import { migrateAgentTurnSessionActor } from "../../../migrations/0005_agent_turn_session_actor"; -import { migrateRedisConversationState } from "../../../migrations/0006_redis_conversation_state"; -import { migrateConversationsToSql } from "../../../migrations/0007_conversations_to_sql"; -import { repairConversationUsage } from "../../../migrations/0009_conversation_usage"; -import { - CONVERSATION_ID, - SLACK_DESTINATION, - inboundMessage, -} from "../../fixtures/conversation-work"; -import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; -const ORIGINAL_ENV = vi.hoisted(() => { - const original = { - DATABASE_URL: process.env.DATABASE_URL, - JUNIOR_STATE_ADAPTER: process.env.JUNIOR_STATE_ADAPTER, - }; - process.env.DATABASE_URL = "postgres://configured.example.test/neon"; - process.env.JUNIOR_STATE_ADAPTER = "memory"; - return original; -}); const ORIGINAL_CWD = process.cwd(); -const OTHER_SLACK_DESTINATION = { - ...SLACK_DESTINATION, - channelId: "C999", -} as const; -const migrationLoader = createJiti(import.meta.url, { moduleCache: false }); - -const stateOnlyConversationStore: ConversationStore = { - get: async () => undefined, - getDestinationVisibility: async () => undefined, - recordActivity: async () => {}, - recordExecution: async () => {}, - listByActivity: async () => [], -}; - -function restoreEnv(name: string, value: string | undefined): void { - if (value === undefined) { - delete process.env[name]; - return; - } - process.env[name] = value; -} - -async function persistActiveTurn( - conversationId: string, - activeTurnId?: string, -): Promise { - await persistThreadStateById(conversationId, { - conversation: { - schemaVersion: 1, - backfill: {}, - compactions: [], - messages: [], - processing: { - activeTurnId, - }, - stats: { - compactedMessageCount: 0, - estimatedContextTokens: 0, - totalMessageCount: 0, - updatedAtMs: 2_000, - }, - vision: { - byFileId: {}, - }, - }, - }); -} - -describe("upgrade CLI migrations", () => { - beforeEach(async () => { - process.env.DATABASE_URL = "postgres://configured.example.test/neon"; - process.env.JUNIOR_STATE_ADAPTER = "memory"; - await disconnectStateAdapter(); - }); - afterEach(async () => { +describe("upgrade CLI", () => { + afterEach(() => { process.chdir(ORIGINAL_CWD); - await disconnectStateAdapter(); - restoreEnv("DATABASE_URL", ORIGINAL_ENV.DATABASE_URL); - restoreEnv("JUNIOR_STATE_ADAPTER", ORIGINAL_ENV.JUNIOR_STATE_ADAPTER); - vi.restoreAllMocks(); }); - it("loads source app plugins for upgrade when virtual config is unavailable", async () => { + it("loads source app plugins when virtual config is unavailable", async () => { const tempDir = mkdtempSync(path.join(tmpdir(), "junior-upgrade-plugins-")); writeFileSync( path.join(tempDir, "plugins.ts"), @@ -129,1244 +35,4 @@ export const plugins = { rmSync(tempDir, { force: true, recursive: true }); } }); - - it("migrates legacy requester fields in turn-session state", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const conversationId = "slack:C123:legacy-actor"; - const sessionId = "turn-legacy-actor"; - const requester = { - platform: "slack", - teamId: "T123", - userId: "U123", - userName: "alice", - }; - const summary = { - version: 1, - conversationId, - cumulativeDurationMs: 0, - lastProgressAtMs: 2, - requester, - sessionId, - sliceId: 1, - startedAtMs: 1, - state: "completed", - updatedAtMs: 3, - }; - - await stateAdapter.appendToList( - "junior:agent_turn_session:index", - summary, - { ttlMs: 60_000 }, - ); - await stateAdapter.appendToList( - `junior:agent_turn_session:conversation:${conversationId}:index`, - summary, - { ttlMs: 60_000 }, - ); - await stateAdapter.set( - `junior:agent_turn_session:${conversationId}:${sessionId}`, - { ...summary, committedSeq: -1 }, - 60_000, - ); - - await expect( - migrateAgentTurnSessionActor({ - io: { info: () => {} }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 3, - missing: 0, - scanned: 3, - }); - - const { listBoundedAgentTurnSessionSummariesForConversation } = - await import("@/chat/state/turn-session"); - await expect( - listBoundedAgentTurnSessionSummariesForConversation(conversationId), - ).resolves.toEqual([ - expect.objectContaining({ actor: requester, sessionId }), - ]); - const migratedRecord = await stateAdapter.get( - `junior:agent_turn_session:${conversationId}:${sessionId}`, - ); - expect(migratedRecord).toEqual( - expect.objectContaining({ actor: requester }), - ); - expect(migratedRecord).not.toHaveProperty("requester"); - - await expect( - migrateAgentTurnSessionActor({ - io: { info: () => {} }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 1, - migrated: 0, - missing: 0, - scanned: 3, - }); - }); - - it("discovers legacy turn sessions outside the bounded global index", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const conversationId = "slack:C123:older-legacy-actor"; - const sessionId = "turn-older-legacy-actor"; - const requester = { - platform: "slack", - teamId: "T123", - userId: "U123", - }; - const summary = { - version: 1, - conversationId, - cumulativeDurationMs: 0, - lastProgressAtMs: 2, - requester, - sessionId, - sliceId: 1, - startedAtMs: 1, - state: "completed", - updatedAtMs: 3, - }; - - await stateAdapter.appendToList( - `junior:agent_turn_session:conversation:${conversationId}:index`, - summary, - { ttlMs: 60_000 }, - ); - await stateAdapter.set( - `junior:agent_turn_session:${conversationId}:${sessionId}`, - { ...summary, committedSeq: -1 }, - 60_000, - ); - - const statePrefix = getChatConfig().state.keyPrefix; - const redisConversationIndexKey = [ - "chat-sdk:list", - ...(statePrefix ? [statePrefix] : []), - `junior:agent_turn_session:conversation:${conversationId}:index`, - ].join(":"); - const redisStateAdapter = { - getClient: () => ({ - sendCommand: async (args: readonly string[]) => { - expect(args).toEqual([ - "SCAN", - "0", - "MATCH", - `*:list:${statePrefix ? `${statePrefix}:` : ""}junior:agent_turn_session:conversation:*:index`, - "COUNT", - "500", - ]); - return ["0", [redisConversationIndexKey]]; - }, - }), - } as unknown as RedisStateAdapter; - - await expect( - migrateAgentTurnSessionActor({ - io: { info: () => {} }, - redisStateAdapter, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 2, - missing: 0, - scanned: 2, - }); - - const { listBoundedAgentTurnSessionSummariesForConversation } = - await import("@/chat/state/turn-session"); - await expect( - listBoundedAgentTurnSessionSummariesForConversation(conversationId), - ).resolves.toEqual([ - expect.objectContaining({ actor: requester, sessionId }), - ]); - await expect( - stateAdapter.get( - `junior:agent_turn_session:${conversationId}:${sessionId}`, - ), - ).resolves.toEqual(expect.objectContaining({ actor: requester })); - }); - it("runs journaled TypeScript state migrations exactly once", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const actor = { - platform: "slack", - teamId: "T123", - userId: "migration-user", - } as const; - const summary = { - version: 1, - conversationId: CONVERSATION_ID, - cumulativeDurationMs: 0, - lastProgressAtMs: 2, - sessionId: "session-one", - sliceId: 1, - startedAtMs: 1, - state: "completed", - updatedAtMs: 3, - requester: actor, - }; - await stateAdapter.appendToList("junior:agent_turn_session:index", summary); - await stateAdapter.appendToList( - `junior:agent_turn_session:conversation:${CONVERSATION_ID}:index`, - summary, - ); - await stateAdapter.set( - `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, - summary, - ); - - try { - await expect( - migrateSchema(fixture.sql, { - loadTypeScript: async (migrationPath) => - await migrationLoader.import>( - migrationPath, - ), - mode: "all", - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 10, - scanned: 10, - skipped: 0, - }); - await expect( - stateAdapter.getList("junior:agent_turn_session:index"), - ).resolves.toEqual([ - expect.objectContaining({ - actor, - conversationId: CONVERSATION_ID, - sessionId: "session-one", - }), - ]); - await expect( - stateAdapter.get( - `junior:agent_turn_session:${CONVERSATION_ID}:session-one`, - ), - ).resolves.toEqual( - expect.objectContaining({ - actor, - conversationId: CONVERSATION_ID, - sessionId: "session-one", - }), - ); - await expect( - migrateSchema(fixture.sql, { - loadTypeScript: async (migrationPath) => - await migrationLoader.import>( - migrationPath, - ), - mode: "all", - stateAdapter, - }), - ).resolves.toEqual({ - existing: 10, - migrated: 0, - scanned: 10, - skipped: 0, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("passes the host state adapter through without wrapping its identity", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const observedStates: unknown[] = []; - - try { - await migrateSchema(fixture.sql, { - loadTypeScript: async () => ({ - default: { - apiVersion: 1, - async up(context: MigrationContextV1) { - observedStates.push(context.state); - }, - }, - }), - mode: "all", - stateAdapter, - }); - - expect(observedStates).toHaveLength(5); - expect(observedStates.every((state) => state === stateAdapter)).toBe( - true, - ); - } finally { - await fixture.close(); - } - }); - - it("migrates legacy conversation work before SQL conversation backfill", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const legacyMessage = inboundMessage("legacy-sql"); - await stateAdapter.set( - `junior:conversation-work:state:${CONVERSATION_ID}`, - { - schemaVersion: 1, - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - messages: [legacyMessage], - needsRun: true, - updatedAtMs: 2_000, - }, - ); - await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - const context = { - io: { info: () => {} }, - stateAdapter, - }; - const results = [ - await migrateRedisConversationState(context), - await migrateConversationsToSql(context, { target: sqlStore }), - ]; - - expect(results).toEqual([ - { - existing: 0, - migrated: 1, - missing: 0, - scanned: 1, - }, - { - existing: 0, - migrated: 1, - missing: 0, - scanned: 1, - }, - ]); - await expect( - stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), - ).resolves.toMatchObject({ - conversationId: CONVERSATION_ID, - execution: { - inboundMessageIds: ["legacy-sql"], - pendingCount: 1, - status: "pending", - }, - }); - const sqlConversation = await sqlStore.get({ - conversationId: CONVERSATION_ID, - }); - expect(sqlConversation).toMatchObject({ - conversationId: CONVERSATION_ID, - execution: { - status: "pending", - }, - }); - expect(sqlConversation?.execution).not.toHaveProperty("pendingCount"); - expect(sqlConversation?.execution).not.toHaveProperty("pendingMessages"); - } finally { - await fixture.close(); - } - }, 15_000); - - it("copies a bounded SQL conversation backfill slice", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - for (let index = 0; index < 3; index++) { - const conversationId = `slack:C123:page-${index}`; - await appendInboundMessage({ - message: inboundMessage(`page-${index}`, { conversationId }), - nowMs: 1_000 + index, - conversationStore: stateOnlyConversationStore, - state: stateAdapter, - }); - } - - await expect( - migrateConversationsToSql( - { - io: { info: () => {} }, - stateAdapter, - }, - { batchSize: 2, target: sqlStore }, - ), - ).resolves.toEqual({ - existing: 0, - migrated: 2, - missing: 0, - scanned: 2, - }); - await expect(sqlStore.listByActivity({ limit: 10 })).resolves.toEqual([ - expect.objectContaining({ conversationId: "slack:C123:page-2" }), - expect.objectContaining({ conversationId: "slack:C123:page-1" }), - ]); - } finally { - await fixture.close(); - } - }, 15_000); - - it("backfills retained conversation metrics without replacing SQL totals", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - const seedMs = Date.now() - 1_000; - await sqlStore.recordExecution({ - conversationId: CONVERSATION_ID, - createdAtMs: seedMs, - destination: SLACK_DESTINATION, - execution: { - runId: "run-two", - status: "idle", - updatedAtMs: seedMs, - }, - lastActivityAtMs: seedMs, - metrics: null, - updatedAtMs: seedMs, - }); - await recordAgentTurnSessionSummary({ - conversationId: CONVERSATION_ID, - cumulativeDurationMs: 1_000, - cumulativeUsage: { - inputTokens: 40, - outputTokens: 10, - reasoningTokens: 3, - cost: { total: 0.001 }, - }, - destination: SLACK_DESTINATION, - conversationStore: stateOnlyConversationStore, - sessionId: "run-one", - sliceId: 1, - state: "completed", - surface: "slack", - }); - await recordAgentTurnSessionSummary({ - conversationId: CONVERSATION_ID, - cumulativeDurationMs: 2_000, - cumulativeUsage: { - inputTokens: 80, - outputTokens: 20, - reasoningTokens: 7, - cost: { total: 0.002 }, - }, - destination: SLACK_DESTINATION, - conversationStore: stateOnlyConversationStore, - sessionId: "run-two", - sliceId: 1, - state: "completed", - surface: "slack", - }); - - await migrateConversationsToSql( - { io: { info: () => {} }, stateAdapter }, - { target: sqlStore }, - ); - - const readMetrics = async () => { - const [row] = await fixture.sql.query<{ - durationMs: number; - executionDurationMs: number; - executionUsage: { - inputTokens?: number; - outputTokens?: number; - } | null; - usage: { - cost?: { total?: number }; - inputTokens?: number; - outputTokens?: number; - reasoningTokens?: number; - totalTokens?: number; - } | null; - }>( - ` -SELECT - duration_ms AS "durationMs", - usage_json AS usage, - execution_duration_ms AS "executionDurationMs", - execution_usage_json AS "executionUsage" -FROM junior_conversations -WHERE conversation_id = $1 -`, - [CONVERSATION_ID], - ); - return row; - }; - await expect(readMetrics()).resolves.toMatchObject({ - durationMs: 3_000, - executionDurationMs: 2_000, - executionUsage: { - inputTokens: 80, - outputTokens: 20, - }, - usage: { - inputTokens: 120, - outputTokens: 30, - reasoningTokens: 10, - cost: { total: 0.003 }, - }, - }); - - const futureMs = Date.now() + 60_000; - await sqlStore.recordExecution({ - conversationId: CONVERSATION_ID, - createdAtMs: futureMs, - execution: { - runId: "run-three", - status: "idle", - updatedAtMs: futureMs, - }, - lastActivityAtMs: futureMs, - metrics: { - durationMs: 500, - usage: { - inputTokens: 5, - outputTokens: 5, - reasoningTokens: 1, - cost: { total: 0.0005 }, - }, - }, - updatedAtMs: futureMs, - }); - await migrateConversationsToSql( - { io: { info: () => {} }, stateAdapter }, - { target: sqlStore }, - ); - - await expect(readMetrics()).resolves.toMatchObject({ - durationMs: 3_500, - executionDurationMs: 500, - usage: { - reasoningTokens: 11, - totalTokens: 160, - cost: { total: 0.0035 }, - }, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("repairs SQL usage in bounded batches without changing duration", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - const eventStore = createSqlConversationEventStore(fixture.sql); - const mixedConversationId = `${CONVERSATION_ID}:mixed`; - const unsafeConversationId = `${CONVERSATION_ID}:unsafe`; - const firstAssistant = { - role: "assistant", - content: [{ type: "text", text: "first" }], - api: "responses", - provider: "openai", - model: "test-model", - stopReason: "stop", - timestamp: 2_000, - usage: { - input: 10, - output: 2, - cacheRead: 3, - cacheWrite: 1, - reasoning: 1, - totalTokens: 16, - cost: { - input: 0.001, - output: 0.002, - cacheRead: 0.0003, - cacheWrite: 0.0004, - total: 0.0037, - }, - }, - } as PiMessage; - const secondAssistant = { - role: "assistant", - content: [{ type: "text", text: "second" }], - api: "responses", - provider: "openai", - model: "test-model", - stopReason: "stop", - timestamp: 3_000, - usage: { - input: 20, - output: 5, - cacheRead: 0, - cacheWrite: 0, - reasoning: 2, - totalTokens: 25, - cost: { - input: 0.004, - output: 0.006, - cacheRead: 0, - cacheWrite: 0, - total: 0.01, - }, - }, - } as PiMessage; - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - await sqlStore.recordExecution({ - conversationId: CONVERSATION_ID, - createdAtMs: 1_000, - execution: { - runId: "run-two", - status: "idle", - updatedAtMs: 4_000, - }, - lastActivityAtMs: 4_000, - metrics: { - durationMs: 9_000, - usage: { totalTokens: 999, cost: { total: 0.999 } }, - }, - updatedAtMs: 4_000, - }); - await eventStore.append(CONVERSATION_ID, [ - { - data: { type: "agent_step", message: firstAssistant }, - createdAtMs: 2_000, - }, - ]); - await eventStore.replaceHistory(CONVERSATION_ID, { - createdAtMs: 2_500, - data: { - type: "compaction", - modelProfile: "standard", - modelId: "test-model", - replacementHistory: [{ message: firstAssistant }], - }, - }); - await eventStore.append(CONVERSATION_ID, [ - { - data: { type: "agent_step", message: secondAssistant }, - createdAtMs: 3_000, - }, - ]); - await sqlStore.recordExecution({ - conversationId: mixedConversationId, - createdAtMs: 1_000, - execution: { status: "idle", updatedAtMs: 4_000 }, - lastActivityAtMs: 4_000, - metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, - updatedAtMs: 4_000, - }); - await eventStore.append(mixedConversationId, [ - { - data: { - type: "agent_step", - message: { - role: "assistant", - usage: { input: 3, totalTokens: 999 }, - } as PiMessage, - }, - createdAtMs: 2_000, - }, - { - data: { - type: "agent_step", - message: { - role: "assistant", - usage: { totalTokens: 5 }, - } as PiMessage, - }, - createdAtMs: 3_000, - }, - { - data: { - type: "agent_step", - message: { - role: "user", - usage: { input: 1_000, output: 1_000 }, - } as unknown as PiMessage, - }, - createdAtMs: 3_500, - }, - ]); - await sqlStore.recordExecution({ - conversationId: unsafeConversationId, - createdAtMs: 1_000, - execution: { status: "idle", updatedAtMs: 4_000 }, - lastActivityAtMs: 4_000, - metrics: { durationMs: 1_000, usage: { totalTokens: 777 } }, - updatedAtMs: 4_000, - }); - await eventStore.append(unsafeConversationId, [ - { - data: { - type: "agent_step", - message: { - role: "assistant", - usage: { input: 9_007_199_254_740_992 }, - } as PiMessage, - }, - createdAtMs: 3_000, - }, - ]); - const context = { io: { info: vi.fn() }, stateAdapter }; - await expect( - repairConversationUsage(context, { - batchSize: 1, - executor: fixture.sql, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 2, - missing: 1, - scanned: 3, - }); - const [metrics] = await fixture.sql.query<{ - durationMs: number; - executionDurationMs: number; - usage: Record; - }>( - ` -SELECT - duration_ms AS "durationMs", - execution_duration_ms AS "executionDurationMs", - usage_json AS usage -FROM junior_conversations -WHERE conversation_id = $1 -`, - [CONVERSATION_ID], - ); - expect(metrics).toEqual({ - durationMs: 9_000, - executionDurationMs: 9_000, - usage: { - inputTokens: 30, - outputTokens: 7, - cachedInputTokens: 3, - cacheCreationTokens: 1, - reasoningTokens: 3, - cost: { - input: 0.005, - output: 0.008, - cacheRead: 0.0003, - cacheWrite: 0.0004, - total: 0.0137, - }, - }, - }); - const [mixed] = await fixture.sql.query<{ usage: unknown }>( - `SELECT usage_json AS usage FROM junior_conversations WHERE conversation_id = $1`, - [mixedConversationId], - ); - expect(mixed?.usage).toEqual({ totalTokens: 8 }); - const [unsafe] = await fixture.sql.query<{ usage: unknown }>( - `SELECT usage_json AS usage FROM junior_conversations WHERE conversation_id = $1`, - [unsafeConversationId], - ); - expect(unsafe?.usage).toEqual({ totalTokens: 777 }); - - await expect( - repairConversationUsage(context, { - batchSize: 1, - executor: fixture.sql, - }), - ).resolves.toEqual({ - existing: 2, - migrated: 0, - missing: 1, - scanned: 3, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("commits completed usage batches before a later batch fails", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - const eventStore = createSqlConversationEventStore(fixture.sql); - const conversationIds = ["local:usage-batch-a", "local:usage-batch-b"]; - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - for (const [index, conversationId] of conversationIds.entries()) { - await sqlStore.recordExecution({ - conversationId, - createdAtMs: 1_000, - execution: { status: "idle", updatedAtMs: 2_000 }, - lastActivityAtMs: 2_000, - metrics: { durationMs: 1_000, usage: { totalTokens: 999 } }, - updatedAtMs: 2_000, - }); - await eventStore.append(conversationId, [ - { - data: { - type: "agent_step", - message: { - role: "assistant", - usage: { input: index + 1, totalTokens: index + 1 }, - } as PiMessage, - }, - createdAtMs: 2_000, - }, - ]); - } - - let queryCount = 0; - const failingExecutor = new Proxy(fixture.sql, { - get(target, key, receiver) { - if (key === "query") { - return async ( - statement: string, - params: readonly unknown[] = [], - ) => { - queryCount += 1; - if (queryCount === 2) { - throw new Error("later usage batch failed"); - } - return target.query(statement, params); - }; - } - const value = Reflect.get(target, key, receiver) as unknown; - return typeof value === "function" ? value.bind(target) : value; - }, - }); - const context = { io: { info: vi.fn() }, stateAdapter }; - await expect( - repairConversationUsage(context, { - batchSize: 1, - executor: failingExecutor, - }), - ).rejects.toThrow("later usage batch failed"); - - const beforeRetry = await fixture.sql.query<{ - conversationId: string; - usage: unknown; - }>( - `SELECT conversation_id AS "conversationId", usage_json AS usage - FROM junior_conversations ORDER BY conversation_id`, - ); - expect(beforeRetry).toEqual([ - { - conversationId: conversationIds[0], - usage: { inputTokens: 1 }, - }, - { conversationId: conversationIds[1], usage: { totalTokens: 999 } }, - ]); - await expect( - repairConversationUsage(context, { - batchSize: 1, - executor: fixture.sql, - }), - ).resolves.toEqual({ - existing: 1, - migrated: 1, - missing: 0, - scanned: 2, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("reports version-guard misses as skipped", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - - try { - let queryCount = 0; - const skippedExecutor = new Proxy(fixture.sql, { - get(target, key, receiver) { - if (key === "query") { - return async () => { - queryCount += 1; - return queryCount === 1 - ? [ - { - changed: false, - conversationId: CONVERSATION_ID, - matched: false, - repairable: true, - }, - ] - : []; - }; - } - const value = Reflect.get(target, key, receiver) as unknown; - return typeof value === "function" ? value.bind(target) : value; - }, - }); - - await expect( - repairConversationUsage( - { io: { info: vi.fn() }, stateAdapter }, - { executor: skippedExecutor }, - ), - ).resolves.toEqual({ - existing: 0, - migrated: 0, - missing: 0, - scanned: 1, - skipped: 1, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("repairs a conversation on rerun after it becomes idle", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - const eventStore = createSqlConversationEventStore(fixture.sql); - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - await sqlStore.recordExecution({ - conversationId: CONVERSATION_ID, - createdAtMs: 1_000, - execution: { - runId: "active-run", - status: "running", - updatedAtMs: 2_000, - }, - lastActivityAtMs: 2_000, - metrics: { - durationMs: 7_777, - usage: { totalTokens: 999, cost: { total: 0.999 } }, - }, - updatedAtMs: 2_000, - }); - await eventStore.append(CONVERSATION_ID, [ - { - data: { - type: "agent_step", - message: { - role: "assistant", - usage: { input: 4, output: 2, totalTokens: 6 }, - } as PiMessage, - }, - createdAtMs: 2_000, - }, - ]); - const context = { io: { info: vi.fn() }, stateAdapter }; - await expect( - repairConversationUsage(context, { executor: fixture.sql }), - ).resolves.toEqual({ - existing: 0, - migrated: 0, - missing: 0, - scanned: 0, - }); - - await fixture.sql.execute( - `UPDATE junior_conversations SET execution_status = 'idle' WHERE conversation_id = $1`, - [CONVERSATION_ID], - ); - await expect( - repairConversationUsage(context, { executor: fixture.sql }), - ).resolves.toEqual({ - existing: 0, - migrated: 1, - missing: 0, - scanned: 1, - }); - const [metrics] = await fixture.sql.query<{ - durationMs: number; - usage: Record; - }>( - `SELECT duration_ms AS "durationMs", usage_json AS usage - FROM junior_conversations WHERE conversation_id = $1`, - [CONVERSATION_ID], - ); - expect(metrics).toEqual({ - durationMs: 7_777, - usage: { inputTokens: 4, outputTokens: 2 }, - }); - } finally { - await fixture.close(); - } - }, 15_000); - - it("seeds active awaiting continuations into conversation work", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await recordAgentTurnSessionSummary({ - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - resumeReason: "timeout", - sessionId: "turn-timeout", - sliceId: 2, - state: "awaiting_resume", - conversationStore: stateOnlyConversationStore, - }); - await persistActiveTurn(CONVERSATION_ID, "turn-timeout"); - - await expect( - migrateRedisConversationState({ - io: { info: () => {} }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 1, - missing: 0, - scanned: 1, - }); - await expect( - stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), - ).resolves.toMatchObject({ - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - execution: { - pendingCount: 0, - pendingMessages: [], - status: "pending", - }, - }); - await expect( - stateAdapter.get(CONVERSATION_ACTIVE_INDEX_KEY), - ).resolves.toEqual([ - { - conversationId: CONVERSATION_ID, - score: expect.any(Number), - }, - ]); - }); - - it("merges legacy pending work when the conversation record already exists", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await requestConversationWork({ - conversationId: CONVERSATION_ID, - conversationStore: stateOnlyConversationStore, - destination: SLACK_DESTINATION, - nowMs: 2_000, - state: stateAdapter, - }); - await stateAdapter.delete(CONVERSATION_BY_ACTIVITY_INDEX_KEY); - await stateAdapter.delete(CONVERSATION_ACTIVE_INDEX_KEY); - await stateAdapter.set( - `junior:conversation-work:state:${CONVERSATION_ID}`, - { - schemaVersion: 1, - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - messages: [inboundMessage("m1")], - needsRun: true, - updatedAtMs: 3_000, - }, - ); - await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); - - await expect( - migrateRedisConversationState({ - io: { info: () => {} }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 1, - migrated: 0, - missing: 0, - scanned: 1, - }); - await expect( - stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), - ).resolves.toBeNull(); - await expect( - stateAdapter.get(`junior:conversation:${CONVERSATION_ID}`), - ).resolves.toMatchObject({ - conversationId: CONVERSATION_ID, - lastActivityAtMs: 2_000, - updatedAtMs: 3_000, - execution: { - inboundMessageIds: ["m1"], - pendingCount: 1, - pendingMessages: [expect.objectContaining({ inboundMessageId: "m1" })], - status: "pending", - updatedAtMs: 3_000, - }, - }); - await expect( - stateAdapter.get("junior:conversation-work:index"), - ).resolves.toBeNull(); - await expect( - stateAdapter.get(CONVERSATION_BY_ACTIVITY_INDEX_KEY), - ).resolves.toEqual([ - { - conversationId: CONVERSATION_ID, - score: 2_000, - }, - ]); - await expect( - stateAdapter.get(CONVERSATION_ACTIVE_INDEX_KEY), - ).resolves.toEqual([ - { - conversationId: CONVERSATION_ID, - score: 3_000, - }, - ]); - }); - - it("does not merge legacy pending work with a different destination", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await requestConversationWork({ - conversationId: CONVERSATION_ID, - conversationStore: stateOnlyConversationStore, - destination: SLACK_DESTINATION, - nowMs: 2_000, - state: stateAdapter, - }); - await stateAdapter.set( - `junior:conversation-work:state:${CONVERSATION_ID}`, - { - schemaVersion: 1, - conversationId: CONVERSATION_ID, - destination: OTHER_SLACK_DESTINATION, - messages: [ - { - ...inboundMessage("m1"), - destination: OTHER_SLACK_DESTINATION, - }, - ], - needsRun: true, - updatedAtMs: 3_000, - }, - ); - await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); - - await expect( - migrateRedisConversationState({ - io: { info: () => {} }, - stateAdapter, - }), - ).rejects.toThrow( - `Legacy conversation work destination does not match conversation ${CONVERSATION_ID}`, - ); - await expect( - stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), - ).resolves.toEqual(expect.objectContaining({ needsRun: true })); - }); - - it("rejects legacy pending work with a different message destination", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await stateAdapter.set( - `junior:conversation-work:state:${CONVERSATION_ID}`, - { - schemaVersion: 1, - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - messages: [ - { - ...inboundMessage("m1"), - destination: OTHER_SLACK_DESTINATION, - }, - ], - needsRun: true, - updatedAtMs: 3_000, - }, - ); - await stateAdapter.set("junior:conversation-work:index", [CONVERSATION_ID]); - - await expect( - migrateRedisConversationState({ - io: { info: () => {} }, - stateAdapter, - }), - ).rejects.toThrow( - `Legacy conversation work state is invalid for ${CONVERSATION_ID}`, - ); - await expect( - stateAdapter.get(`junior:conversation-work:state:${CONVERSATION_ID}`), - ).resolves.toEqual(expect.objectContaining({ needsRun: true })); - }); - - it("ignores malformed legacy conversation work indexes", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await stateAdapter.set("junior:conversation-work:index", { - conversationId: CONVERSATION_ID, - }); - - await expect( - migrateRedisConversationState({ - io: { info: () => {} }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 0, - missing: 0, - scanned: 0, - }); - }); - - it("backfills retained conversation record into SQL when configured", async () => { - const stateAdapter = getStateAdapter(); - await stateAdapter.connect(); - await requestConversationWork({ - conversationId: CONVERSATION_ID, - conversationStore: stateOnlyConversationStore, - destination: SLACK_DESTINATION, - nowMs: 2_000, - state: stateAdapter, - }); - const fixture = await createLocalJuniorSqlFixture(); - const sqlStore = createSqlStore(fixture.sql); - - try { - await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); - const context = { - io: { info: () => {} }, - stateAdapter, - }; - const results = [ - await migrateRedisConversationState(context), - await migrateConversationsToSql(context, { target: sqlStore }), - ]; - - expect(results).toEqual([ - { - existing: 0, - migrated: 0, - missing: 0, - scanned: 0, - }, - { - existing: 0, - migrated: 1, - missing: 0, - scanned: 1, - }, - ]); - const sqlConversation = await sqlStore.get({ - conversationId: CONVERSATION_ID, - }); - expect(sqlConversation).toMatchObject({ - conversationId: CONVERSATION_ID, - destination: SLACK_DESTINATION, - execution: { - status: "pending", - }, - }); - expect(sqlConversation?.execution).not.toHaveProperty("pendingCount"); - } finally { - await fixture.close(); - } - }, 15_000); }); diff --git a/packages/junior/tests/component/memory-plugin-storage.test.ts b/packages/junior/tests/component/memory-plugin-storage.test.ts index 02d219d6d..fe39fd488 100644 --- a/packages/junior/tests/component/memory-plugin-storage.test.ts +++ b/packages/junior/tests/component/memory-plugin-storage.test.ts @@ -1,7 +1,5 @@ import path from "node:path"; import { readdirSync } from "node:fs"; -import { createMemoryState } from "@chat-adapter/state-memory"; -import { resolveMigrations } from "@sentry/junior-migrations"; import { afterAll, describe, expect, it, vi } from "vitest"; import { createMemoryPlugin, @@ -10,13 +8,15 @@ import { } from "@sentry/junior-memory"; import { createSlackSource, + defineJuniorPlugin, PluginToolInputError, } from "@sentry/junior-plugin-api"; import { defineJuniorPlugins } from "@/plugins"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; -import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; +import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { readMigrationFiles } from "drizzle-orm/migrator"; import { closeDb } from "@/chat/db"; -import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; +import { runUpgrade } from "@/cli/upgrade"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const NEON = vi.hoisted(() => ({ @@ -86,7 +86,7 @@ function memoryMigrationFiles(): string[] { async function migrateMemorySchema( fixture: Awaited>, ) { - await bootstrapPluginSchemas(fixture.sql, [ + await migratePluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -97,7 +97,9 @@ async function migrateMemorySchema( describe("memory plugin host wiring", () => { it("adopts exact legacy migration hashes without replaying them", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrations = await resolveMigrations(memoryMigrationsDir()); + const migrations = readMigrationFiles({ + migrationsFolder: memoryMigrationsDir(), + }); const migrationFiles = memoryMigrationFiles(); expect(migrationFiles).toHaveLength(migrations.length); @@ -128,7 +130,7 @@ CREATE TABLE junior_schema_migrations ( } await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -146,8 +148,9 @@ CREATE TABLE junior_schema_migrations ( it("does not adopt an unknown memory legacy checksum", async () => { const fixture = await createLocalJuniorSqlFixture(); - const migrationCount = (await resolveMigrations(memoryMigrationsDir())) - .length; + const migrationCount = readMigrationFiles({ + migrationsFolder: memoryMigrationsDir(), + }).length; const [baselineFile] = memoryMigrationFiles(); expect(baselineFile).toBeDefined(); @@ -165,7 +168,7 @@ CREATE TABLE junior_schema_migrations ( ); await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -181,40 +184,51 @@ CREATE TABLE junior_schema_migrations ( } }); - it("applies packaged migrations through plugin discovery", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); + it("reports core and nonempty plugin migration journals", async () => { const fixture = await createLocalJuniorSqlFixture(); NEON.sql = fixture.sql; try { - const migrationCount = (await resolveMigrations(memoryMigrationsDir())) - .length; - await expect( - migratePluginJournals({ - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([createMemoryPlugin()]), - stateAdapter, + const coreMigrationCount = readMigrationFiles({ + migrationsFolder: path.resolve(process.cwd(), "migrations"), + }).length; + const memoryMigrationCount = readMigrationFiles({ + migrationsFolder: memoryMigrationsDir(), + }).length; + const lines: string[] = []; + const pluginSet = defineJuniorPlugins([ + createMemoryPlugin(), + defineJuniorPlugin({ + manifest: { + description: "Plugin without SQL migrations", + displayName: "Empty", + name: "empty", + }, }), - ).resolves.toEqual({ - existing: 0, - migrated: migrationCount, - missing: 0, - scanned: migrationCount, - }); + ]); - await expect( - fixture.sql.query<{ table_name: string }>( - ` -SELECT table_name -FROM information_schema.tables -WHERE table_name = 'junior_memory_memories' -`, - ), - ).resolves.toEqual([{ table_name: "junior_memory_memories" }]); + await runUpgrade({ info: (line) => lines.push(line) }, { pluginSet }); + expect(lines).toEqual([ + "Running Junior upgrade migrations...", + "Running migration core-migrations...", + `Finished migration core-migrations: scanned=${coreMigrationCount} migrated=${coreMigrationCount} existing=0 missing=0 skipped=0`, + "Running migration plugin-migrations...", + `Finished migration plugin-migrations: scanned=${memoryMigrationCount} migrated=${memoryMigrationCount} existing=0 missing=0`, + "Junior upgrade complete.", + ]); + + lines.length = 0; + await runUpgrade({ info: (line) => lines.push(line) }, { pluginSet }); + expect(lines).toEqual([ + "Running Junior upgrade migrations...", + "Running migration core-migrations...", + `Finished migration core-migrations: scanned=${coreMigrationCount} migrated=0 existing=${coreMigrationCount} missing=0 skipped=0`, + "Running migration plugin-migrations...", + `Finished migration plugin-migrations: scanned=${memoryMigrationCount} migrated=0 existing=${memoryMigrationCount} missing=0`, + "Junior upgrade complete.", + ]); } finally { NEON.sql = undefined; - await stateAdapter.disconnect(); await fixture.close(); } }, 15_000); diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index 14ef04386..4722768d4 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -1,61 +1,16 @@ import path from "node:path"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { createMemoryState } from "@chat-adapter/state-memory"; -import { - resolveMigrations, - type MigrationContextV1, -} from "@sentry/junior-migrations"; -import { defineJuniorPlugin } from "@sentry/junior-plugin-api"; -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { readMigrationFiles } from "drizzle-orm/migrator"; +import { describe, expect, it } from "vitest"; import { createSchedulerSqlStore, - schedulerPlugin, type SchedulerDb, type ScheduledTask, } from "@sentry/junior-scheduler"; -import schedulerStateToSqlMigration from "../../../junior-scheduler/migrations/0002_scheduler_state_to_sql"; -import { createSchedulerStore } from "../../../junior-scheduler/src/store"; -import { defineJuniorPlugins } from "@/plugins"; -import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; -import { - bootstrapPluginSchemas, - migratePluginSchemas, -} from "@/chat/plugins/migrations"; -import { createPluginState } from "@/chat/plugins/state"; -import { disconnectStateAdapter } from "@/chat/state/adapter"; -import { migratePluginJournals } from "@/cli/upgrade/migrations/plugin-journal"; +import { migratePluginSchemas } from "@/chat/plugins/migrations"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; -const NEON = vi.hoisted(() => ({ - originalDatabaseUrl: process.env.DATABASE_URL, - sql: undefined as - | Awaited>["sql"] - | undefined, -})); - -vi.hoisted(() => { - process.env.DATABASE_URL = "postgres://configured.example.test/neon"; - process.env.JUNIOR_STATE_ADAPTER = "memory"; -}); - -vi.mock("@/db/executor", () => ({ - createJuniorSqlExecutor: vi.fn(() => { - if (!NEON.sql) { - throw new Error("Missing test SQL executor"); - } - return { - db: NEON.sql.db.bind(NEON.sql), - execute: NEON.sql.execute.bind(NEON.sql), - query: NEON.sql.query.bind(NEON.sql), - transaction: NEON.sql.transaction.bind(NEON.sql), - withLock: NEON.sql.withLock.bind(NEON.sql), - withMigrationLock: NEON.sql.withMigrationLock.bind(NEON.sql), - close: async () => {}, - }; - }), -})); - const TEST_RUN_AT_MS = Date.parse("2026-05-26T12:00:00.000Z"); const TEST_NOW_MS = Date.parse("2026-05-26T12:05:00.000Z"); const LEGACY_SCHEDULER_MIGRATION_CHECKSUM = @@ -72,7 +27,7 @@ function memoryMigrationsDir(): string { async function migrateSchedulerSchema( fixture: Awaited>, ) { - await bootstrapPluginSchemas(fixture.sql, [ + await migratePluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -80,21 +35,6 @@ async function migrateSchedulerSchema( ]); } -async function runSchedulerStateMigration(args: { - fixture: Awaited>; - stateAdapter: ReturnType; -}) { - return await schedulerStateToSqlMigration.up({ - database: args.fixture.sql, - log: () => {}, - progress: { - load: async () => undefined, - save: async () => {}, - }, - state: createPluginMigrationStateV1("scheduler", args.stateAdapter), - }); -} - function createTask(overrides: Partial = {}): ScheduledTask { return { id: "sched_sql_1", @@ -122,140 +62,6 @@ function createTask(overrides: Partial = {}): ScheduledTask { } describe("scheduler SQL plugin storage", () => { - afterAll(() => { - if (NEON.originalDatabaseUrl === undefined) { - delete process.env.DATABASE_URL; - } else { - process.env.DATABASE_URL = NEON.originalDatabaseUrl; - } - }); - - afterEach(async () => { - NEON.sql = undefined; - await disconnectStateAdapter(); - }); - - it("scopes migration state while preserving plugin legacy keys", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const state = createPluginMigrationStateV1("scheduler", stateAdapter); - - try { - await stateAdapter.set("global-key", "global-value"); - await stateAdapter.set("junior:memory:secret", "memory-value"); - await state.set("global-key", "scheduler-value"); - await state.set("junior:scheduler:legacy", "legacy-value"); - - await expect(state.get("global-key")).resolves.toBe("scheduler-value"); - await expect(stateAdapter.get("global-key")).resolves.toBe( - "global-value", - ); - await expect(state.get("junior:memory:secret")).resolves.toBeUndefined(); - await expect(state.get("junior:scheduler:legacy")).resolves.toBe( - "legacy-value", - ); - await expect(stateAdapter.get("junior:scheduler:legacy")).resolves.toBe( - "legacy-value", - ); - - await stateAdapter.appendToList("global-list", "global-item"); - await state.appendToList("global-list", "scheduler-item"); - await expect(state.getList("global-list")).resolves.toEqual([ - "scheduler-item", - ]); - await expect(stateAdapter.getList("global-list")).resolves.toEqual([ - "global-item", - ]); - - const scopedLock = await state.acquireLock("migration-lock", 1_000); - const globalLock = await stateAdapter.acquireLock( - "migration-lock", - 1_000, - ); - expect(scopedLock).not.toBeNull(); - expect(globalLock).not.toBeNull(); - if (scopedLock) { - await state.releaseLock(scopedLock); - } - if (globalLock) { - await stateAdapter.releaseLock(globalLock); - } - } finally { - await stateAdapter.disconnect(); - } - }); - - it("passes scoped state through the plugin migration host", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - const migrationsDir = mkdtempSync( - path.join(tmpdir(), "junior-plugin-state-migration-"), - ); - mkdirSync(path.join(migrationsDir, "meta")); - writeFileSync( - path.join(migrationsDir, "meta", "_journal.json"), - JSON.stringify({ - version: "7", - dialect: "postgresql", - entries: [ - { - idx: 0, - version: "7", - when: 2_026_071_600_003, - tag: "0000_state_scope", - breakpoints: true, - }, - ], - }), - ); - writeFileSync( - path.join(migrationsDir, "0000_state_scope.ts"), - "export default { apiVersion: 1, async up() {} };\n", - ); - const observed: unknown[] = []; - - try { - await stateAdapter.set("global-key", "global-value"); - await stateAdapter.set("junior:other:secret", "other-value"); - await migratePluginSchemas( - fixture.sql, - [{ dir: migrationsDir, pluginName: "isolation" }], - { - mode: "all", - stateAdapter, - loadTypeScript: async () => ({ - default: { - apiVersion: 1, - async up(context: MigrationContextV1) { - observed.push( - await context.state.get("global-key"), - await context.state.get("junior:other:secret"), - ); - await context.state.set("global-key", "scoped-value"); - return { scoped: true }; - }, - }, - }), - }, - ); - - expect(observed).toEqual([undefined, undefined]); - await expect(stateAdapter.get("global-key")).resolves.toBe( - "global-value", - ); - await expect( - createPluginMigrationStateV1("isolation", stateAdapter).get( - "global-key", - ), - ).resolves.toBe("scoped-value"); - } finally { - rmSync(migrationsDir, { force: true, recursive: true }); - await stateAdapter.disconnect(); - await fixture.close(); - } - }); - it("adopts deployed scheduler schema state into its Drizzle journal", async () => { const fixture = await createLocalJuniorSqlFixture(); @@ -316,19 +122,16 @@ INSERT INTO junior_scheduler_tasks ( ); await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", }, ]), - ).resolves.toEqual({ - existing: 1, - migrated: 1, - scanned: 3, - skipped: 1, + ).resolves.toEqual({ existing: 1, migrated: 1, scanned: 2 }); + const migrations = readMigrationFiles({ + migrationsFolder: schedulerMigrationsDir(), }); - const migrations = await resolveMigrations(schedulerMigrationsDir()); const migrationRows = await fixture.sql.query<{ createdAt: string; hash: string; @@ -338,12 +141,10 @@ FROM drizzle.${migrationTable!.tablename} ORDER BY created_at `); expect(migrationRows).toEqual( - migrations - .filter((migration) => migration.kind === "sql") - .map((migration) => ({ - createdAt: String(migration.when), - hash: migration.hash, - })), + migrations.map((migration) => ({ + createdAt: String(migration.folderMillis), + hash: migration.hash, + })), ); const [migratedTask] = await fixture.sql.query<{ record: unknown }>( `SELECT record FROM junior_scheduler_tasks WHERE id = $1`, @@ -354,18 +155,13 @@ ORDER BY created_at }); expect(migratedTask?.record).not.toHaveProperty("credentialSubject"); await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", }, ]), - ).resolves.toEqual({ - existing: 2, - migrated: 0, - scanned: 3, - skipped: 1, - }); + ).resolves.toEqual({ existing: 2, migrated: 0, scanned: 2 }); await expect( fixture.sql.query<{ createdAt: string; @@ -387,24 +183,18 @@ ORDER BY created_at { dir: schedulerMigrationsDir(), pluginName: "scheduler" }, { dir: memoryMigrationsDir(), pluginName: "memory" }, ]; - const migrations = ( - await Promise.all( - roots.map(async (root) => await resolveMigrations(root.dir)), - ) - ).flat(); - const migrationCount = migrations.length; - const sqlMigrationCount = migrations.filter( - (migration) => migration.kind === "sql", - ).length; + const migrationCount = roots.reduce( + (count, root) => + count + readMigrationFiles({ migrationsFolder: root.dir }).length, + 0, + ); + try { - await expect(bootstrapPluginSchemas(fixture.sql, roots)).resolves.toEqual( - { - existing: 0, - migrated: sqlMigrationCount, - scanned: migrationCount, - skipped: 1, - }, - ); + await expect(migratePluginSchemas(fixture.sql, roots)).resolves.toEqual({ + existing: 0, + migrated: migrationCount, + scanned: migrationCount, + }); const migrationTables = await fixture.sql.query<{ tablename: string }>(` SELECT tablename FROM pg_tables @@ -414,12 +204,11 @@ ORDER BY tablename `); expect(migrationTables).toHaveLength(2); await expect( - bootstrapPluginSchemas(fixture.sql, [...roots].reverse()), + migratePluginSchemas(fixture.sql, [...roots].reverse()), ).resolves.toEqual({ - existing: sqlMigrationCount, + existing: migrationCount, migrated: 0, scanned: migrationCount, - skipped: 1, }); } finally { await fixture.close(); @@ -439,12 +228,12 @@ ORDER BY tablename try { await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: missingJournal, pluginName: "missing" }, ]), ).rejects.toThrow("Can't find meta/_journal.json file"); await expect( - bootstrapPluginSchemas(fixture.sql, [ + migratePluginSchemas(fixture.sql, [ { dir: invalidJournal, pluginName: "invalid" }, ]), ).rejects.toThrow("Expected property name"); @@ -658,284 +447,6 @@ ORDER BY tablename } }, 15_000); - it("migrates existing scheduler plugin state into SQL idempotently", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - - try { - await migrateSchedulerSchema(fixture); - const db = fixture.sql.db() as unknown as SchedulerDb; - const stateStore = createSchedulerStore( - createPluginState("scheduler", stateAdapter), - ); - const task = createTask({ id: "sched_state_sql" }); - await stateStore.saveTask(task); - const run = await stateStore.claimDueRun({ nowMs: TEST_NOW_MS }); - expect(run).toBeDefined(); - - await expect( - runSchedulerStateMigration({ fixture, stateAdapter }), - ).resolves.toEqual({ - existing: 0, - migrated: 2, - missing: 0, - scanned: 2, - }); - await expect( - runSchedulerStateMigration({ fixture, stateAdapter }), - ).resolves.toEqual({ - existing: 2, - migrated: 0, - missing: 0, - scanned: 2, - }); - - const { credentialMode: _credentialMode, ...legacySqlTask } = task; - await fixture.sql.execute( - "UPDATE junior_scheduler_tasks SET record = $2::jsonb WHERE id = $1", - [ - task.id, - JSON.stringify({ - ...legacySqlTask, - credentialSubject: { - allowedWhen: "private-direct-conversation", - type: "user", - userId: "U123", - }, - }), - ], - ); - await expect( - runSchedulerStateMigration({ fixture, stateAdapter }), - ).resolves.toEqual({ - existing: 1, - migrated: 1, - missing: 0, - scanned: 2, - }); - - const sqlStore = createSchedulerSqlStore(db); - await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ - id: task.id, - }); - await expect(sqlStore.getRun(run!.id)).resolves.toMatchObject({ - id: run!.id, - taskId: task.id, - }); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }, 15_000); - - it("skips malformed scheduler state records during SQL storage migration", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - - try { - await migrateSchedulerSchema(fixture); - const db = fixture.sql.db() as unknown as SchedulerDb; - const state = createPluginState("scheduler", stateAdapter); - const task = createTask({ id: "sched_state_sql_valid_after_bad" }); - const { credentialMode: _credentialMode, ...legacyTask } = task; - const badRunId = `${task.id}:${TEST_RUN_AT_MS}`; - await state.set( - "junior:scheduler:tasks", - ["sched_state_sql_bad", "sched_state_sql_missing_required", task.id], - 5 * 60 * 1000, - ); - await state.set( - `junior:scheduler:task:${task.id}`, - { - ...legacyTask, - credentialSubject: { - type: "user", - userId: "U123", - allowedWhen: "private-direct-conversation", - }, - }, - 5 * 60 * 1000, - ); - await state.set( - "junior:scheduler:task:sched_state_sql_bad", - { - ...task, - id: "sched_state_sql_bad", - task: { text: 123 }, - }, - 5 * 60 * 1000, - ); - const { schedule: _schedule, ...missingSchedule } = task; - await state.set( - "junior:scheduler:task:sched_state_sql_missing_required", - { ...missingSchedule, id: "sched_state_sql_missing_required" }, - 5 * 60 * 1000, - ); - await state.set( - `junior:scheduler:active:${task.id}`, - { - claimedAtMs: TEST_NOW_MS, - runId: badRunId, - scheduledForMs: TEST_RUN_AT_MS, - }, - 5 * 60 * 1000, - ); - await state.set( - `junior:scheduler:run:${badRunId}`, - { - id: badRunId, - scheduledForMs: TEST_RUN_AT_MS, - status: "pending", - taskId: task.id, - }, - 5 * 60 * 1000, - ); - - await expect( - runSchedulerStateMigration({ fixture, stateAdapter }), - ).resolves.toEqual({ - existing: 0, - migrated: 1, - missing: 2, - scanned: 3, - }); - - const sqlStore = createSchedulerSqlStore(db); - await expect(sqlStore.getTask(task.id)).resolves.toMatchObject({ - credentialMode: "system", - id: task.id, - }); - await expect(sqlStore.getTask("sched_state_sql_bad")).resolves.toBe( - undefined, - ); - await expect( - sqlStore.getTask("sched_state_sql_missing_required"), - ).resolves.toBe(undefined); - await expect(sqlStore.getRun(badRunId)).resolves.toBe(undefined); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }, 15_000); - - it("does not apply scheduler SQL migrations from package-only config", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - NEON.sql = fixture.sql; - - try { - await expect( - migratePluginJournals({ - io: { info: () => {} }, - pluginCatalogConfig: { packages: ["@sentry/junior-scheduler"] }, - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 0, - missing: 0, - scanned: 0, - }); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }); - - it("applies scheduler SQL migrations from registration-only config", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - NEON.sql = fixture.sql; - - try { - await expect( - migratePluginJournals({ - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([schedulerPlugin()]), - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 3, - missing: 0, - scanned: 3, - }); - - const db = fixture.sql.db() as unknown as SchedulerDb; - const store = createSchedulerSqlStore(db); - const task = createTask({ id: "sched_schema_registration_config" }); - await store.saveTask(task); - await expect(store.getTask(task.id)).resolves.toMatchObject({ - id: task.id, - }); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }); - - it("runs a migration-only plugin registration", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - NEON.sql = fixture.sql; - const scheduler = schedulerPlugin(); - const migrationOnly = defineJuniorPlugin({ - manifest: scheduler.manifest, - packageName: scheduler.packageName, - }); - - try { - await expect( - migratePluginJournals({ - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([migrationOnly]), - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 3, - missing: 0, - scanned: 3, - }); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }); - - it("does not duplicate scheduler SQL migrations for explicit registrations", async () => { - const stateAdapter = createMemoryState(); - await stateAdapter.connect(); - const fixture = await createLocalJuniorSqlFixture(); - NEON.sql = fixture.sql; - - try { - await expect( - migratePluginJournals({ - io: { info: () => {} }, - pluginSet: defineJuniorPlugins([ - "@sentry/junior-scheduler", - schedulerPlugin(), - ]), - stateAdapter, - }), - ).resolves.toEqual({ - existing: 0, - migrated: 3, - missing: 0, - scanned: 3, - }); - } finally { - await stateAdapter.disconnect(); - await fixture.close(); - } - }); - it("skips malformed SQL records while claiming due runs", async () => { const fixture = await createLocalJuniorSqlFixture(); diff --git a/packages/junior/tests/integration/conversation-sql.test.ts b/packages/junior/tests/integration/conversation-sql.test.ts index b77f56fb0..8e168750e 100644 --- a/packages/junior/tests/integration/conversation-sql.test.ts +++ b/packages/junior/tests/integration/conversation-sql.test.ts @@ -1,7 +1,7 @@ import { fileURLToPath } from "node:url"; import { getTableColumns, getTableName } from "drizzle-orm"; import { readMigrationFiles } from "drizzle-orm/migrator"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; import { createPostgresJuniorSqlExecutor } from "@/db/postgres"; @@ -266,21 +266,19 @@ VALUES ('host-migration', 9999999999999) const fixture = await createLocalJuniorSqlFixture(); try { - const migrationLock = vi.spyOn(fixture.sql, "withMigrationLock"); await expect(migrateSchema(fixture.sql)).resolves.toEqual({ existing: 0, migrated: coreMigrations.length, scanned: coreMigrations.length, + skipped: 0, }); - expect(migrationLock).toHaveBeenCalledOnce(); - migrationLock.mockClear(); await expect(migrateSchema(fixture.sql)).resolves.toEqual({ existing: coreMigrations.length, migrated: 0, scanned: coreMigrations.length, + skipped: 0, }); - expect(migrationLock).not.toHaveBeenCalled(); const conversation = buildJuniorSqlConversation({ conversationId: "slack:C123:1718123456.000000", From b76652a4458fc1d45efef9fbc6e9ec1157be6e21 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 22 Jul 2026 22:33:35 -0700 Subject: [PATCH 10/15] fix(migrations): Defer state initialization Require callers to select full migration or schema-bootstrap mode explicitly. Resolve the state context only when a pending TypeScript migration runs, so SQL-only upgrades do not depend on Redis. Align the migrations package with the current lockstep release version. --- packages/junior-migrations/package.json | 2 +- packages/junior-migrations/src/runner.ts | 7 +++- .../junior-migrations/tests/runner.test.ts | 2 +- .../src/chat/conversations/sql/migrations.ts | 42 ++++++++++--------- .../junior/src/chat/plugins/migrations.ts | 8 ++-- packages/junior/src/cli/upgrade.ts | 11 +++-- .../cli/upgrade/migrations/core-journal.ts | 3 +- .../cli/upgrade/migrations/plugin-journal.ts | 3 +- packages/junior/src/cli/upgrade/types.ts | 8 +++- .../component/conversation-sql-store.test.ts | 36 ++++++++-------- .../conversation-storage-sql.test.ts | 40 +++++++++--------- .../component/memory-plugin-storage.test.ts | 32 +++++++++++--- .../component/scheduler-sql-plugin.test.ts | 26 ++++++------ .../api/conversations/list.test.ts | 4 +- .../integration/conversation-sql.test.ts | 23 ++++++---- 15 files changed, 147 insertions(+), 100 deletions(-) diff --git a/packages/junior-migrations/package.json b/packages/junior-migrations/package.json index 36af03dbf..4cfc55d1c 100644 --- a/packages/junior-migrations/package.json +++ b/packages/junior-migrations/package.json @@ -1,6 +1,6 @@ { "name": "@sentry/junior-migrations", - "version": "0.104.2", + "version": "0.108.0", "private": false, "publishConfig": { "access": "public" diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index dd9ae0a83..9a3c88e59 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -72,7 +72,7 @@ interface RunAllMigrationJournalOptions extends RunMigrationJournalBaseOptions { createContext: (args: { migration: ResolvedMigration; progress: MigrationContextV1["progress"]; - }) => MigrationContextV1; + }) => MigrationContextV1 | Promise; loadTypeScript: TypeScriptMigrationLoader; mode?: "all"; } @@ -270,7 +270,10 @@ async function runTypeScriptMigration(args: { }, }; try { - const context = args.createContext({ migration: args.migration, progress }); + const context = await args.createContext({ + migration: args.migration, + progress, + }); const currentSource = await readFile(args.migration.path, "utf8"); const currentHash = createHash("sha256") .update(currentSource) diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index 7c1baa0cc..04ac6dcfc 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -167,7 +167,7 @@ describe("runMigrationJournal", () => { migrationsFolder: folder, migrationsTable: "__drizzle_test", loadTypeScript: async () => ({ default: migration }), - createContext: ({ progress }) => ({ + createContext: async ({ progress }) => ({ log: () => {}, progress, database: executor, diff --git a/packages/junior/src/chat/conversations/sql/migrations.ts b/packages/junior/src/chat/conversations/sql/migrations.ts index f63cc7244..ffcc1f1a5 100644 --- a/packages/junior/src/chat/conversations/sql/migrations.ts +++ b/packages/junior/src/chat/conversations/sql/migrations.ts @@ -82,11 +82,13 @@ function assertSupportedMigrationState(state: CoreMigrationState): void { export type MigrateSchemaOptions = | { mode: "schema-bootstrap" } | { + getStateContext: () => Promise<{ + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; + }>; loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; mode: "all"; - redisStateAdapter?: RedisStateAdapter; - stateAdapter: StateAdapter; }; export { schema }; @@ -94,7 +96,7 @@ export { schema }; /** Apply all migrations, or bootstrap an empty test database to the latest schema. */ export async function migrateSchema( executor: JuniorSqlMigrationExecutor, - options: MigrateSchemaOptions = { mode: "schema-bootstrap" }, + options: MigrateSchemaOptions, ): Promise { const migrationsFolder = migrationFolder(); const runAll = options.mode === "all"; @@ -114,22 +116,24 @@ export async function migrateSchema( } return await runMigrationJournal({ ...baseOptions, - createContext: ({ progress }): MigrationContextV1 => ({ - database: executor, - log: options.log ?? (() => {}), - progress, - ...(options.redisStateAdapter - ? { - redis: { - sendCommand: async (args: readonly string[]) => - await options - .redisStateAdapter!.getClient() - .sendCommand([...args]), - }, - } - : {}), - state: options.stateAdapter, - }), + createContext: async ({ progress }): Promise => { + const { redisStateAdapter, stateAdapter } = + await options.getStateContext(); + return { + database: executor, + log: options.log ?? (() => {}), + progress, + ...(redisStateAdapter + ? { + redis: { + sendCommand: async (args: readonly string[]) => + await redisStateAdapter.getClient().sendCommand([...args]), + }, + } + : {}), + state: stateAdapter, + }; + }, loadTypeScript: options.loadTypeScript, mode: "all", }); diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index 25c0b61ff..4f0f6d5d1 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -26,10 +26,10 @@ type PluginMigrationResult = { type PluginMigrationOptions = | { mode: "schema-bootstrap" } | { + getStateAdapter: () => Promise; loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; mode: "all"; - stateAdapter: StateAdapter; }; const LEGACY_SCHEDULER_BASELINE_HASH = @@ -135,7 +135,7 @@ CREATE TABLE IF NOT EXISTS drizzle.${args.table} ( export async function migratePluginSchemas( executor: JuniorSqlMigrationExecutor, roots: readonly PluginMigrationRoot[], - options: PluginMigrationOptions = { mode: "schema-bootstrap" }, + options: PluginMigrationOptions, ): Promise { const result: PluginMigrationResult = { existing: 0, @@ -165,13 +165,13 @@ export async function migratePluginSchemas( const pluginResult = runAll ? await runMigrationJournal({ ...baseOptions, - createContext: ({ progress }): MigrationContextV1 => ({ + createContext: async ({ progress }): Promise => ({ database: executor, log: options.log ?? (() => {}), progress, state: createPluginMigrationStateV1( root.pluginName, - options.stateAdapter, + await options.getStateAdapter(), ), }), loadTypeScript: options.loadTypeScript, diff --git a/packages/junior/src/cli/upgrade.ts b/packages/junior/src/cli/upgrade.ts index 3b4884ac2..a69ed5348 100644 --- a/packages/junior/src/cli/upgrade.ts +++ b/packages/junior/src/cli/upgrade.ts @@ -9,6 +9,7 @@ import { migratePluginJournals } from "./upgrade/migrations/plugin-journal"; import type { MigrationContext, MigrationResult, + MigrationStateContext, UpgradeIo, } from "./upgrade/types"; import { type JuniorPluginSet } from "@/plugins"; @@ -90,19 +91,21 @@ export async function runUpgrade( io: UpgradeIo = DEFAULT_IO, options: { pluginSet?: JuniorPluginSet | null } = {}, ): Promise { + let stateContext: Promise | undefined; + const getStateContext = (): Promise => { + stateContext ??= getConnectedStateContext(); + return stateContext; + }; try { - const { redisStateAdapter, stateAdapter } = - await getConnectedStateContext(); const pluginSet = options.pluginSet === undefined ? await resolveUpgradePluginSet() : (options.pluginSet ?? undefined); io.info("Running Junior upgrade migrations..."); await runUpgradeMigrations({ + getStateContext, io, pluginSet, - redisStateAdapter, - stateAdapter, }); io.info("Junior upgrade complete."); } finally { diff --git a/packages/junior/src/cli/upgrade/migrations/core-journal.ts b/packages/junior/src/cli/upgrade/migrations/core-journal.ts index 7951c77d8..e682a0757 100644 --- a/packages/junior/src/cli/upgrade/migrations/core-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/core-journal.ts @@ -17,12 +17,11 @@ export async function migrateCoreJournal( }); try { const result = await migrateSchema(executor, { + getStateContext: context.getStateContext, loadTypeScript: async (path) => await migrationLoader.import>(path), log: context.io.info, mode: "all", - redisStateAdapter: context.redisStateAdapter, - stateAdapter: context.stateAdapter, }); return { existing: result.existing, diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts index c5a7e988b..ea1e8587f 100644 --- a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -24,11 +24,12 @@ export async function migratePluginJournals( executor, pluginCatalogRuntime.getMigrationRoots(), { + getStateAdapter: async () => + (await context.getStateContext()).stateAdapter, loadTypeScript: async (path) => await migrationLoader.import>(path), log: context.io.info, mode: "all", - stateAdapter: context.stateAdapter, }, ); return { diff --git a/packages/junior/src/cli/upgrade/types.ts b/packages/junior/src/cli/upgrade/types.ts index 913af8b3b..078edb4e8 100644 --- a/packages/junior/src/cli/upgrade/types.ts +++ b/packages/junior/src/cli/upgrade/types.ts @@ -7,12 +7,16 @@ export interface UpgradeIo { info: (line: string) => void; } +export interface MigrationStateContext { + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; +} + export interface MigrationContext { + getStateContext: () => Promise; io: UpgradeIo; pluginCatalogConfig?: PluginCatalogConfig; pluginSet?: JuniorPluginSet; - redisStateAdapter?: RedisStateAdapter; - stateAdapter: StateAdapter; } export type MigrationResult = { diff --git a/packages/junior/tests/component/conversation-sql-store.test.ts b/packages/junior/tests/component/conversation-sql-store.test.ts index f1caaa4b2..f1550c592 100644 --- a/packages/junior/tests/component/conversation-sql-store.test.ts +++ b/packages/junior/tests/component/conversation-sql-store.test.ts @@ -77,7 +77,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -210,7 +210,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const identity = await upsertIdentity( fixture.sql, @@ -319,7 +319,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -355,7 +355,7 @@ describe("conversation SQL store", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql .db() @@ -428,7 +428,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -475,7 +475,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const destination = inboundMessage("visibility").destination; // Slack reports this C-prefixed channel private (channel_type: group). @@ -542,7 +542,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); // A write without a live source signal fails closed to private even // though the channel id is C-prefixed. @@ -572,7 +572,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await fixture.sql.execute( ` INSERT INTO junior_conversations ( @@ -632,7 +632,7 @@ INSERT INTO junior_conversations ( try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, @@ -685,7 +685,7 @@ INSERT INTO junior_conversations ( try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -762,7 +762,7 @@ WHERE conversation_id = $1 try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, @@ -812,7 +812,7 @@ WHERE conversation_id = $1 try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: CONVERSATION_ID, @@ -848,7 +848,7 @@ WHERE conversation_id = $1 try { await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -882,7 +882,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 302_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -917,7 +917,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 2_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -955,7 +955,7 @@ WHERE conversation_id = $1 vi.useFakeTimers({ now: 600_000 }); await disconnectStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordExecution({ conversationId: CONVERSATION_ID, createdAtMs: 1_000, @@ -1009,7 +1009,7 @@ WHERE conversation_id = $1 await disconnectStateAdapter(); const state = getStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await appendInboundMessage({ message: inboundMessage("check-in"), conversationStore: store, @@ -1062,7 +1062,7 @@ WHERE conversation_id = $1 await disconnectStateAdapter(); const state = getStateAdapter(); const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await appendInboundMessage({ message: inboundMessage("drain-sql"), conversationStore: store, diff --git a/packages/junior/tests/component/conversation-storage-sql.test.ts b/packages/junior/tests/component/conversation-storage-sql.test.ts index a96b683ae..23462edb8 100644 --- a/packages/junior/tests/component/conversation-storage-sql.test.ts +++ b/packages/junior/tests/component/conversation-storage-sql.test.ts @@ -309,8 +309,8 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const [applied] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", @@ -325,7 +325,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -379,7 +379,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const store = createSqlConversationEventStore(fixture.sql); const firstEvent = { data: { type: "agent_step" as const, message: userMessage("first") }, @@ -452,7 +452,7 @@ describe("SQL conversation storage", () => { const store = createSqlConversationEventStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.append(CONVERSATION_ID, [ { idempotencyKey: "event:repeated", @@ -493,7 +493,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); const lifecycle = new ConversationTurnLifecycleService(store); @@ -537,7 +537,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -570,7 +570,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -610,7 +610,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -679,7 +679,7 @@ describe("SQL conversation storage", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ @@ -719,7 +719,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -770,7 +770,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); const conversation = coerceThreadConversationState({}); @@ -842,7 +842,7 @@ WHERE conversation_id = $1 AND seq = 0 const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); const data = { @@ -864,7 +864,7 @@ WHERE conversation_id = $1 AND seq = 0 const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ @@ -921,7 +921,7 @@ WHERE conversation_id = $1 AND seq = 0 const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -965,7 +965,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -998,7 +998,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); @@ -1037,7 +1037,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); const store = createSqlConversationEventStore(fixture.sql); await store.append(CONVERSATION_ID, [ @@ -1119,7 +1119,7 @@ INSERT INTO junior_conversation_events ( } try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); // Seed an old activity clock; content writes must refresh the window. await seedConversation(fixture, CONVERSATION_ID); await fixture.sql @@ -1188,7 +1188,7 @@ INSERT INTO junior_conversation_events ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await seedConversation(fixture, CONVERSATION_ID); await seedConversation(fixture, CHILD_CONVERSATION_ID, CONVERSATION_ID); const events = createSqlConversationEventStore(fixture.sql); diff --git a/packages/junior/tests/component/memory-plugin-storage.test.ts b/packages/junior/tests/component/memory-plugin-storage.test.ts index fe39fd488..080a4f6cb 100644 --- a/packages/junior/tests/component/memory-plugin-storage.test.ts +++ b/packages/junior/tests/component/memory-plugin-storage.test.ts @@ -13,10 +13,10 @@ import { } from "@sentry/junior-plugin-api"; import { defineJuniorPlugins } from "@/plugins"; import { getPluginTools, setPlugins } from "@/chat/plugins/agent-hooks"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import { readMigrationFiles } from "drizzle-orm/migrator"; import { closeDb } from "@/chat/db"; -import { runUpgrade } from "@/cli/upgrade"; +import { runUpgrade, runUpgradeMigrations } from "@/cli/upgrade"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const NEON = vi.hoisted(() => ({ @@ -86,7 +86,7 @@ function memoryMigrationFiles(): string[] { async function migrateMemorySchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -130,7 +130,7 @@ CREATE TABLE junior_schema_migrations ( } await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -168,7 +168,7 @@ CREATE TABLE junior_schema_migrations ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: memoryMigrationsDir(), pluginName: "memory", @@ -233,6 +233,28 @@ CREATE TABLE junior_schema_migrations ( } }, 15_000); + it("does not connect state for SQL-only migration journals", async () => { + const fixture = await createLocalJuniorSqlFixture(); + NEON.sql = fixture.sql; + + try { + const getStateContext = vi.fn(async () => { + throw new Error("SQL-only migrations must not connect state"); + }); + await expect( + runUpgradeMigrations({ + getStateContext, + io: { info: () => {} }, + pluginSet: defineJuniorPlugins([createMemoryPlugin()]), + }), + ).resolves.toHaveLength(2); + expect(getStateContext).not.toHaveBeenCalled(); + } finally { + NEON.sql = undefined; + await fixture.close(); + } + }, 15_000); + it("registers memory tools with runtime-provided plugin DB access", async () => { const fixture = await createLocalJuniorSqlFixture(); const previousPlugins = setPlugins([createMemoryPlugin()]); diff --git a/packages/junior/tests/component/scheduler-sql-plugin.test.ts b/packages/junior/tests/component/scheduler-sql-plugin.test.ts index 4722768d4..b7c106ed0 100644 --- a/packages/junior/tests/component/scheduler-sql-plugin.test.ts +++ b/packages/junior/tests/component/scheduler-sql-plugin.test.ts @@ -8,7 +8,7 @@ import { type SchedulerDb, type ScheduledTask, } from "@sentry/junior-scheduler"; -import { migratePluginSchemas } from "@/chat/plugins/migrations"; +import { bootstrapPluginSchemas } from "@/chat/plugins/migrations"; import { createLocalJuniorSqlFixture } from "../fixtures/sql"; const TEST_RUN_AT_MS = Date.parse("2026-05-26T12:00:00.000Z"); @@ -27,7 +27,7 @@ function memoryMigrationsDir(): string { async function migrateSchedulerSchema( fixture: Awaited>, ) { - await migratePluginSchemas(fixture.sql, [ + await bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -122,7 +122,7 @@ INSERT INTO junior_scheduler_tasks ( ); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -155,7 +155,7 @@ ORDER BY created_at }); expect(migratedTask?.record).not.toHaveProperty("credentialSubject"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: schedulerMigrationsDir(), pluginName: "scheduler", @@ -190,11 +190,13 @@ ORDER BY created_at ); try { - await expect(migratePluginSchemas(fixture.sql, roots)).resolves.toEqual({ - existing: 0, - migrated: migrationCount, - scanned: migrationCount, - }); + await expect(bootstrapPluginSchemas(fixture.sql, roots)).resolves.toEqual( + { + existing: 0, + migrated: migrationCount, + scanned: migrationCount, + }, + ); const migrationTables = await fixture.sql.query<{ tablename: string }>(` SELECT tablename FROM pg_tables @@ -204,7 +206,7 @@ ORDER BY tablename `); expect(migrationTables).toHaveLength(2); await expect( - migratePluginSchemas(fixture.sql, [...roots].reverse()), + bootstrapPluginSchemas(fixture.sql, [...roots].reverse()), ).resolves.toEqual({ existing: migrationCount, migrated: 0, @@ -228,12 +230,12 @@ ORDER BY tablename try { await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: missingJournal, pluginName: "missing" }, ]), ).rejects.toThrow("Can't find meta/_journal.json file"); await expect( - migratePluginSchemas(fixture.sql, [ + bootstrapPluginSchemas(fixture.sql, [ { dir: invalidJournal, pluginName: "invalid" }, ]), ).rejects.toThrow("Expected property name"); diff --git a/packages/junior/tests/integration/api/conversations/list.test.ts b/packages/junior/tests/integration/api/conversations/list.test.ts index 26d45a6ca..d11ad7ad4 100644 --- a/packages/junior/tests/integration/api/conversations/list.test.ts +++ b/packages/junior/tests/integration/api/conversations/list.test.ts @@ -18,7 +18,7 @@ describe("conversation list API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ actor: { email: "alice@example.com", @@ -193,7 +193,7 @@ describe("conversation list API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:root", nowMs: 1_000, diff --git a/packages/junior/tests/integration/conversation-sql.test.ts b/packages/junior/tests/integration/conversation-sql.test.ts index 8e168750e..b3c7cf33a 100644 --- a/packages/junior/tests/integration/conversation-sql.test.ts +++ b/packages/junior/tests/integration/conversation-sql.test.ts @@ -40,7 +40,7 @@ describe("conversation SQL local mode", () => { const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const rows = await fixture.sql.query<{ column_name: string; @@ -221,7 +221,7 @@ INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ('host-migration', 9999999999999) `); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const [host] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_migrations", @@ -250,7 +250,10 @@ VALUES ('host-migration', 9999999999999) second.query("SELECT 1"), ]); - await Promise.all([migrateSchema(fixture.sql), migrateSchema(second)]); + await Promise.all([ + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + migrateSchema(second, { mode: "schema-bootstrap" }), + ]); const [journal] = await fixture.sql.query<{ count: number }>( "SELECT count(*)::integer AS count FROM drizzle.__drizzle_junior_core", ); @@ -266,14 +269,18 @@ VALUES ('host-migration', 9999999999999) const fixture = await createLocalJuniorSqlFixture(); try { - await expect(migrateSchema(fixture.sql)).resolves.toEqual({ + await expect( + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + ).resolves.toEqual({ existing: 0, migrated: coreMigrations.length, scanned: coreMigrations.length, skipped: 0, }); - await expect(migrateSchema(fixture.sql)).resolves.toEqual({ + await expect( + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + ).resolves.toEqual({ existing: coreMigrations.length, migrated: 0, scanned: coreMigrations.length, @@ -369,7 +376,9 @@ CREATE TABLE junior_conversations ( ) `); - await expect(migrateSchema(fixture.sql)).rejects.toThrow( + await expect( + migrateSchema(fixture.sql, { mode: "schema-bootstrap" }), + ).rejects.toThrow( "Stop old Junior workers, install @sentry/junior@0.107.1, run `junior upgrade`, then restore this Junior version", ); await expectNoDrizzleMigrationState(fixture.sql); @@ -383,7 +392,7 @@ CREATE TABLE junior_conversations ( const fixture = await createLocalJuniorSqlFixture(); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const store = createSqlStore(fixture.sql); await recordAgentTurnSessionSummary({ From 01a096574f46e2d1b8023891b90dda87ada48dbd Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 22 Jul 2026 22:44:41 -0700 Subject: [PATCH 11/15] docs(migrations): Document maintainer authoring workflow Show when to use schema versus data migrations, the generation commands, generated file shapes, and the checks required before committing. --- CONTRIBUTING.md | 2 + packages/docs/astro.config.mjs | 9 +- .../docs/contribute/database-migrations.md | 113 ++++++++++++++++++ .../content/docs/contribute/development.md | 1 + 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 packages/docs/src/content/docs/contribute/database-migrations.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 25619e00f..57da71747 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,8 @@ pnpm release:check - `policies/` contains durable repo-wide engineering rules. - Package and module `README.md` files explain implemented architecture and non-obvious invariants near the code they describe. +- The [database migration guide](https://junior.sentry.dev/contribute/database-migrations/) + explains when and how to generate schema and data migrations. - Code, schemas, exported types, and tests are the implementation authority. - `openspec/changes/` contains temporary implementation plans. Move durable explanation beside the code and delete a plan when its work is complete. diff --git a/packages/docs/astro.config.mjs b/packages/docs/astro.config.mjs index 5f47981d5..15740f23d 100644 --- a/packages/docs/astro.config.mjs +++ b/packages/docs/astro.config.mjs @@ -178,6 +178,10 @@ export default defineConfig({ label: "Contribute", items: [ { label: "Development", link: "/contribute/development/" }, + { + label: "Database Migrations", + link: "/contribute/database-migrations/", + }, { label: "Testing", link: "/contribute/testing/" }, { label: "Local Agent Validation", @@ -191,10 +195,7 @@ export default defineConfig({ ], }, ], - plugins: [ - sentryStarlightTheme(), - sentryAgentMarkdown(), - ], + plugins: [sentryStarlightTheme(), sentryAgentMarkdown()], }), ], markdown: { diff --git a/packages/docs/src/content/docs/contribute/database-migrations.md b/packages/docs/src/content/docs/contribute/database-migrations.md new file mode 100644 index 000000000..fa8e7e5f4 --- /dev/null +++ b/packages/docs/src/content/docs/contribute/database-migrations.md @@ -0,0 +1,113 @@ +--- +title: Database Migrations +description: Generate and review Junior schema and data migrations. +type: tutorial +summary: Choose the right migration type, generate it in the owning package, and verify the result. +prerequisites: + - /contribute/development/ +related: + - /cli/upgrade/ + - /contribute/testing/ + - /contribute/releasing/ +--- + +Junior keeps schema and data migrations in the same ordered Drizzle journal. +Each journal entry has exactly one source file: + +| Change | Generate | Result | +| ----------------------------------------------------------------------------- | ---------------- | ----------------------------------- | +| Tables, columns, indexes, or a simple SQL backfill | Schema migration | `.sql` plus a Drizzle snapshot | +| Data that needs application code, state storage, Redis, or resumable progress | Data migration | `.ts` with no new snapshot | + +Generate the migration in the package that owns the schema. Core Junior uses +`@sentry/junior`; plugin tables use their plugin package. + +## Generate a schema migration + +First change the owning Drizzle schema, such as +`packages/junior/src/db/schema.ts`. Then run: + +```bash +pnpm --filter @sentry/junior db:generate --name add_delivery_status +``` + +For a plugin, replace the package name: + +```bash +pnpm --filter @sentry/junior-memory db:generate --name add_memory_source +``` + +Drizzle adds a SQL file, updates `meta/_journal.json`, and writes a schema +snapshot. The generated SQL looks like: + +```sql title="migrations/0007_add_delivery_status.sql" +ALTER TABLE "junior_conversations" +ADD COLUMN "delivery_status" text; +``` + +Review the SQL against the schema change. Prefer SQL when the work is entirely +inside the same database and does not need application-level decoding. + +## Generate a data migration + +Use a data migration when SQL is not enough—for example, when moving data from +state storage, preserving a Redis index, decoding an old application record, or +checkpointing a long backfill. + +```bash +pnpm --filter @sentry/junior db:generate:data --name backfill_delivery_status +``` + +The command adds a TypeScript entry to the same journal: + +```ts title="migrations/0008_backfill_delivery_status.ts" +import type { MigrationV1 } from "@sentry/junior-migrations"; + +const migration = { + apiVersion: 1, + async up(context) { + void context; + }, +} satisfies MigrationV1; + +export default migration; +``` + +Implement `up` with the capabilities on `context`: + +- `database` for queries, transactions, and database locks +- `state` for Junior or plugin-scoped state +- `redis` when a migration must preserve raw Redis structures +- `progress` for a checkpoint that survives a failed or interrupted run +- `log` for operator-facing progress messages + +Keep one-off decoding and transformation logic in the migration file. Do not +import current Junior runtime modules from a data migration. + +## Review and verify + +Before committing: + +1. Confirm the new journal entry has exactly one `.sql` or `.ts` file. +2. Commit the updated `meta/_journal.json`. +3. Commit the new snapshot for a schema migration; a data migration should not + add one. +4. Do not edit, rename, reorder, or delete a migration that has shipped. +5. Make long-running data migrations safe to retry and use `progress` when + partial work must resume. + +Run the focused migration checks: + +```bash +pnpm --filter @sentry/junior-migrations test +pnpm typecheck +``` + +`junior upgrade` runs schema and data entries in journal order. Schema-bootstrap +mode is only for constructing empty test databases; it is not an upgrade path +for an existing installation. + +## Next step + +Add focused coverage using the guidance in [Testing](/contribute/testing/), +then review the release path in [Releasing](/contribute/releasing/). diff --git a/packages/docs/src/content/docs/contribute/development.md b/packages/docs/src/content/docs/contribute/development.md index 8d0822711..dacd3bf52 100644 --- a/packages/docs/src/content/docs/contribute/development.md +++ b/packages/docs/src/content/docs/contribute/development.md @@ -5,6 +5,7 @@ type: tutorial summary: Set up Junior locally, run checks, and use isolated worktrees for parallel agent or contributor tasks. prerequisites: [] related: + - /contribute/database-migrations/ - /contribute/testing/ - /contribute/releasing/ - /start-here/quickstart/ From 86106a39a1b5f921e13b6f13f9b3b30d73ba69a9 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 23 Jul 2026 08:17:50 -0700 Subject: [PATCH 12/15] fix(migrations): Preserve PGlite progress on failure Serialize PGlite migration runs outside database transactions so data migration checkpoints and failure state survive an interrupted run. --- packages/junior-testing/src/pglite.ts | 29 +++++++++++++++ packages/junior/tests/fixtures/sql.ts | 2 +- packages/junior/tests/unit/sql/pglite.test.ts | 37 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 packages/junior/tests/unit/sql/pglite.test.ts diff --git a/packages/junior-testing/src/pglite.ts b/packages/junior-testing/src/pglite.ts index a76152d2b..933183e5a 100644 --- a/packages/junior-testing/src/pglite.ts +++ b/packages/junior-testing/src/pglite.ts @@ -21,11 +21,16 @@ export interface LocalPgliteFixture { ): Promise; transaction(callback: () => Promise): Promise; withLock(lockName: string, callback: () => Promise): Promise; + withMigrationLock( + lockName: string, + callback: () => Promise, + ): Promise; close(): Promise; } class LocalPgliteExecutor implements LocalPgliteFixture { private activeTransaction: Transaction | undefined; + private readonly migrationLocks = new Map>(); constructor( readonly client: PGlite, @@ -85,6 +90,30 @@ class LocalPgliteExecutor implements LocalPgliteFixture { }); } + async withMigrationLock( + lockName: string, + callback: () => Promise, + ): Promise { + if (!lockName) { + throw new Error("Migration lock name is required"); + } + const previous = this.migrationLocks.get(lockName) ?? Promise.resolve(); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + this.migrationLocks.set(lockName, current); + await previous; + try { + return await callback(); + } finally { + release(); + if (this.migrationLocks.get(lockName) === current) { + this.migrationLocks.delete(lockName); + } + } + } + close(): Promise { return this.client.close(); } diff --git a/packages/junior/tests/fixtures/sql.ts b/packages/junior/tests/fixtures/sql.ts index b7bd6a235..fd1ee29ba 100644 --- a/packages/junior/tests/fixtures/sql.ts +++ b/packages/junior/tests/fixtures/sql.ts @@ -47,7 +47,7 @@ export async function createLocalJuniorSqlFixture(): Promise fixture.transaction(callback), withLock: (lockName, callback) => fixture.withLock(lockName, callback), withMigrationLock: (migrationTable, callback) => - fixture.withLock(`junior:migrate:${migrationTable}`, callback), + fixture.withMigrationLock(`junior:migrate:${migrationTable}`, callback), }; return { diff --git a/packages/junior/tests/unit/sql/pglite.test.ts b/packages/junior/tests/unit/sql/pglite.test.ts new file mode 100644 index 000000000..f6710c8f3 --- /dev/null +++ b/packages/junior/tests/unit/sql/pglite.test.ts @@ -0,0 +1,37 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + createLocalPgliteFixture, + type LocalPgliteFixture, +} from "@sentry/junior-testing/pglite"; + +let fixture: LocalPgliteFixture | undefined; + +afterEach(async () => { + await fixture?.close(); + fixture = undefined; +}); + +describe("PGlite migration lock", () => { + it("preserves writes when a migration callback fails", async () => { + fixture = await createLocalPgliteFixture({}); + await fixture.execute(` +CREATE TABLE migration_progress ( + cursor INTEGER NOT NULL +) +`); + + await expect( + fixture.withMigrationLock("test-journal", async () => { + await fixture!.execute( + "INSERT INTO migration_progress (cursor) VALUES ($1)", + [1], + ); + throw new Error("migration interrupted"); + }), + ).rejects.toThrow("migration interrupted"); + + await expect( + fixture.query("SELECT cursor FROM migration_progress"), + ).resolves.toEqual([{ cursor: 1 }]); + }); +}); From 64ff6a43cfc2a5b3e7400fdaba43495a728979bd Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 23 Jul 2026 08:27:38 -0700 Subject: [PATCH 13/15] fix(migrations): Bootstrap rebased SQL tests Pass the explicit schema-bootstrap mode in SQL tests added on main after the migration helper became mode-aware. --- packages/junior/tests/component/conversation-sql-store.test.ts | 2 +- .../junior/tests/integration/api/conversations/list.test.ts | 2 +- packages/junior/tests/integration/api/people/profile.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/junior/tests/component/conversation-sql-store.test.ts b/packages/junior/tests/component/conversation-sql-store.test.ts index f1550c592..f2bc6af2e 100644 --- a/packages/junior/tests/component/conversation-sql-store.test.ts +++ b/packages/junior/tests/component/conversation-sql-store.test.ts @@ -36,7 +36,7 @@ describe("conversation SQL store", () => { try { const store = createSqlStore(fixture.sql); - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); const at = new Date(1); await fixture.sql.db().insert(juniorConversations).values({ conversationId: "parent-without-root", diff --git a/packages/junior/tests/integration/api/conversations/list.test.ts b/packages/junior/tests/integration/api/conversations/list.test.ts index d11ad7ad4..160a7aa51 100644 --- a/packages/junior/tests/integration/api/conversations/list.test.ts +++ b/packages/junior/tests/integration/api/conversations/list.test.ts @@ -237,7 +237,7 @@ describe("conversation list API", () => { const fixture = createConfiguredJuniorSqlFixture(); const store = createSqlStore(fixture.sql); try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: "slack:C1:invalid-root", nowMs: 1_000, diff --git a/packages/junior/tests/integration/api/people/profile.test.ts b/packages/junior/tests/integration/api/people/profile.test.ts index 5afad1fa6..148bd8a34 100644 --- a/packages/junior/tests/integration/api/people/profile.test.ts +++ b/packages/junior/tests/integration/api/people/profile.test.ts @@ -21,7 +21,7 @@ describe("people profile API", () => { const ownerChildConversationId = "slack:C-public:owner-child"; try { - await migrateSchema(fixture.sql); + await migrateSchema(fixture.sql, { mode: "schema-bootstrap" }); await store.recordActivity({ conversationId: rootConversationId, actor: { From db6a3ff75993916021a2adcf23ef6b2459f54ad4 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 23 Jul 2026 08:35:25 -0700 Subject: [PATCH 14/15] fix(migrations): Address follow-up review feedback Allow edited failed data migrations to restart with clean progress while keeping completed entries immutable. Avoid import-guard false positives, honor generator output overrides, and align the package version with the current release. --- packages/junior-migrations/package.json | 2 +- packages/junior-migrations/src/generate.ts | 60 +++++++++--- packages/junior-migrations/src/journal.ts | 6 +- packages/junior-migrations/src/runner.ts | 32 +++++-- .../junior-migrations/tests/generate.test.ts | 30 ++++++ .../junior-migrations/tests/journal.test.ts | 27 ++++++ .../junior-migrations/tests/runner.test.ts | 95 +++++++++++++++++++ 7 files changed, 228 insertions(+), 24 deletions(-) diff --git a/packages/junior-migrations/package.json b/packages/junior-migrations/package.json index 4cfc55d1c..dc8c8aa75 100644 --- a/packages/junior-migrations/package.json +++ b/packages/junior-migrations/package.json @@ -1,6 +1,6 @@ { "name": "@sentry/junior-migrations", - "version": "0.108.0", + "version": "0.109.0", "private": false, "publishConfig": { "access": "public" diff --git a/packages/junior-migrations/src/generate.ts b/packages/junior-migrations/src/generate.ts index 97beca0c7..c934f4a6c 100644 --- a/packages/junior-migrations/src/generate.ts +++ b/packages/junior-migrations/src/generate.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; -import { readFile, rename, rm, writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { relative, resolve, sep } from "node:path"; import { readMigrationJournal } from "./journal"; /** Drizzle Kit inputs used to create one TypeScript migration entry. */ @@ -36,6 +36,49 @@ function isMissingJournal(error: unknown): boolean { ); } +async function runDrizzleGenerate( + options: GenerateTypeScriptMigrationOptions, + cwd: string, + folder: string, +): Promise { + const configDirectory = await mkdtemp( + resolve(cwd, ".junior-migrations-config-"), + ); + try { + const configPath = resolve(cwd, options.configPath); + const relativeConfigPath = relative(configDirectory, configPath) + .split(sep) + .join("/"); + const configSpecifier = relativeConfigPath.startsWith(".") + ? relativeConfigPath + : `./${relativeConfigPath}`; + const relativeOutputPath = relative(cwd, folder).split(sep).join("/"); + const configOutputPath = relativeOutputPath.startsWith(".") + ? relativeOutputPath + : `./${relativeOutputPath}`; + const overrideConfigPath = resolve(configDirectory, "drizzle.config.ts"); + await writeFile( + overrideConfigPath, + `import config from ${JSON.stringify(configSpecifier)};\nexport default { ...config, out: ${JSON.stringify(configOutputPath)} };\n`, + "utf8", + ); + await run( + "drizzle-kit", + [ + "generate", + "--custom", + "--config", + overrideConfigPath, + "--name", + options.name, + ], + cwd, + ); + } finally { + await rm(configDirectory, { recursive: true, force: true }); + } +} + /** Create a journaled TypeScript migration through Drizzle Kit's custom generator. */ export async function generateTypeScriptMigration( options: GenerateTypeScriptMigrationOptions, @@ -49,18 +92,7 @@ export async function generateTypeScriptMigration( if (!isMissingJournal(error)) throw error; before = []; } - await run( - "drizzle-kit", - [ - "generate", - "--custom", - "--config", - options.configPath, - "--name", - options.name, - ], - cwd, - ); + await runDrizzleGenerate(options, cwd, folder); const after = await readMigrationJournal(folder); if (after.length !== before.length + 1) { throw new Error("Drizzle Kit did not create exactly one migration entry"); diff --git a/packages/junior-migrations/src/journal.ts b/packages/junior-migrations/src/journal.ts index ca1b92847..142ac7573 100644 --- a/packages/junior-migrations/src/journal.ts +++ b/packages/junior-migrations/src/journal.ts @@ -106,9 +106,13 @@ function validateTypeScriptSource(tag: string, source: string): void { } validateRuntimeSpecifier(match[2]); } + const executableSource = source.replace( + /\/\*[\s\S]*?\*\/|\/\/[^\r\n]*|"(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`/g, + (match) => match.replace(/[^\r\n]/g, " "), + ); if ( /^\s*import\s*["']/m.test(source) || - /\b(?:import\s*\(|require\s*\()/.test(source) + /\b(?:import\s*\(|require\s*\()/.test(executableSource) ) { throw new Error(`TypeScript migration ${tag} cannot load runtime modules`); } diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index 9a3c88e59..c75816c54 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -232,16 +232,28 @@ async function runTypeScriptMigration(args: { table: string; }): Promise { const qualified = qualifiedTable(args.table); - if (args.row && args.row.hash !== args.migration.hash) { + const sourceChanged = + args.row !== undefined && args.row.hash !== args.migration.hash; + if (args.row && sourceChanged && args.row.status !== "failed") { throw new Error(`Migration ${args.migration.tag} changed after it started`); } if (args.row) { - await args.executor.execute( - `UPDATE ${qualified} - SET name = $1, kind = 'typescript', status = 'running', started_at = NOW() - WHERE created_at = $2`, - [args.migration.tag, args.migration.when], - ); + if (sourceChanged) { + await args.executor.execute( + `UPDATE ${qualified} + SET hash = $1, name = $2, kind = 'typescript', status = 'running', + progress = NULL, result = NULL, started_at = NOW(), completed_at = NULL + WHERE created_at = $3`, + [args.migration.hash, args.migration.tag, args.migration.when], + ); + } else { + await args.executor.execute( + `UPDATE ${qualified} + SET name = $1, kind = 'typescript', status = 'running', started_at = NOW() + WHERE created_at = $2`, + [args.migration.tag, args.migration.when], + ); + } } else { await args.executor.execute( `INSERT INTO ${qualified} @@ -339,7 +351,11 @@ export async function runMigrationJournal( }; for (const migration of migrations) { const row = rows.get(migration.when); - if (row && row.hash !== migration.hash) { + if ( + row && + row.hash !== migration.hash && + !(migration.kind === "typescript" && row.status === "failed") + ) { throw new Error( `Migration ${migration.tag} changed after it started`, ); diff --git a/packages/junior-migrations/tests/generate.test.ts b/packages/junior-migrations/tests/generate.test.ts index d594a1a67..6536f7192 100644 --- a/packages/junior-migrations/tests/generate.test.ts +++ b/packages/junior-migrations/tests/generate.test.ts @@ -120,3 +120,33 @@ it("does not invoke Drizzle when an existing journal is invalid", async () => { }), ).rejects.toThrow("Unsupported Drizzle journal"); }); + +it("forwards the requested output folder to Drizzle Kit", async () => { + const relativeRoot = `.tmp-migration-output-${Date.now()}`; + const root = join(process.cwd(), relativeRoot); + temporaryDirectories.push(root); + await mkdir(root); + await writeFile( + join(root, "schema.ts"), + 'import { pgTable, text } from "drizzle-orm/pg-core";\nexport const users = pgTable("users", { id: text("id").primaryKey() });\n', + ); + await writeFile( + join(root, "drizzle.config.ts"), + `import { defineConfig } from "drizzle-kit";\nexport default defineConfig({ dialect: "postgresql", schema: "./${relativeRoot}/schema.ts", out: "./${relativeRoot}/configured-migrations" });\n`, + ); + + const typescriptPath = await generateTypeScriptMigration({ + configPath: `${relativeRoot}/drizzle.config.ts`, + cwd: process.cwd(), + migrationsFolder: `${relativeRoot}/requested-migrations`, + name: "backfill", + }); + + expect(typescriptPath).toBe( + join(root, "requested-migrations", "0000_backfill.ts"), + ); + await expect(access(typescriptPath)).resolves.toBeUndefined(); + await expect(access(join(root, "configured-migrations"))).rejects.toThrow( + "ENOENT", + ); +}); diff --git a/packages/junior-migrations/tests/journal.test.ts b/packages/junior-migrations/tests/journal.test.ts index 38a2c9875..ce2f3e1da 100644 --- a/packages/junior-migrations/tests/journal.test.ts +++ b/packages/junior-migrations/tests/journal.test.ts @@ -86,6 +86,33 @@ describe("resolveMigrations", () => { ]); }); + it("allows import syntax in comments and strings", async () => { + const folder = await migrationFolder(["0000_documented"]); + await writeFile( + join(folder, "0000_documented.ts"), + `// Never call import("@/db") from a migration. +const guidance = "require(\\"@/db\\") is unsupported"; +export default { apiVersion: 1, async up() { void guidance; } }; +`, + ); + + await expect(resolveMigrations(folder)).resolves.toMatchObject([ + { kind: "typescript", tag: "0000_documented" }, + ]); + }); + + it("still rejects executable dynamic imports", async () => { + const folder = await migrationFolder(["0000_dynamic"]); + await writeFile( + join(folder, "0000_dynamic.ts"), + 'export default { apiVersion: 1, async up() { await import("@/db"); } };\n', + ); + + await expect(resolveMigrations(folder)).rejects.toThrow( + "cannot load runtime modules", + ); + }); + it("rejects runtime re-exports of Junior internals", async () => { const folder = await migrationFolder(["0000_reexport"]); await writeFile( diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index 04ac6dcfc..1ae05196d 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -51,6 +51,12 @@ class FakeExecutor implements MigrationDatabaseAdapter { if (normalized.includes("progress = $1")) { row.progress = parameters[0]; } + if (normalized.includes("hash = $1")) { + row.hash = String(parameters[0]); + } + if (normalized.includes("progress = NULL")) { + row.progress = undefined; + } if (normalized.includes("status = 'running'")) { row.status = "running"; } else if (normalized.includes("status = 'completed'")) { @@ -298,6 +304,95 @@ describe("runMigrationJournal", () => { ]); }); + it("restarts a failed TypeScript migration after its source changes", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + let attempts = 0; + let resumedProgress: unknown; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + attempts += 1; + if (attempts === 1) { + await context.progress.save({ cursor: 1 }); + throw new Error("needs a source fix"); + } + resumedProgress = await context.progress.load(); + }, + }; + const options = { + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }; + + await expect(runMigrationJournal(options)).rejects.toThrow( + "needs a source fix", + ); + const failedHash = executor.rows.get(2_001)?.hash; + await writeFile( + join(folder, "0001_data.ts"), + "export default { apiVersion: 1, async up() { return { fixed: true }; } };\n", + ); + + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 1, + migrated: 2, + scanned: 3, + skipped: 0, + }); + expect(resumedProgress).toBeUndefined(); + expect(executor.rows.get(2_001)).toMatchObject({ + status: "completed", + }); + expect(executor.rows.get(2_001)?.hash).not.toBe(failedHash); + }); + + it("rejects source changes after a TypeScript migration completes", async () => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + const migration: MigrationV1 = { + apiVersion: 1, + async up() {}, + }; + const options = { + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }; + await runMigrationJournal(options); + await writeFile( + join(folder, "0001_data.ts"), + "export default { apiVersion: 1, async up() { return { changed: true }; } };\n", + ); + + await expect(runMigrationJournal(options)).rejects.toThrow( + "changed after it started", + ); + }); + it("rejects non-JSON migration results before completing the ledger row", async () => { const folder = await mixedFolder(); const executor = new FakeExecutor(); From 5a93cff28dd87e999ef7b6109da2cc440d88b662 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 23 Jul 2026 09:06:17 -0700 Subject: [PATCH 15/15] fix(migrations): Address migration recovery gaps Allow edited TypeScript migrations to recover from stale running rows once the migration lock is reacquired. Pass the host Redis capability into plugin data migrations and cover both paths with tests. --- packages/junior-migrations/src/runner.ts | 12 +- .../junior-migrations/tests/runner.test.ts | 112 ++++++++++-------- .../junior/src/chat/plugins/migrations.ts | 38 ++++-- .../cli/upgrade/migrations/plugin-journal.ts | 3 +- .../migrations/mixed-runner-database.test.ts | 51 ++++++++ 5 files changed, 150 insertions(+), 66 deletions(-) diff --git a/packages/junior-migrations/src/runner.ts b/packages/junior-migrations/src/runner.ts index c75816c54..b1a66b15b 100644 --- a/packages/junior-migrations/src/runner.ts +++ b/packages/junior-migrations/src/runner.ts @@ -234,7 +234,12 @@ async function runTypeScriptMigration(args: { const qualified = qualifiedTable(args.table); const sourceChanged = args.row !== undefined && args.row.hash !== args.migration.hash; - if (args.row && sourceChanged && args.row.status !== "failed") { + if ( + args.row && + sourceChanged && + args.row.status !== "failed" && + args.row.status !== "running" + ) { throw new Error(`Migration ${args.migration.tag} changed after it started`); } if (args.row) { @@ -354,7 +359,10 @@ export async function runMigrationJournal( if ( row && row.hash !== migration.hash && - !(migration.kind === "typescript" && row.status === "failed") + !( + migration.kind === "typescript" && + (row.status === "failed" || row.status === "running") + ) ) { throw new Error( `Migration ${migration.tag} changed after it started`, diff --git a/packages/junior-migrations/tests/runner.test.ts b/packages/junior-migrations/tests/runner.test.ts index 1ae05196d..a9429550e 100644 --- a/packages/junior-migrations/tests/runner.test.ts +++ b/packages/junior-migrations/tests/runner.test.ts @@ -304,60 +304,68 @@ describe("runMigrationJournal", () => { ]); }); - it("restarts a failed TypeScript migration after its source changes", async () => { - const folder = await mixedFolder(); - const executor = new FakeExecutor(); - let attempts = 0; - let resumedProgress: unknown; - const migration: MigrationV1 = { - apiVersion: 1, - async up(context) { - attempts += 1; - if (attempts === 1) { - await context.progress.save({ cursor: 1 }); - throw new Error("needs a source fix"); - } - resumedProgress = await context.progress.load(); - }, - }; - const options = { - executor, - migrationsFolder: folder, - migrationsTable: "__drizzle_test", - loadTypeScript: async () => ({ default: migration }), - createContext: ({ - progress, - }: { - progress: MigrationContextV1["progress"]; - }) => ({ - database: executor, - log: () => {}, - progress, - state: fakeMigrationState(), - }), - }; + it.each(["failed", "running"] as const)( + "restarts a %s TypeScript migration after its source changes", + async (retryStatus) => { + const folder = await mixedFolder(); + const executor = new FakeExecutor(); + let attempts = 0; + let resumedProgress: unknown; + const migration: MigrationV1 = { + apiVersion: 1, + async up(context) { + attempts += 1; + if (attempts === 1) { + await context.progress.save({ cursor: 1 }); + throw new Error("needs a source fix"); + } + resumedProgress = await context.progress.load(); + }, + }; + const options = { + executor, + migrationsFolder: folder, + migrationsTable: "__drizzle_test", + loadTypeScript: async () => ({ default: migration }), + createContext: ({ + progress, + }: { + progress: MigrationContextV1["progress"]; + }) => ({ + database: executor, + log: () => {}, + progress, + state: fakeMigrationState(), + }), + }; - await expect(runMigrationJournal(options)).rejects.toThrow( - "needs a source fix", - ); - const failedHash = executor.rows.get(2_001)?.hash; - await writeFile( - join(folder, "0001_data.ts"), - "export default { apiVersion: 1, async up() { return { fixed: true }; } };\n", - ); + await expect(runMigrationJournal(options)).rejects.toThrow( + "needs a source fix", + ); + const interruptedRow = executor.rows.get(2_001); + expect(interruptedRow?.status).toBe("failed"); + if (interruptedRow) { + interruptedRow.status = retryStatus; + } + const failedHash = interruptedRow?.hash; + await writeFile( + join(folder, "0001_data.ts"), + "export default { apiVersion: 1, async up() { return { fixed: true }; } };\n", + ); - await expect(runMigrationJournal(options)).resolves.toEqual({ - existing: 1, - migrated: 2, - scanned: 3, - skipped: 0, - }); - expect(resumedProgress).toBeUndefined(); - expect(executor.rows.get(2_001)).toMatchObject({ - status: "completed", - }); - expect(executor.rows.get(2_001)?.hash).not.toBe(failedHash); - }); + await expect(runMigrationJournal(options)).resolves.toEqual({ + existing: 1, + migrated: 2, + scanned: 3, + skipped: 0, + }); + expect(resumedProgress).toBeUndefined(); + expect(executor.rows.get(2_001)).toMatchObject({ + status: "completed", + }); + expect(executor.rows.get(2_001)?.hash).not.toBe(failedHash); + }, + ); it("rejects source changes after a TypeScript migration completes", async () => { const folder = await mixedFolder(); diff --git a/packages/junior/src/chat/plugins/migrations.ts b/packages/junior/src/chat/plugins/migrations.ts index 4f0f6d5d1..8d9bbe191 100644 --- a/packages/junior/src/chat/plugins/migrations.ts +++ b/packages/junior/src/chat/plugins/migrations.ts @@ -7,6 +7,7 @@ import { type TypeScriptMigrationLoader, } from "@sentry/junior-migrations"; import type { StateAdapter } from "chat"; +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; import { createPluginMigrationStateV1 } from "@/chat/plugins/migration-state"; import type { JuniorSqlMigrationExecutor } from "@/db/db"; @@ -26,7 +27,10 @@ type PluginMigrationResult = { type PluginMigrationOptions = | { mode: "schema-bootstrap" } | { - getStateAdapter: () => Promise; + getStateContext: () => Promise<{ + redisStateAdapter?: RedisStateAdapter; + stateAdapter: StateAdapter; + }>; loadTypeScript: TypeScriptMigrationLoader; log?: MigrationContextV1["log"]; mode: "all"; @@ -165,15 +169,29 @@ export async function migratePluginSchemas( const pluginResult = runAll ? await runMigrationJournal({ ...baseOptions, - createContext: async ({ progress }): Promise => ({ - database: executor, - log: options.log ?? (() => {}), - progress, - state: createPluginMigrationStateV1( - root.pluginName, - await options.getStateAdapter(), - ), - }), + createContext: async ({ progress }): Promise => { + const { redisStateAdapter, stateAdapter } = + await options.getStateContext(); + return { + database: executor, + log: options.log ?? (() => {}), + progress, + ...(redisStateAdapter + ? { + redis: { + sendCommand: async (args: readonly string[]) => + await redisStateAdapter + .getClient() + .sendCommand([...args]), + }, + } + : {}), + state: createPluginMigrationStateV1( + root.pluginName, + stateAdapter, + ), + }; + }, loadTypeScript: options.loadTypeScript, mode: "all", }) diff --git a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts index ea1e8587f..4c1b1c9f9 100644 --- a/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts +++ b/packages/junior/src/cli/upgrade/migrations/plugin-journal.ts @@ -24,8 +24,7 @@ export async function migratePluginJournals( executor, pluginCatalogRuntime.getMigrationRoots(), { - getStateAdapter: async () => - (await context.getStateContext()).stateAdapter, + getStateContext: context.getStateContext, loadTypeScript: async (path) => await migrationLoader.import>(path), log: context.io.info, diff --git a/packages/junior/tests/component/migrations/mixed-runner-database.test.ts b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts index a972b989d..cfbfb4df8 100644 --- a/packages/junior/tests/component/migrations/mixed-runner-database.test.ts +++ b/packages/junior/tests/component/migrations/mixed-runner-database.test.ts @@ -6,6 +6,9 @@ import { type MigrationContextV1, type MigrationStateV1, } from "@sentry/junior-migrations"; +import type { RedisStateAdapter } from "@chat-adapter/state-redis"; +import type { StateAdapter } from "chat"; +import { migratePluginSchemas } from "@/chat/plugins/migrations"; import { createJiti } from "jiti"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -92,6 +95,54 @@ afterEach(async () => { }); describe("mixed migration runner database contract", () => { + it("provides the host Redis capability to plugin data migrations", async () => { + const fixture = await createLocalJuniorSqlFixture(); + const folder = await createMigrationFolder({ + source: ` +export default { + apiVersion: 1, + async up(context) { + return { + response: await context.redis.sendCommand(["PING"]), + }; + }, +}; +`, + tag: "0000_plugin_redis", + when: 2_026_071_600_000, + }); + const sendCommand = vi.fn(async () => "PONG"); + + try { + await expect( + migratePluginSchemas( + fixture.sql, + [{ dir: folder, pluginName: "redis-test" }], + { + getStateContext: async () => ({ + redisStateAdapter: { + getClient: () => ({ sendCommand }), + } as unknown as RedisStateAdapter, + stateAdapter: {} as StateAdapter, + }), + loadTypeScript: async (migrationPath) => + await migrationLoader.import>( + migrationPath, + ), + mode: "all", + }, + ), + ).resolves.toEqual({ + existing: 0, + migrated: 1, + scanned: 1, + }); + expect(sendCommand).toHaveBeenCalledWith(["PING"]); + } finally { + await fixture.close(); + } + }); + it("persists failed progress and resumes a TypeScript migration", async () => { const fixture = await createLocalJuniorSqlFixture(); const folder = await createMigrationFolder({