diff --git a/.changeset/binary-no-dotenv-autoload.md b/.changeset/binary-no-dotenv-autoload.md new file mode 100644 index 00000000..92936ac7 --- /dev/null +++ b/.changeset/binary-no-dotenv-autoload.md @@ -0,0 +1,5 @@ +--- +'@noormdev/cli': patch +--- + +Stop the `noorm` binary from loading `.env` in the working directory. Bun's loader expanded `$` and cut values at `#`, so a `NOORM_CONNECTION_PASSWORD` containing those characters reached the database as a different password and login failed. `NOORM_*` variables now come only from the process environment: export them in the shell or set them in CI. diff --git a/.changeset/connection-error-reasons.md b/.changeset/connection-error-reasons.md new file mode 100644 index 00000000..cea2dbe2 --- /dev/null +++ b/.changeset/connection-error-reasons.md @@ -0,0 +1,6 @@ +--- +'@noormdev/cli': patch +'@noormdev/sdk': patch +--- + +Say why a database connection failed: refused port, unknown host, timeout, rejected TLS certificate, disabled, locked, or expired account, missing grant, connection limit, missing password, and SQLite file or directory faults. Where the server withholds the reason (SQL Server 18456, PostgreSQL 28P01, MySQL 1045), the message says so and lists the usual causes. Exhausted retries report the server's last error. `connection:error` log entries carry `serverCode` and `serverMessage`. diff --git a/.changeset/mssql-unprivileged-logins.md b/.changeset/mssql-unprivileged-logins.md new file mode 100644 index 00000000..c5f1d580 --- /dev/null +++ b/.changeset/mssql-unprivileged-logins.md @@ -0,0 +1,6 @@ +--- +'@noormdev/cli': patch +'@noormdev/sdk': patch +--- + +Let MSSQL logins without server-level access connect. Connecting no longer detours through `master` to look the target up in `sys.databases`, so contained database users (the usual account on Azure SQL Database) and logins without `VIEW ANY DATABASE` can connect, and the config add/edit connection test passes for them. A database that is missing, or that the login cannot open, now fails with that reason instead of "Login failed". diff --git a/.changeset/slow-statement-watcher.md b/.changeset/slow-statement-watcher.md new file mode 100644 index 00000000..b20d3cec --- /dev/null +++ b/.changeset/slow-statement-watcher.md @@ -0,0 +1,6 @@ +--- +'@noormdev/cli': minor +'@noormdev/sdk': minor +--- + +Show what a long-running SQL file is doing. After 10 seconds the runner asks the server about the file's session every 10 seconds, from a second pooled connection, and emits `file:progress`: elapsed time, the command's own progress (PostgreSQL `pg_stat_progress_*`, SQL Server `percent_complete`, MySQL stage counters), and the sessions it is waiting on. The TUI run and change screens render it under the running file, and the log records it. Build and exec runs in the TUI cancel on a second `Escape` within 2 seconds; PostgreSQL and MySQL stop the running statement on the server. diff --git a/.changeset/tui-explore-duplicate-keys.md b/.changeset/tui-explore-duplicate-keys.md new file mode 100644 index 00000000..2066d8d9 --- /dev/null +++ b/.changeset/tui-explore-duplicate-keys.md @@ -0,0 +1,6 @@ +--- +'@noormdev/cli': patch +'@noormdev/sdk': patch +--- + +Stop TUI lists from leaving stale rows on screen when two rows share an identity: explore indexes and foreign keys whose names repeat across tables (SQL Server's `IX_UserId`, MySQL's `PRIMARY`), PostgreSQL function and procedure overloads, and settings rules with the same description. `listFunctions` and `listProcedures` now return a `signature` on PostgreSQL that tells overloads apart. diff --git a/.changeset/tui-typing-blocks-shortcuts.md b/.changeset/tui-typing-blocks-shortcuts.md new file mode 100644 index 00000000..94a03224 --- /dev/null +++ b/.changeset/tui-typing-blocks-shortcuts.md @@ -0,0 +1,5 @@ +--- +'@noormdev/cli': patch +--- + +Stop global TUI shortcuts from firing while you type. A capital `L` or `Q` typed into a form field, search box, or the SQL editor used to open the log viewer or the SQL terminal, and `?`, `D`, and `F` could open help or toggle dry-run and force mode. While a text field is taking input, those keys now type their character. diff --git a/.claude/rules/wiki/cli.md b/.claude/rules/wiki/cli.md new file mode 100644 index 00000000..ab2421b4 --- /dev/null +++ b/.claude/rules/wiki/cli.md @@ -0,0 +1,25 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/cli/*.ts" + - "src/cli/ci/**" + - "src/cli/config/**" + - "src/cli/dev/**" + - "src/cli/lock/**" + - "src/cli/settings/**" + - "packages/cli/**" + - "skills/noorm/**" + - "tests/cli/*" + - "tests/integration/cli/**" +--- + +Domain: cli. Citty CLI command groups and leaf commands, headless mode, binary distribution. + +Map: + - docs/wiki/cli.md +Contracts: + - docs/spec/v1-49-54-cli-field-defects.md +Designs: + - docs/design/v1-49-54-cli-field-defects.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-change.md b/.claude/rules/wiki/core-change.md new file mode 100644 index 00000000..a9de5c1f --- /dev/null +++ b/.claude/rules/wiki/core-change.md @@ -0,0 +1,17 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/change/**" + - "src/cli/change/**" + - "tests/core/change/**" + - "tests/integration/change/**" +--- + +Domain: core-change. Versioned DB changes: scaffold, parse, execute, revert, history (timestamps hydrated as UTC on pg/mysql). + +Map: + - docs/wiki/core-change.md +Contracts: + - docs/spec/v1-45-rewind-tiebreak.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-db.md b/.claude/rules/wiki/core-db.md new file mode 100644 index 00000000..9c44baa7 --- /dev/null +++ b/.claude/rules/wiki/core-db.md @@ -0,0 +1,28 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/db/**" + - "src/core/connection/**" + - "src/core/explore/**" + - "src/core/teardown/**" + - "src/core/transfer/**" + - "src/cli/db/**" + - "tests/core/connection/**" + - "tests/core/db/**" + - "tests/core/explore/**" + - "tests/core/teardown/**" + - "tests/core/transfer/**" + - "tests/integration/connection/**" + - "tests/integration/error-diagnostics/**" + - "tests/integration/explore/**" + - "tests/integration/impersonate/**" + - "tests/integration/teardown/**" + - "tests/integration/transfer/**" +--- + +Domain: core-db. DB lifecycle: create/drop, connections and session helpers, explore schema, teardown, cross-DB transfer. + +Map: + - docs/wiki/core-db.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-identity.md b/.claude/rules/wiki/core-identity.md new file mode 100644 index 00000000..54acf0d7 --- /dev/null +++ b/.claude/rules/wiki/core-identity.md @@ -0,0 +1,24 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/identity/**" + - "src/core/vault/**" + - "src/core/logger/**" + - "src/core/sql-terminal/**" + - "src/cli/identity/**" + - "src/cli/secret/**" + - "src/cli/vault/**" + - "src/cli/sql/**" + - "tests/core/identity/**" + - "tests/core/vault/**" + - "tests/core/logger/**" + - "tests/core/sql-terminal/**" + - "tests/integration/sql-terminal/**" +--- + +Domain: core-identity. Identity keypairs, vault secrets, structured logger, SQL terminal history. + +Map: + - docs/wiki/core-identity.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-policy.md b/.claude/rules/wiki/core-policy.md new file mode 100644 index 00000000..b1b9e4f5 --- /dev/null +++ b/.claude/rules/wiki/core-policy.md @@ -0,0 +1,17 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/policy/**" + - "tests/core/policy/**" +--- + +Domain: core-policy. Access-control policy: role×permission matrix, SQL statement classifier, legacy `protected`→`access` migration. + +Map: + - docs/wiki/core-policy.md +Contracts: + - docs/spec/config-access-roles.md +Designs: + - docs/design/config-access-roles.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-runner.md b/.claude/rules/wiki/core-runner.md new file mode 100644 index 00000000..766e925d --- /dev/null +++ b/.claude/rules/wiki/core-runner.md @@ -0,0 +1,17 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/runner/**" + - "src/core/template/**" + - "src/cli/run/**" + - "tests/core/runner/**" + - "tests/core/template/**" + - "tests/integration/runner/**" +--- + +Domain: core-runner. SQL file execution with checksum dedup, Eta templating, and long-running statement progress and cancel. + +Map: + - docs/wiki/core-runner.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/core-state.md b/.claude/rules/wiki/core-state.md new file mode 100644 index 00000000..9237745d --- /dev/null +++ b/.claude/rules/wiki/core-state.md @@ -0,0 +1,30 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/state/**" + - "src/core/settings/**" + - "src/core/config/**" + - "src/core/lifecycle/**" + - "src/core/version/**" + - "src/core/project.ts" + - "src/core/project-init.ts" + - "src/core/environment.ts" + - "src/core/observer.ts" + - "tests/core/state/**" + - "tests/core/settings/**" + - "tests/core/config/**" + - "tests/core/lifecycle/**" + - "tests/core/version/**" + - "tests/integration/version/**" +--- + +Domain: core-state. Encrypted state, settings.yml, config resolution, lifecycle, version migration, observer events. + +Map: + - docs/wiki/core-state.md +Contracts: + - docs/spec/config-access-roles.md +Designs: + - docs/design/config-access-roles.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/infra.md b/.claude/rules/wiki/infra.md new file mode 100644 index 00000000..1f558620 --- /dev/null +++ b/.claude/rules/wiki/infra.md @@ -0,0 +1,18 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - ".github/**" + - "scripts/**" + - "examples/**" + - "docs/**" + - "tsup.*.config.ts" + - "docker-compose.test.yml" + - "bunfig.toml" +--- + +Domain: infra. CI, build pipeline, binary release, example projects, VitePress docs. + +Map: + - docs/wiki/infra.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/mcp-rpc.md b/.claude/rules/wiki/mcp-rpc.md new file mode 100644 index 00000000..75b4b19f --- /dev/null +++ b/.claude/rules/wiki/mcp-rpc.md @@ -0,0 +1,16 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/mcp/**" + - "src/rpc/**" + - "src/cli/mcp/**" + - "tests/core/mcp/**" + - "tests/core/rpc/**" +--- + +Domain: mcp-rpc. MCP server over stdio wrapping flat RPC command registry, permission-gated dispatch. + +Map: + - docs/wiki/mcp-rpc.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/sdk.md b/.claude/rules/wiki/sdk.md new file mode 100644 index 00000000..1dd120aa --- /dev/null +++ b/.claude/rules/wiki/sdk.md @@ -0,0 +1,22 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/sdk/**" + - "src/core/dt/**" + - "packages/sdk/**" + - "tests/sdk/**" + - "tests/integration/sdk/**" +--- + +Domain: sdk. Programmatic API (`createContext`) + DT binary serialization format. + +Map: + - docs/wiki/sdk.md +Contracts: + - docs/spec/sdk-with-schema.md +References: + - docs/reference/sdk.md +Designs: + - docs/design/sdk-with-schema.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/tui.md b/.claude/rules/wiki/tui.md new file mode 100644 index 00000000..f6c482f6 --- /dev/null +++ b/.claude/rules/wiki/tui.md @@ -0,0 +1,20 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/tui/**" + - ".claude/rules/tui-development.md" + - "tests/cli/components/**" + - "tests/cli/hooks/**" + - "tests/cli/screens/**" +--- + +Domain: tui. Ink/React TUI with focus manager, keyboard routing, registered screens. + +Map: + - docs/wiki/tui.md +Contracts: + - docs/spec/update-download-progress.md +Designs: + - docs/design/update-download-progress.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/.claude/rules/wiki/worker-bridge.md b/.claude/rules/wiki/worker-bridge.md new file mode 100644 index 00000000..81b34869 --- /dev/null +++ b/.claude/rules/wiki/worker-bridge.md @@ -0,0 +1,15 @@ +--- +# generated by /refresh-wiki — do not hand-edit; regenerated every refresh +paths: + - "src/core/worker-bridge/**" + - "src/workers/**" + - "tests/core/worker-bridge/**" + - "tests/workers/**" +--- + +Domain: worker-bridge. Hub-and-spoke worker threads for DT serialization and DB connection worker. + +Map: + - docs/wiki/worker-bridge.md + +Consult the map before changing behavior here. Behavior changes stale the pages above. Renames or removals stale mentions beyond them: grep the old name across docs/ before shipping. diff --git a/docs/dev/runner.md b/docs/dev/runner.md index c40869a2..ac6584a6 100644 --- a/docs/dev/runner.md +++ b/docs/dev/runner.md @@ -279,6 +279,7 @@ An empty file (or one that contains only comments after stripping `GO`s) is trea | `file:before` | `{ filepath, checksum, configName }` | About to execute a file | | `file:after` | `{ filepath, status, durationMs, error? }` | File execution completed | | `file:skip` | `{ filepath, reason }` | File skipped; `reason` is `'unchanged'` or `'already-run'` | +| `file:progress` | `{ filepath, elapsedMs, sessionId, status }` | File still running past the watch delay; repeats every watch interval. See [Long-Running Statements](#long-running-statements) | | `file:dry-run` | `{ filepath, status, outputPath?, error? }` | File rendered to `tmp/` (or failed to render) | ```typescript @@ -301,6 +302,75 @@ observer.on('build:complete', ({ filesRun, filesSkipped, durationMs }) => { ``` +## Long-Running Statements + +A file runs as one call, so from the client a 20-minute `CREATE INDEX` and a file queued behind another session's lock look the same. `StatementWatcher` ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) tells them apart by asking the server. The runner routes every file's SQL through it with one watcher per run; the change executor uses one watcher per change. + +A file that finishes inside 10 seconds costs one extra query (its session id) and nothing else. A file that runs longer gets a report every 10 seconds until it ends: + +```mermaid +sequenceDiagram + participant R as runner + participant W as StatementWatcher + participant P as pinned connection + participant S as side connection + R->>W: run(filepath, db, fn) + W->>P: SESSION_ID_SQL + W->>P: fn(conn) runs the file + Note over W: 10s delay passes + W->>S: checkout (first slow file only) + loop every 10s until the file ends + W->>S: STATEMENT_PROBES[dialect](sessionId) + W-->>R: emit file:progress + end + P-->>W: file done, timer cleared + R->>W: close() at end of run returns S to the pool +``` + +The file is pinned with `db.connection()` so the session id read first is the session the SQL runs on. A transaction executor (postgres changes) is already one connection and runs as is. The side connection comes from the same pool, is checked out only when a file first runs long, and is held until `close()` so later slow files reuse it. + +`status` in the event is what the server reported: + +| Dialect | Source | Reports | +|---------|--------|---------| +| postgres | `pg_stat_activity`, `pg_blocking_pids()`, `pg_stat_progress_{create_index,vacuum,cluster,copy,analyze}` | state, wait event, blockers with query and age, parallel workers, phase and done/total | +| mssql | `sys.dm_exec_requests` | status, wait type, blocking session, `percent_complete` | +| mysql | `information_schema.processlist`, `sys.innodb_lock_waits`, `sys.schema_table_lock_waits`, `performance_schema.events_stages_current` | thread state, blockers, stage done/estimated | +| sqlite | none | `status` is null; elapsed time only | + +Each part of a probe is attempted on its own. A missing privilege (`VIEW SERVER STATE` on mssql, the `sys` schema on mysql) drops that part of the report, not the report. + +What bites: + +| Condition | Effect | +|-----------|--------| +| Pool has no spare connection (`connection.pool.max: 1`) | The side checkout waits 5 seconds, then the watcher gives it up for the rest of the run: reports carry elapsed time only and cancel stops only between files. A checkout that arrives after its file ended, or after the watcher gave up, is returned to the pool at once | +| Transaction-mode pooler (PgBouncer `pool_mode = transaction`, RDS Proxy, Supabase port 6543) | The session-id read and the file can land on different backends. Reports can describe another client's session and a cancel can stop another client's statement. Point noorm at the database directly, or at a session-mode pooler | +| Postgres change on a transaction | The file already runs on the change's transaction, so no second pin happens; the side connection is still a separate pooled session | + +### Cancellation + +Aborting `RunContext.signal` stops the run. The runner starts no further file, marks the remaining ones skipped, and returns `error: 'Run cancelled'`. What happens to the file already running depends on the dialect: + +| Dialect | Running file | +|---------|--------------| +| postgres | `pg_cancel_backend(pid)` from the side connection; the file fails and its implicit transaction rolls back | +| mysql | `KILL QUERY id` from the side connection | +| mssql | Runs to completion. Kysely's `MssqlDialect` never exposes tedious's `Request`, and `KILL` would end the session, not the request | +| sqlite | Runs to completion. It is in-process on one connection, so there is no second session to send a cancel from | + +A file the signal aborted before its SQL was sent does not start, and is skipped with the rest. The pinned connection goes back to the pool only after an in-flight cancel returns, so a cancel that lands late cannot hit the next query on that session. A last file that completes despite the abort leaves the run successful, because nothing was cut short. + +The session-id queries and the cancel table live in [`src/core/connection/session.ts`](../../src/core/connection/session.ts), shared with the SQL terminal's cancel. + +```typescript +const controller = new AbortController(); + +const result = await runBuild({ ...context, signal: controller.signal }, sqlPath); +// elsewhere: controller.abort(); +``` + + ## Additional Utilities The runner module exports several utility functions: @@ -520,6 +590,7 @@ interface RunContext { config?: Record // Config object for template context secrets?: Record // Config-scoped secrets globalSecrets?: Record // Global secrets from state + signal?: AbortSignal // abort to cancel the run } ``` diff --git a/docs/guide/sql-files/execution.md b/docs/guide/sql-files/execution.md index a79b72fd..dacda022 100644 --- a/docs/guide/sql-files/execution.md +++ b/docs/guide/sql-files/execution.md @@ -236,6 +236,15 @@ PostgreSQL runs a multi-statement file as-is, in one implicit transaction. SQLit For the gory details on what the runner does under the hood, see [MSSQL Batch Handling](/dev/runner#mssql-batch-handling). +## Long-Running Files + +A file that is still running after 10 seconds gets a status report every 10 seconds until it finishes: elapsed time, the progress the database reports for the running command (an index build's phase and block count, for example), and any session it is waiting on for a lock. That tells a slow file apart from a stuck one. The TUI shows the report under the running file, and the log records a `Still running (Ns)` line with the full report attached. + +The report comes from a second pooled connection, opened only when a file first passes 10 seconds and kept until the run ends. With `connection.pool.max: 1` there is no second connection to take, so reports carry elapsed time only. Behind a transaction-mode pooler (PgBouncer transaction mode, RDS Proxy, Supabase port 6543) the report can describe the wrong session; connect directly for builds you want to watch. + +To cancel a build from the TUI, see [Long-Running Files](/tui#long-running-files). + + ## Summary | Command | Purpose | diff --git a/docs/tui.md b/docs/tui.md index f822c8b1..d1277d9a 100644 --- a/docs/tui.md +++ b/docs/tui.md @@ -241,6 +241,31 @@ keypress. Raise `connection.connectTimeoutMs` for a link that is slow but working. +### Long-Running Files + +A file that runs longer than 10 seconds gets a status block under its name, +refreshed every 10 seconds, on the run screens (Build, Exec, Dir, File) and the +change screens (Run, Revert, Fast Forward, Next, Rewind): + +``` +Current: 06_items.sql +Running for 17m05s pid 4412 +CREATE INDEX on items · building index: loading tuples · 4210/9800 (42%) 2 parallel workers +Waiting on pid 4411 (CREATE INDEX items_embedding ..., 17m04s) · Lock: transactionid +``` + +The third line appears when the server reports progress for the running +command (index builds, `VACUUM`, `CLUSTER`, `COPY`, `ANALYZE` on PostgreSQL; +`percent_complete` on SQL Server; stage counters on MySQL). The fourth appears when the file is waiting +on another session's lock. SQLite shows elapsed time only. + +On Run Build and Run Exec, `Escape` pressed once shows a warning; pressed again +within 2 seconds it cancels the run. No further file starts. On PostgreSQL and +MySQL the running statement is cancelled on the server; on SQL Server and +SQLite it finishes first, and the screen says so. Leaving the screen does not +cancel the run. + + ## Screen Reference diff --git a/docs/wiki/core-change.md b/docs/wiki/core-change.md index ceb187a2..9e043b4d 100644 --- a/docs/wiki/core-change.md +++ b/docs/wiki/core-change.md @@ -1,64 +1,151 @@ --- type: Domain -description: Versioned database changes — scaffold, parse, execute, revert, and track history +description: Versioned database changes, scaffold, parse, execute/revert with checksum skip, and history +tags: [core, change-management, sql-execution] --- # core-change ## What it does -Manages versioned database changes: scaffold (create/add/remove/rename/reorder change files on disk), parse (discover + validate change folders), execute (forward/revert with checksum-based skip detection), and history (per-change and per-file execution records). +Changes are versioned, forward/revert SQL migrations tracked by checksum, so a change (or a single file inside it) that already ran with an unchanged checksum is skipped rather than re-executed. That makes `noorm change ff` safe to run repeatedly without re-applying finished work, and lets a database recover from a partial failure without an operator manually diagnosing what already happened. -A change directory holds a `change/` folder, an optional `revert/` folder, an optional `changelog.md`, and SQL or `.txt` manifest files. Execution state is stored in the `__noorm_change__` and `__noorm_executions__` tables ([`src/core/shared/tables.ts`](../../src/core/shared/tables.ts)). +## How it works -## CLI code +`executeChange` and `revertChange` ([`src/core/change/executor.ts`](../../src/core/change/executor.ts)) are the entry points every caller (CLI, TUI, SDK, RPC) funnels through. -- [`src/cli/change/index.ts`](../../src/cli/change/index.ts) — registers the `change` command group: `add|edit|ff|list|next|rm|run|revert|history|rewind|history-detail` -- [`src/cli/change/_prompt.ts`](../../src/cli/change/_prompt.ts) — shared interactive change-name pickers (`selectChangeFromFs`, `selectChangeFromStatus`, `requireTty`) used across the offline (add/edit/rm) and DB-aware (run/revert/rewind/history-detail) commands -- [`src/cli/change/add.ts`](../../src/cli/change/add.ts) — offline; scaffolds a new change via `createChange` -- [`src/cli/change/edit.ts`](../../src/cli/change/edit.ts) — offline; spawns `$EDITOR`/`$VISUAL`/`code` against the change folder -- [`src/cli/change/rm.ts`](../../src/cli/change/rm.ts) — offline; gates on `change:rm` via `checkConfigPolicy` (not `assertPolicy`), then calls `deleteChange` -- [`src/cli/change/run.ts`](../../src/cli/change/run.ts) — applies one named change -- [`src/cli/change/next.ts`](../../src/cli/change/next.ts) — applies the next N pending changes -- [`src/cli/change/ff.ts`](../../src/cli/change/ff.ts) — fast-forward: applies all pending changes; warns rather than fails when the changes directory is missing -- [`src/cli/change/revert.ts`](../../src/cli/change/revert.ts) — reverts one applied change -- [`src/cli/change/rewind.ts`](../../src/cli/change/rewind.ts) — reverts applied changes back to (and including) a named change -- [`src/cli/change/list.ts`](../../src/cli/change/list.ts) — lists all changes with status; an orphaned change appends `, orphaned` inside the same parenthetical (e.g. `myname (success, orphaned)`) -- [`src/cli/change/history.ts`](../../src/cli/change/history.ts) — combined change/revert execution history -- [`src/cli/change/history-detail.ts`](../../src/cli/change/history-detail.ts) — per-file history for one change's operations +`executeChange` passes the policy gate, the content gate, and the checksum gate before a lock is taken. `revertChange` has a different gate pair: the `hasRevertFiles` throw, then `tracker.canRevert`, which checks status rather than checksum. -## Docs +```mermaid +flowchart TD + %% source: src/core/change/executor.ts + A[executeChange] --> B{assertChangePolicy} + B -->|denied| X[throw] + B -->|allowed| C[validateChange / hasRevertFiles] + C --> D{"files.length === 0"} + D -->|yes| X2[throw] + D -->|no| E["validateFilesHaveContent (hasExecutableSql gate)"] + E --> F[computeCombinedChecksum] + F --> G{needsRun / canRevert} + G -->|no| H["emit change:skip"] + G -->|yes| I["lockManager.acquire"] + I --> J[executeFiles] + J --> K["lockManager.release"] +``` -- [`docs/dev/change.md`](../dev/change.md) — developer reference for change internals -- [`docs/guide/changes/overview.md`](../guide/changes/overview.md) — user-facing: what changes are -- [`docs/guide/changes/forward-revert.md`](../guide/changes/forward-revert.md) — forward and revert semantics -- [`docs/guide/changes/history.md`](../guide/changes/history.md) — history querying -- [`docs/cli/run.md`](../cli/run.md) — run command docs for `noorm run` (build/file/dir/files/exec); a separate command family from `noorm change` +A change directory holds a `change/` folder, an optional `revert/` folder, an optional `changelog.md`, and SQL or `.txt` manifest files. Execution state lives in the `__noorm_change__` and `__noorm_executions__` tables. + +### Dry-run and preview bypass the run gate + +`opts.dryRun`/`opts.preview` are checked before `history.needsRun`/`tracker.canRevert` (`executor.ts:174`), so both modes render every file's SQL regardless of whether the change already ran: + +```mermaid +flowchart TD + %% source: src/core/change/executor.ts + A[executeChange] --> B{dryRun or preview} + B -->|dryRun| C[executeDryRun] + B -->|preview| D[executePreview] + B -->|neither| E["continues to needsRun gate"] +``` + +### File execution and the statement watcher + +`executeFiles` picks a path by dialect: non-transactional dialects run `runFileBatch` directly against `context.db`; Postgres (`TRANSACTIONAL_DIALECTS` in `executor.ts`) wraps the same batch in `context.db.transaction()` so a failed change leaves neither DDL nor history rows behind, surfacing the failure only through the returned `ChangeResult` (unwrapped from a thrown `ChangeRollback` sentinel). + +`executeFiles` creates one `StatementWatcher` per call and passes it into `runFileBatch`; each file's SQL runs through `watcher.run(file.path, executor, (conn) => sql.raw(sqlContent).execute(conn))`. The watcher is closed in a `finally` on the non-transactional path and explicitly after the transaction settles on Postgres. + +```mermaid +flowchart TD + %% source: src/core/change/executor.ts + A[executeFiles] --> B[new StatementWatcher] + B --> C{TRANSACTIONAL_DIALECTS.has dialect} + C -->|no| D["runFileBatch(context.db)"] + C -->|yes, postgres| E["context.db.transaction(...)"] + D --> F["watcher.close (finally)"] + E --> G["watcher.close (after settle)"] +``` + +Inside the transaction, `batchResult.status !== 'success'` is the rollback trigger. + +### Per-file skip on retry + +Per file inside `runFileBatch`, `history.needsRunFile` can still skip a file whose checksum already succeeded, even when the overall change checksum changed because a sibling file needed fixing: + +```mermaid +flowchart TD + %% source: src/core/change/executor.ts, src/core/change/history.ts + A["for each expanded file"] --> B{needsRunFile} + B -->|no| C["record status: skipped"] --> A + B -->|yes| D[loadAndRenderFile] + D -->|error| E["record failed, break loop"] + D -->|ok| F["watcher.run(file.path, executor, ...)"] + F -->|error| G["record failed, break loop"] + F -->|ok| H["record success"] --> A +``` + +### Change lifecycle + +A change's `OperationStatus` moves between `pending`, `success`, `failed`, `reverted`, and `stale`, computed by `ChangeManager`/`ChangeHistory` from `__noorm_change__`/`__noorm_executions__` rows. `isPendingChange` ([`src/core/change/types.ts`](../../src/core/change/types.ts)) treats only `pending`, `reverted`, and `stale` as "needs a forward run" for `ff`/`next`, and only when the item is not orphaned; `failed` re-applies only through an explicit run, via `needsRun`'s own `failed` branch. + +```mermaid +stateDiagram-v2 + [*] --> pending + pending --> success: executeChange + success --> reverted: revertChange + reverted --> success: executeChange + success --> stale: teardown (ChangeTracker.markAllAsStale) + stale --> success: executeChange + pending --> failed: SQL failure + reverted --> failed: SQL failure + stale --> failed: SQL failure + failed --> success: executeChange (retry) + success --> failed: SQL failure (re-run on changed checksum) + success --> success: executeChange (checksum changed, re-run) + pending --> stale: teardown (ChangeTracker.markAllAsStale) + failed --> stale: teardown (ChangeTracker.markAllAsStale) + failed --> reverted: revertChange + %% source: src/core/change/types.ts, src/core/change/tracker.ts, src/core/change/history.ts +``` + +`ChangeTracker.markAllAsStale` (`tracker.ts:245`) flips every `success`, `failed`, and `pending` row to `stale` on teardown, not only `success` rows. `revertChange` marks the original as `reverted` only when the revert itself succeeds (`tracker.ts:171-172`, `executor.ts:411`), so a `failed` change can still be reverted (`canRevert` allows it) and the revert's own success is what moves the record to `reverted`. + +`orphaned` is not a status: it is a boolean on `ChangeListItem`, set when a change has DB rows but no folder on disk. Because of the Postgres rollback above, a failed run there leaves the stored status at its prior value. + +## Where it lives + +| Path | Responsibility | +|------|-----------------| +| [`src/core/change/executor.ts`](../../src/core/change/executor.ts) | `executeChange`/`revertChange` entry points; policy gate; per-dialect transactional dispatch; `StatementWatcher` wiring; dry-run and preview modes | +| [`src/core/change/manager.ts`](../../src/core/change/manager.ts) | `ChangeManager`, public API combining parser, history, and executor (`list`, `run`, `next`, `ff`, `revert`, `rewind`) | +| [`src/core/change/parser.ts`](../../src/core/change/parser.ts) | `parseChange`/`discoverChanges`, scans a change folder, validates structure, resolves `.txt` manifests, parses sequence/date prefixes | +| [`src/core/change/scaffold.ts`](../../src/core/change/scaffold.ts) | Creates/deletes/renames/reorders change files and folders on disk | +| [`src/core/change/tracker.ts`](../../src/core/change/tracker.ts) | `ChangeTracker` (extends `Tracker`), `canRevert`, `markAsReverted`, `markAllAsStale` | +| [`src/core/change/history.ts`](../../src/core/change/history.ts) | `ChangeHistory`, `needsRun`/`needsRunFile`, operation/file record CRUD, `hydrateDate` UTC normalization | +| [`src/core/change/types.ts`](../../src/core/change/types.ts) | `Change`, `ChangeContext`, `ChangeOptions`, `ChangeResult`, error classes, `isPendingChange` | +| [`src/core/change/validation.ts`](../../src/core/change/validation.ts) | `validateChangeContent`/`SQL_TEMPLATE`, used only by TUI pre-flight checks, not by the executor's own content gate | +| [`src/cli/change/index.ts`](../../src/cli/change/index.ts) | `change` command group registration (`add`, `edit`, `rm`, `run`, `next`, `ff`, `revert`, `rewind`, `list`, `history`, `history-detail`) | +| [`src/cli/change/_prompt.ts`](../../src/cli/change/_prompt.ts) | Shared interactive change-name picker helper used across the CLI commands, not a command itself | +| `tests/core/change/*.test.ts` | Executor, manager, tracker, history, parser, scaffold, and type-contract tests, plus `executor-retry.test.ts` for per-file skip-on-retry | + +## Constraints + +- Only Postgres (`TRANSACTIONAL_DIALECTS` in `executor.ts`) wraps a change's file execution in a database transaction. MySQL's DDL implicitly commits, MSSQL's GO-batch execution is unverified under a wrapping transaction, and SQLite is excluded so per-file partial success (used by unit tests) keeps working. On MySQL, MSSQL, and SQLite, a change failing partway leaves earlier files' DDL applied and their history rows persisted, since nothing rolls either back. +- `executor.ts`'s pre-execution content gate (`hasExecutableSql`, preceded by the `files.length === 0` throw) checks for any non-blank, non-`--`-comment line; it does not call `validateChangeContent` from `validation.ts`, which is a stale check still used only by the TUI's `ChangeFFScreen`/`ChangeRunScreen`. A stub worded to pass `validateChangeContent` can still fail the executor's own gate, so TUI pre-flight and `noorm change run` can disagree about whether a change is runnable. +- `createChange` always scaffolds a stub file into both `change/` and `revert/`. An empty `change/`+`revert/` pair fails `parseChange`'s validation (`scaffold.ts:146-149`), and without the stub the caller sees that misreported as "change not found" instead of "needs editing". +- Change directory names follow `YYYY-MM-DD-`; a name without a date prefix parses with `date: null`, and `discoverChanges` sorts by raw name (`a.name.localeCompare(b.name)`, `parser.ts:215`), so an undated `add-users` sorts after every `2024-...` change and moves in `ff`/`next` order. Files inside are ordered by `filename.localeCompare` (`parser.ts:440`), not by parsed sequence number, so an unpadded sequence prefix (`2_foo.sql` before `10_bar.sql`) sorts and runs out of numeric order. +- `ChangeHistory.needsRunFile` bounds its lookback at the most recent opposite-direction operation, so a prior success only licenses a per-file skip while no revert/re-apply has happened since. It also retires a prior success once the parent operation's own status is `reverted` or `stale` (`history.ts:494-509`); without both conditions, every apply -> revert -> apply cycle silently no-ops its files instead of re-running them. +- `RESET_MARKER = '__reset__'` is a reserved change name written by `ChangeHistory.recordReset` for teardown audit rows. A user change named `__reset__` collides with it: `getAllStatuses` filters that name out, so the change disappears from `change list` even though it still shows in `getHistory`/`getUnifiedHistory`. +- `history.ts`'s `hydrateDate` normalizes `executed_at` to UTC for Postgres/MySQL (reinterpreted field-by-field) and SQLite (text with `Z` appended); MSSQL is left unmodified on purpose, because its driver's behavior was never measured. On a host whose local zone isn't UTC, MSSQL's `executed_at` can render in the TUI's relative-time display shifted by the host's UTC offset. +- Add a new status to `isPendingChange` only. An inlined copy of the check drifts, and `ff` then reports success while work is still outstanding. +- `DEFAULT_OPTIONS`/`DEFAULT_BATCH` in `executor.ts`/`manager.ts` duplicate `DEFAULT_CHANGE_OPTIONS`/`DEFAULT_BATCH_OPTIONS` from `types.ts` rather than importing them, so a default changed in `types.ts` does not reach `executeChange`/`ChangeManager`. +- Change runs get `file:progress` reports but no server-side cancel: `executeFiles` passes no `signal` into `new StatementWatcher(...)` (`executor.ts:477`), so a running change cannot be aborted mid-file the way a runner-driven run with a signal can. ## Coupling -- Calls `runner`'s checksum utilities (`computeChecksum`, `computeCombinedChecksum` from [`src/core/runner/checksum.ts`](../../src/core/runner/checksum.ts)) — checksum algorithm changes propagate here. -- `ChangeTracker` ([`src/core/change/tracker.ts`](../../src/core/change/tracker.ts)) extends `Tracker` from [`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts) — base tracker changes affect revert/stale logic. -- Reads config via [`src/core/config/`](../../src/core/config) to resolve the active database connection — config schema changes affect `ChangeContext` construction. -- Emits `change:*` events (`change:start`, `change:file`, `change:complete`, `change:skip`, `change:created`, `file:dry-run`) via [`src/core/observer.ts`](../../src/core/observer.ts) — the TUI's `useChangeProgress` hook ([`src/tui/hooks/useChangeProgress.ts`](../../src/tui/hooks/useChangeProgress.ts)) subscribes, consumed by `ChangeNextScreen`, `ChangeFFScreen`, `ChangeRevertScreen`, `ChangeRewindScreen`, `ChangeRunScreen` under [`src/tui/screens/change/`](../../src/tui/screens/change). -- Writes to `__noorm_change__` and `__noorm_executions__` tables defined in [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts) — table renames propagate to executor and history queries. -- `executeChange`/`revertChange` call `assertPolicy` from [`src/core/policy/`](../../src/core/policy) before executing, gated on `change:run`/`change:revert`; [`src/cli/change/rm.ts`](../../src/cli/change/rm.ts) gates `change:rm` separately via `checkConfigPolicy` — `ChangeContext` carries `access`/`channel` for the gate; policy-matrix changes in [`src/core/policy/matrix.ts`](../../src/core/policy/matrix.ts) affect which roles can run/revert/rm changes. -- `ChangeTracker.markAllAsStale` is called from [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) (core-db domain) after a teardown, to mark applied changes as needing re-application. -- [`src/sdk/namespaces/changes.ts`](../../src/sdk/namespaces/changes.ts) wraps `ChangeManager` and the scaffold functions for the programmatic SDK — SDK's `Changes` namespace API shape changes with `ChangeManager`'s public methods. -- [`src/rpc/commands/changes.ts`](../../src/rpc/commands/changes.ts) exposes change operations (e.g. `change_history`) as MCP/RPC commands, delegating to `ctx.noorm.changes` — same SDK surface as above. -- CLI commands in [`src/cli/change/`](../../src/cli/change) call `ChangeManager` + scaffold functions directly — `ChangeManager`/scaffold signature changes require CLI command updates. - -## Conventions worth knowing - -- Change directory names follow `YYYY-MM-DD-` (`DATE_PREFIX_REGEX` in `parser.ts`); a name without a date prefix is parsed with `date: null` and the whole name as `description`. -- Change files are ordered by 3-digit sequence prefix: `NNN_description.{sql,sql.tmpl,txt}` (`SEQUENCE_REGEX`); `.txt` files are manifests referencing other SQL files, resolved in the manifest's own line order (not re-sorted). -- `createChange` always scaffolds one stub file into `change/` and one into `revert/` (`CHANGE_STUB_TEMPLATE` / `REVERT_STUB_TEMPLATE` in `scaffold.ts`) — an empty `change/`+`revert/` pair fails `parseChange`'s validation, so the stub exists purely so the change is runnable immediately. -- `executor.ts`'s pre-execution content gate (`hasExecutableSql`) checks for any non-blank, non-`--`-comment line — it does not call `validateChangeContent` from `validation.ts`. `validation.ts`'s `SQL_TEMPLATE` constant (`'-- TODO: Add SQL statements here\n'`) is a stale exact-match check no longer used at the executor seam; it is still imported and called only by the TUI's `ChangeFFScreen.tsx` and `ChangeRunScreen.tsx` for pre-flight UI checks. -- `DEFAULT_CHANGE_OPTIONS` and `DEFAULT_BATCH_OPTIONS` (`types.ts`) define `force`/`dryRun`/`preview`/`output`/`abortOnError` defaults; `executor.ts` and `manager.ts` each keep their own local copy of the same defaults (`DEFAULT_OPTIONS`, `DEFAULT_BATCH`). -- Error classes (`ChangeValidationError`, `ChangeNotFoundError`, `ChangeAlreadyAppliedError`, `ChangeNotAppliedError`, `ChangeOrphanedError`, `ManifestReferenceError`) extend `Error` with a `name` and structured fields; callers distinguish failure modes by class, not a `code` field. -- Only Postgres wraps a change's file execution in a DB transaction (`TRANSACTIONAL_DIALECTS` in `executor.ts`): MySQL's DDL implicitly commits, MSSQL's GO-batch execution hasn't been verified to compose with a wrapping transaction, and SQLite is excluded so per-file partial success (used by unit tests) keeps working. On a failed Postgres change, neither the DDL nor its history rows persist — the caller still sees the failure via the returned `ChangeResult`, unwrapped from a thrown `ChangeRollback` sentinel. -- `history.ts`'s `hydrateDate` normalizes `executed_at` to UTC: Postgres and MySQL drivers (`pg`, `mysql2`) parse the naive `timestamp`/`datetime2` column in the host's local zone, so their `Date` values are reinterpreted field-by-field as UTC; SQLite returns text and is parsed by appending `Z`. MSSQL (`tedious`) is deliberately left unmodified — not measured, left as-is to avoid a correction in the wrong direction. -- `ChangeStatus`/`ChangeListItem` carry `appliedHistoryId?: number | null` — the `__noorm_change__` row's autoincrement id, used as the true apply-order tiebreaker (over second-precision `appliedAt`) in `ChangeManager.rewind()`. -- `ChangeHistory.needsRunFile` excludes `pending` and `skipped` execution rows from its lookback, and bounds the lookback at the most recent opposite-direction operation's id — a prior success only licenses a per-file skip while no revert/re-apply has happened since. -- `RESET_MARKER = '__reset__'` is a reserved change name: `ChangeHistory.recordReset` writes a `db teardown` audit row under this name so it appears in `getHistory`/`getUnifiedHistory`, but `getAllStatuses` explicitly filters it out so it never appears in `change list`. -- `isPendingChange` (types.ts) is the single shared predicate for "needs a forward run" (`pending`, `reverted`, or `stale` status, and not orphaned) — used by `ChangeManager.next`/`ff` and the CLI's interactive pickers; the file's own doc comment warns this predicate must be updated everywhere at once when a new status is added. +- **core-runner** ([`src/core/runner/`](../../src/core/runner)): `executeFiles` creates one `StatementWatcher` per call and runs every file's SQL through `watcher.run(...)`; the watcher's lifecycle, dialect probes, and cancel semantics belong there ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)). Also calls `computeChecksum`/`computeCombinedChecksum` from [`src/core/runner/checksum.ts`](../../src/core/runner/checksum.ts), checksum algorithm changes propagate here. `ChangeTracker` extends `Tracker` from [`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts), base tracker changes affect revert/stale logic. Also pulls `processFile`/`isTemplate` from [`src/core/template/`](../../src/core/template) to render templated SQL files before execution. +- **core-policy** ([`src/core/policy/`](../../src/core/policy)): `executeChange`/`revertChange` call `assertPolicy`, gated on `change:run`/`change:revert`; [`src/cli/change/rm.ts`](../../src/cli/change/rm.ts) gates `change:rm` separately via `checkConfigPolicy`. Policy-matrix changes in [`src/core/policy/matrix.ts`](../../src/core/policy/matrix.ts) affect which roles can run/revert/rm changes. +- **core-state** ([`src/core/observer.ts`](../../src/core/observer.ts)): emits `change:*` events (`change:start`, `change:file`, `change:complete`, `change:skip`, `change:created`, `file:dry-run`) through the shared observer. +- **core-db**: writes to `__noorm_change__` and `__noorm_executions__` tables; `ChangeTracker.markAllAsStale` is called from [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) after a teardown. +- **tui**: the `useChangeProgress` hook ([`src/tui/hooks/useChangeProgress.ts`](../../src/tui/hooks/useChangeProgress.ts)) subscribes to the watcher's `file:progress` event, consumed by the `Change*Screen` components under [`src/tui/screens/change/`](../../src/tui/screens/change). Those screens import `ChangeHistory`, `discoverChanges`, and `validateChangeContent` directly for read-only status and pre-flight checks. +- **sdk**: [`src/sdk/namespaces/changes.ts`](../../src/sdk/namespaces/changes.ts) wraps `ChangeManager` and the scaffold functions for the programmatic SDK. +- **mcp-rpc**: [`src/rpc/commands/changes.ts`](../../src/rpc/commands/changes.ts) exposes change operations as MCP/RPC commands through the same SDK surface. diff --git a/docs/wiki/core-db.md b/docs/wiki/core-db.md index afdd77bd..7ac932d9 100644 --- a/docs/wiki/core-db.md +++ b/docs/wiki/core-db.md @@ -1,80 +1,154 @@ --- type: Domain -description: Database lifecycle, connection factory, schema exploration, teardown, and cross-database transfer across postgres/mysql/mssql/sqlite +description: Connection, database create/drop, session cancel, schema exploration, teardown, and cross-database transfer for postgres/mysql/mssql/sqlite +tags: [database, connection, transfer] --- # core-db ## What it does -- Owns everything that touches a live database connection: creating/dropping databases ([`src/core/db/`](../../src/core/db)), opening connections per dialect ([`src/core/connection/`](../../src/core/connection)), reading schema metadata ([`src/core/explore/`](../../src/core/explore)), wiping data or dropping objects ([`src/core/teardown/`](../../src/core/teardown)), and moving rows between two databases ([`src/core/transfer/`](../../src/core/transfer)). -- Each of the four operational modules (`db`, `explore`, `teardown`, `transfer`) follows the same dialect-dispatch shape: a `dialects/index.ts` maps `Dialect` (`'postgres' | 'mysql' | 'sqlite' | 'mssql'`) to a per-dialect implementation of a shared `DialectXOperations`/`TeardownDialectOperations`/`TransferDialectOperations` interface. -- [`src/core/transfer/`](../../src/core/transfer) only supports `postgres`, `mysql`, `mssql` (`TRANSFER_SUPPORTED_DIALECTS` in [`src/core/transfer/dialects/index.ts`](../../src/core/transfer/dialects/index.ts)) — sqlite has no dialect module there. -- [`src/cli/db/`](../../src/cli/db) exposes these operations as `noorm db ` via Citty subcommands. - -## Artifacts - -- [`src/core/db/operations.ts`](../../src/core/db/operations.ts) — `checkDbStatus`, `createDb`, `destroyDb`; SQLite's `checkDbStatus` pre-probes existence before `testConnection` because opening a connection to a missing SQLite file auto-creates it. -- [`src/core/db/policy.ts`](../../src/core/db/policy.ts) — `assertDbPolicy`, the shared destructive-lifecycle gate used by `core/db` and `core/teardown` (both reached directly by the TUI and indirectly by the CLI via the SDK). -- [`src/core/db/dual.ts`](../../src/core/db/dual.ts) — `withDualConnection`, generic two-connection lifecycle (connect both, run fn, always cleanup both) used by `transfer` and vault-copy. -- [`src/core/db/dialects/postgres.ts`](../../src/core/db/dialects/postgres.ts), `mysql.ts`, `mssql.ts`, `sqlite.ts` — per-dialect `databaseExists`/`createDatabase`/`dropDatabase`/`getSystemDatabase`. -- [`src/core/connection/factory.ts`](../../src/core/connection/factory.ts) — `createConnection` (retry/backoff via `@logosdx/utils` `retry`, `shouldRetry` skips auth/config failures), `testConnection` (`testServerOnly` tries the target first and falls back to the dialect's system database only when the target does not exist, because a contained MSSQL user cannot open `master`; the system databases are `postgres`, `master`, and none for mysql/sqlite). -- [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) — `ConnectionManager` singleton (`getConnectionManager`); tracks cached (by config name) and ephemeral connections plus `WorkerBridge` instances, closes everything on the `app:shutdown` observer event. -- [`src/core/connection/defaults.ts`](../../src/core/connection/defaults.ts) — `DEFAULT_PORTS` per dialect and the shared `PortSchema` (1-65535) used by `core/config` and `core/settings`. -- [`src/core/connection/dialects/mssql.ts`](../../src/core/connection/dialects/mssql.ts) — `resolveTlsServerName`/`buildTediousOptions`; connecting to MSSQL by IP address needs a synthetic SNI ServerName (`UNVERIFIED_TLS_SERVER_NAME`) because RFC 6066 forbids an IP literal as SNI, and `createMssqlConnection` opens the first pooled connection itself because tedious keeps only the last login error; the dialect records every login error's number and message and hands them to `explainMssqlLoginFailure`. -- [`src/core/connection/errors.ts`](../../src/core/connection/errors.ts) — `explainConnectionError`, applied by `createConnection` to every failure, and `explainMssqlLoginFailure`. Per-dialect tables map a code (SQL Server error number, SQLSTATE, mysql2 code, SQLite code, or a Node network/TLS code found through `cause` and `AggregateError.errors`) to a user message; the codes where the server withholds the reason (18456, 28P01, 1045) list the usual causes. The result is a `DatabaseConnectionError` whose `serverCode`/`serverMessage` keep what the server said, which `createConnection` puts on the `connection:error` event and so into the log. Its messages never say "does not exist" unless a database is missing, because the TUI offers to create one on that phrase. -- [`src/core/connection/dialects/mssql-limit-plugin.ts`](../../src/core/connection/dialects/mssql-limit-plugin.ts) — `MssqlLimitPlugin`, a Kysely `OperationNodeTransformer` that rewrites `LimitNode` → `TopNode` because Kysely 0.28.x's `MssqlQueryCompiler` doesn't override `visitLimit()`. -- [`src/core/connection/dialects/sqlite.ts`](../../src/core/connection/dialects/sqlite.ts) / `sqlite-bun.ts` — `better-sqlite3` vs `bun:sqlite` adapters; `factory.ts` picks the Bun one when `globalThis.Bun` is defined. -- [`src/core/connection/dialects/bun-sqlite.d.ts`](../../src/core/connection/dialects/bun-sqlite.d.ts) — hand-written minimal `bun:sqlite` type declarations, to avoid depending on full `bun-types`. -- [`src/core/explore/operations.ts`](../../src/core/explore/operations.ts) — `fetchOverview`, `fetchList`, `fetchDetail`; overview counts are derived from the same listing calls the detail views use (not separate `COUNT(*)` queries) so the two can't disagree; `__noorm_*` tables are filtered out of `tables`/`indexes`/`foreignKeys`/`triggers` unless `includeNoormTables` is set. -- [`src/core/explore/dialects/postgres.ts`](../../src/core/explore/dialects/postgres.ts), `mysql.ts`, `mssql.ts`, `sqlite.ts` — system-catalog queries per dialect (`information_schema`/`pg_catalog` for postgres, `INFORMATION_SCHEMA` for mysql, `sys.*` for mssql); postgres and mssql each define an `EXCLUDED_SCHEMAS` negative filter, mysql instead scopes every query to `TABLE_SCHEMA = ` via `resolveSchema()` (a positive single-database match, since MySQL has no schema level below the database), sqlite has neither (`assertSchemaSupported` in `operations.ts` rejects a `schema` option on sqlite). -- [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) — `truncateData` (disable FK → truncate → re-enable FK, three separate statement groups so FK re-enable still runs even if truncate fails), `teardownSchema` (drop order: FK constraints → CHECK constraints (mssql only) → procedures → functions → views → tables → types, because MSSQL schema-bound objects hold dependency locks on their tables), `previewTeardown` (dry-run wrapper). -- [`src/core/teardown/dialects/postgres.ts`](../../src/core/teardown/dialects/postgres.ts), `mysql.ts`, `mssql.ts`, `sqlite.ts` — per-dialect DDL generation (`truncateTable`, `dropTable`, `dropView`, `dropFunction`, `dropProcedure`, `dropType`, `dropForeignKey`); only MSSQL implements `dropCheckConstraints` (a CHECK constraint referencing a scalar UDF blocks dropping that function while the table exists — MSSQL error 3729). -- [`src/core/transfer/planner.ts`](../../src/core/transfer/planner.ts) — `planTransfer`; queries source table metadata + FK relations per dialect, topologically sorts tables into dependency order, probes destination schema for missing tables, and (for cross-dialect transfers) builds per-table `columnTypes` via `buildDtSchema` from `core/dt`; `queryMysqlTables` detects the identity column via `INFORMATION_SCHEMA.COLUMNS.EXTRA LIKE '%auto_increment%'`. -- [`src/core/transfer/executor.ts`](../../src/core/transfer/executor.ts) — `executeTransfer`; three per-table strategies picked in `executeTransfer`: `transferTableSameServer` (direct SQL, same dialect + same server + `onConflict: 'fail'` + not cross-dialect), `transferTableCrossDialect` (routes through `DtStreamer`/`createKeysetPager`/`queryDatabaseVersion` from `core/dt`), `transferTableCrossServer` (batched INSERT). -- [`src/core/transfer/same-server.ts`](../../src/core/transfer/same-server.ts) — `isSameServer`; PostgreSQL is *never* same-server (no `dblink`/`postgres_fdw`, so a same-database same-server statement would degenerate to `INSERT INTO t SELECT ... FROM t`); MySQL/MSSQL can query cross-database on one server; SQLite is never same-server. -- [`src/core/transfer/dialects/postgres.ts`](../../src/core/transfer/dialects/postgres.ts), `mysql.ts`, `mssql.ts` — per-dialect FK toggle, identity-insert toggle, sequence reset, conflict-aware INSERT, and direct-transfer SQL builders (no sqlite module — transfer excludes sqlite). -- [`src/core/transfer/events.ts`](../../src/core/transfer/events.ts) — `TransferEvents` observer contract (`transfer:planning`, `transfer:plan:ready`, `transfer:starting`, `transfer:table:before/progress/after`, `transfer:complete`). - -## CLI code - -- [`src/cli/db/index.ts`](../../src/cli/db/index.ts) — registers the `db` command group: `create`, `drop`, `explore`, `reset`, `teardown`, `transfer`, `truncate`. -- [`src/cli/db/create.ts`](../../src/cli/db/create.ts) — `noorm db create`; gates via `checkConfigPolicy(..., 'db:create')` before any status probe (SQLite's probe would otherwise auto-create the file for a denied role). -- [`src/cli/db/drop.ts`](../../src/cli/db/drop.ts) — `noorm db drop`; warns to stderr when `NOORM_CONNECTION_*` env overrides retarget the config's stored database away from what will actually be dropped, gated on `db:destroy`. -- [`src/cli/db/reset.ts`](../../src/cli/db/reset.ts) — `noorm db reset`; thin wrapper requiring `--yes`, delegates to `ctx.noorm.db.reset()` (SDK teardown + build). -- [`src/cli/db/teardown.ts`](../../src/cli/db/teardown.ts) — `noorm db teardown`; `--dry-run`, `--preserve-schemas`; exits 1 (after already dropping objects) if `postScript` was configured but failed to execute. -- [`src/cli/db/truncate.ts`](../../src/cli/db/truncate.ts) — `noorm db truncate`; `--dry-run`, `--preserve`, `--only`. -- [`src/cli/db/transfer.ts`](../../src/cli/db/transfer.ts) — `noorm db transfer`; three mutually-exclusive modes (`--to `, `--export `, `--import `); `.dtzx` export/import prompts for a masked passphrase on an interactive TTY, requires `--passphrase` non-interactively; `fk`/`identity` flags are declared under their positive names so citty's built-in `--no-fk`/`--no-identity` negation works (a `noFk`/`noIdentity` declaration would silently no-op). -- [`src/cli/db/explore.ts`](../../src/cli/db/explore.ts) — `noorm db explore`; bare invocation prints overview counts, subcommands drill into each category. -- [`src/cli/db/explore-tables.ts`](../../src/cli/db/explore-tables.ts), `explore-views.ts`, `explore-procedures.ts`, `explore-functions.ts`, `explore-types.ts`, `explore-indexes.ts`, `explore-fks.ts`, `explore-triggers.ts` — one Citty subcommand per `ExploreCategory`; `views`/`procedures`/`functions`/`types`/`triggers` accept a positional `name` directly for detail view, `tables` instead reaches detail view only via a nested `detail` subcommand (`noorm db explore tables detail `, in `explore-tables-detail.ts`), `indexes`/`fks` are list-only. -- [`src/cli/db/explore-tables-detail.ts`](../../src/cli/db/explore-tables-detail.ts) — `noorm db explore tables detail `, registered as a subcommand of `explore-tables.ts`. - -## Docs - -- [`docs/guide/database/create.md`](../guide/database/create.md) — `noorm db create` walkthrough. -- [`docs/guide/database/explore.md`](../guide/database/explore.md) — `noorm db explore` walkthrough. -- [`docs/guide/database/teardown.md`](../guide/database/teardown.md) — teardown/truncate walkthrough. -- [`docs/guide/database/transfer.md`](../guide/database/transfer.md) — transfer walkthrough. -- [`docs/dev/transfer.md`](../dev/transfer.md) — transfer module design notes (FK ordering, identity preservation, same-server optimization, conflict resolution). +Every live-database command in noorm, from `noorm db create` to `noorm db transfer`, gets one retried, probed connection and a dialect-dispatched create/explore/teardown/transfer through this domain. It opens the connection ([`src/core/connection/`](../../src/core/connection)), creates or drops the database ([`src/core/db/`](../../src/core/db)), reads schema metadata ([`src/core/explore/`](../../src/core/explore)), wipes data or drops objects ([`src/core/teardown/`](../../src/core/teardown)), and moves rows between two databases ([`src/core/transfer/`](../../src/core/transfer)). No connection means nothing else in the domain can run. -## Coupling +Every operational module (`db`, `explore`, `teardown`, `transfer`) dispatches on `Dialect` (`'postgres' | 'mysql' | 'sqlite' | 'mssql'`, [`src/core/connection/types.ts`](../../src/core/connection/types.ts)) through one dialect-operations interface: one `types.ts` interface, one implementation file per dialect under `dialects/`, and a `dialects/index.ts` lookup. [`src/core/transfer/`](../../src/core/transfer) is the one exception: it supports only `postgres`, `mysql`, `mssql` (`TRANSFER_SUPPORTED_DIALECTS`, [`src/core/transfer/dialects/index.ts`](../../src/core/transfer/dialects/index.ts)); sqlite has no dialect module there. -- **core-policy**: every destructive entry point (`assertDbPolicy` in [`src/core/db/policy.ts`](../../src/core/db/policy.ts), `assertPolicy`/`checkConfigPolicy` calls in [`src/core/transfer/index.ts`](../../src/core/transfer/index.ts) and `src/cli/db/*.ts`) resolves against `Permission` values (`db:create`, `db:reset`, `db:destroy`, `db:truncate`, `db:teardown`, `transfer:plan`) and the role matrix defined in [`src/core/policy/matrix.ts`](../../src/core/policy/matrix.ts) and [`src/core/policy/types.ts`](../../src/core/policy/types.ts). Adding a new destructive db operation means adding its permission there first. -- **core-state**: [`src/core/db/operations.ts`](../../src/core/db/operations.ts) and [`src/core/db/dual.ts`](../../src/core/db/dual.ts) call `bootstrapSchema`/`tablesExist`/`ensureSchemaVersion` from [`src/core/version/`](../../src/core/version); [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) subscribes to the `app:shutdown` event from [`src/core/observer.ts`](../../src/core/observer.ts); connection config types come from [`src/core/config/types.ts`](../../src/core/config/types.ts). -- **core-change**: [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) imports `ChangeHistory`/`ChangeTracker` from [`src/core/change/`](../../src/core/change) to mark changes stale and record a reset event when `teardownSchema` is called with `configName`/`executedBy`. -- **sdk** ([`src/core/dt/`](../../src/core/dt)): [`src/core/transfer/planner.ts`](../../src/core/transfer/planner.ts) and `executor.ts` depend on `buildDtSchema`, `DtStreamer`, `createKeysetPager`, `queryDatabaseVersion` from [`src/core/dt/`](../../src/core/dt) for cross-dialect type conversion and streaming — a change to the DT column-type model can break cross-dialect transfer. -- **worker-bridge**: [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) tracks `WorkerBridge` instances (type from [`src/core/worker-bridge/types.ts`](../../src/core/worker-bridge/types.ts)) so they shut down alongside regular connections; [`src/workers/connection.ts`](../../src/workers/connection.ts) (the persistent DB worker) imports `core/connection` to own the actual Kysely instance off the main thread. -- **sdk namespaces** ([`src/sdk/namespaces/db.ts`](../../src/sdk/namespaces/db.ts), `dt.ts`, `transfer.ts`) wrap `core/explore`, `core/teardown`, `core/transfer`, `core/dt` directly; `core/db` is reached only transitively, via `core/transfer/index.ts`'s use of `core/db/dual.ts`'s `withDualConnection` — `db.ts` itself has no `core/db` import. [`src/cli/db/create.ts`](../../src/cli/db/create.ts) and `drop.ts` call `checkDbStatus`/`createDb`/`destroyDb` from `core/db` directly, bypassing the SDK/`withContext` layer entirely. The SDK is otherwise the primary consumer surface for the CLI's `withContext`-based commands (`ctx.noorm.db.*`, `ctx.noorm.transfer.*`). -- **tui**: [`src/tui/hooks/useConnection.ts`](../../src/tui/hooks/useConnection.ts), `useVaultConnection.ts`, and [`src/tui/utils/connection.ts`](../../src/tui/utils/connection.ts), `run-context.ts`, `config-validation.ts`, `change-loader.ts` import `core/connection` directly for the TUI's own connect/validate flows. -- **mcp-rpc**: [`src/rpc/commands/explore.ts`](../../src/rpc/commands/explore.ts) calls into `core/explore` directly. +## How it works + +### Opening a connection + +`createConnection` ([`src/core/connection/factory.ts`](../../src/core/connection/factory.ts)) is the one path every dialect connection takes. It lazy-imports the dialect driver, retries transient failures with backoff, and probes the socket with `SELECT 1` before handing the connection back, because a socket that opens and then goes quiet is invisible to any driver connect timeout. + +```mermaid +sequenceDiagram + participant Caller + participant createConnection + participant openConnection + participant retry + participant Driver + participant ConnectionManager + + Caller->>createConnection: createConnection(config) + createConnection->>openConnection: raceAbort(openConnection(config)) + openConnection->>retry: attempt with backoff + retry->>Driver: createFn(config) + Driver-->>retry: ConnectionResult + retry->>Driver: "SELECT 1" probe + alt probe times out or fails + retry->>Driver: discardConnection + retry-->>openConnection: throw + else probe succeeds + retry-->>openConnection: conn + end + openConnection->>ConnectionManager: track(conn, configName) + openConnection-->>createConnection: trackedConn + createConnection-->>Caller: trackedConn +``` + +`shouldRetry` ([`src/core/connection/factory.ts`](../../src/core/connection/factory.ts)) skips retries for auth, missing-driver, missing-database, and abort failures, and retries only `ECONNREFUSED`/`ETIMEDOUT`/`too many connections`/`connection reset`. Every failure passes through `explainConnectionError` ([`src/core/connection/errors.ts`](../../src/core/connection/errors.ts)), which rewords driver-specific codes into a message that never says "does not exist" unless the database itself is missing, because `testConnection`'s `testServerOnly` mode and the TUI's create-on-missing prompt both key off that exact phrase. + +### Choosing a transfer strategy per table + +`executeTransfer` ([`src/core/transfer/executor.ts`](../../src/core/transfer/executor.ts)) picks a code path for each table in the plan, in this order of preference. Same-server `INSERT ... SELECT` wins only when conflicts can't occur and dialects match; cross-dialect needs resolved `columnTypes`; everything else batches. + +```mermaid +flowchart TD + Start["table in plan.tables"] --> SameServer{"plan.sameServer &&
onConflict == 'fail' &&
!plan.crossDialect"} + SameServer -->|yes| Direct["transferTable
SameServer"] + SameServer -->|no| CrossDialect{"plan.crossDialect &&
tablePlan.columnTypes"} + CrossDialect -->|yes| Dt["transferTable
CrossDialect"] + CrossDialect -->|no| Batch["transferTable
CrossServer"] +``` + +`transferTableSameServer` builds one direct `INSERT ... SELECT` statement. `transferTableCrossDialect` streams rows through `DtStreamer`, paged by `createKeysetPager`, converting column types along the way. `transferTableCrossServer` pages the same way but calls `insertBatch` against the destination, no type conversion involved. + +`isSameServer` ([`src/core/transfer/same-server.ts`](../../src/core/transfer/same-server.ts)) rules PostgreSQL out unconditionally: without `dblink`/`postgres_fdw` it has no way to read a second database, and a same-database same-server statement would degenerate into `INSERT INTO t SELECT ... FROM t`. MySQL and MSSQL qualify when host and port both match after `normalizeHost` folds `127.0.0.1`/`::1`/`localhost.localdomain` into `localhost`. SQLite never qualifies; it has no server. + +### Dropping a schema in dependency order + +`teardownSchema` ([`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts)) drops objects in a fixed order because MSSQL schema-bound objects (`WITH SCHEMABINDING`) hold locks on the tables they reference, and a CHECK constraint referencing a scalar UDF blocks dropping that function while the table still exists (MSSQL error 3729): -## Conventions worth knowing +1. Drop FK constraints. +2. Drop MSSQL CHECK constraints (`dropCheckConstraints`, mssql only), gated on `!keepFunctions` (`teardown/operations.ts:413`) since the CHECK-on-a-UDF dependency is exactly what step 4 needs cleared. +3. Drop procedures, unless `keepProcedures`. +4. Drop functions, unless `keepFunctions`. +5. Drop views, unless `keepViews`. +6. Drop tables. +7. Drop types, unless `keepTypes`. On MSSQL this step is skipped entirely when `keepFunctions` or `keepProcedures` is set (`teardown/operations.ts:504-507`), because the function → TVP → domain-type dependency chain can't be broken safely without `CASCADE`. -- Every dialect-dispatch module (`db`, `explore`, `teardown`, `transfer`) follows the same shape: a `types.ts` interface (`DialectDbOperations`, `DialectExploreOperations`, `TeardownDialectOperations`, `TransferDialectOperations`), one implementation file per dialect under `dialects/`, and a `dialects/index.ts` with a `Record` (or `Partial>` for transfer) lookup plus a `getXOperations(dialect)` accessor. -- `__noorm_*` table names are the noorm-internal tracking-table marker on mysql/sqlite (no schema support); `isNoormTable()` is defined separately in [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) (exported for tests) and [`src/core/explore/operations.ts`](../../src/core/explore/operations.ts) (private), both checking the same prefix. On postgres/mssql, schema migration v2 (**core-state**, [`src/core/version/schema/migrations/v2.ts`](../../src/core/version/schema/migrations/v2.ts)) moves the six tracking tables into a dedicated `noorm` schema with the prefix stripped (`change`, not `__noorm_change__`), so [`src/core/explore/dialects/postgres.ts`](../../src/core/explore/dialects/postgres.ts) and `mssql.ts`'s `EXCLUDED_SCHEMAS` list (`'noorm'` alongside `pg_catalog`/`information_schema`/`pg_toast` or `sys`/`INFORMATION_SCHEMA`/`guest`) is what keeps them out of explore results on those two dialects, not `isNoormTable()`. -- Dialect-specific default schemas are centralized in [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts)'s `DEFAULT_SCHEMAS` (`postgres: 'public'`, `mssql: 'dbo'`) for display-name qualification; MySQL and SQLite have no entry because MySQL's "schema" is the database itself and SQLite has none. -- `CreateDbOptions.precheckedStatus` and `TruncateOptions`/`TeardownOptions`/`DestroyDbOptions.policy` are both optional-but-load-bearing: callers that already ran an equivalent gate (the SDK) omit `policy`; every caller with no gate of its own must supply it. -- `attempt`/`attemptSync` from `@logosdx/utils` wrap operations only where the result is inspected/translated (per [`.claude/rules/typescript.md`](../../.claude/rules/typescript.md)); errors that would just propagate are left unwrapped. -- Tests under [`tests/core/explore/dialects/`](../../tests/core/explore/dialects) use a shared recording harness ([`tests/core/explore/recording-db.ts`](../../tests/core/explore/recording-db.ts), `createRecordingDb`) that builds a real Kysely instance with the dialect's actual adapter/compiler/introspector but a driver that records compiled SQL and replays canned rows per regex-matched rule — this exists because an earlier stub returning `SELECT 1` made wrong `WHERE` predicates structurally undetectable. -- [`tests/integration/`](../../tests/integration) (12 subdirectories: `change`, `cli`, `connection`, `error-diagnostics`, `explore`, `impersonate`, `runner`, `sdk`, `sql-terminal`, `teardown`, `transfer`, `version`) requires live database services and is organized per-dialect within most subdirectories (`postgres.test.ts`, `mysql.test.ts`, `mssql.test.ts`, `sqlite.test.ts` where applicable); several subdirectories (`change`, `runner`, `sdk`, `sql-terminal`, `version`) exercise other domains' code but need a live connection to do so. [`tests/integration/cli/setup.ts`](../../tests/integration/cli/setup.ts) is the shared CLI-integration harness (`noorm()`/`noormJson()` run the built CLI binary via `zx`, `setupTestProject()`/`cleanupTestProject()` manage a per-test SQLite project directory). -- `skipIfNoContainer(dialect)` / `TEST_CONNECTIONS` / `makeTestConfig` / `createTestConnection` / `deployTestSchema` / `seedTestData` (from [`tests/utils/db.ts`](../../tests/utils/db.ts), outside this domain's paths) are the shared fixtures every integration test in this domain uses to skip gracefully when postgres/mysql/mssql containers aren't reachable. +`truncateData` runs a separate sequence (disable FK checks, truncate, re-enable FK checks) and always runs the re-enable phase even when the truncate phase throws, so a mid-truncate failure never leaves FK enforcement off on the destination. + +### Pinning a session to cancel a statement from outside it + +[`src/core/connection/session.ts`](../../src/core/connection/session.ts) holds the per-dialect SQL for reading a connection's own server-side session id (`SESSION_ID_SQL`) and for cancelling a statement a file is running on a different connection (`SERVER_CANCEL`). The runner's statement watcher ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) and the SQL terminal ([`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts)) use it the same way: pin a connection, read its session id before running the statement, then act on that id from a second connection once the first is busy. + +```mermaid +sequenceDiagram + participant Watcher as statement-watcher + participant Primary as pinned connection + participant Side as side connection + participant Server + + Watcher->>Primary: SESSION_ID_SQL[dialect] + Primary->>Server: e.g. "select pg_backend_pid()" + Server-->>Primary: rows + Primary-->>Watcher: readSessionId(rows) + Watcher->>Primary: run statement + Note over Watcher: statement busies Primary,
so the id was read first + Watcher->>Side: #sideConnection() (pool checkout) + Watcher->>Side: SERVER_CANCEL[dialect](side, sessionId) + Side->>Server: e.g. "select pg_cancel_backend($1)" +``` + +`readSessionId` returns `undefined` for anything that is not a positive integer. That check is what keeps the mysql path safe: `SERVER_CANCEL.mysql` builds `kill query ` with `sql.raw`, because `KILL` cannot be prepared, and the id it interpolates has already been forced through this guard. `SESSION_ID_SQL` has an entry for every dialect except sqlite; sqlite is in-process and single-connection, so a second connection has no session to observe. `SERVER_CANCEL` additionally omits `mssql`: tedious exposes `request.cancel()`, but Kysely's `MssqlDialect` never hands out the `Request` object, and `KILL` would end the whole session rather than one statement. `hasServerSideCancel(dialect)` is what a caller checks before wording the outcome as "cancelled" versus "stopped waiting." + +## Where it lives + +| Path | Role | +|------|------| +| [`src/core/connection/factory.ts`](../../src/core/connection/factory.ts) | `createConnection`, `testConnection`, `discardConnection` — retry/backoff, liveness probe, abort handling | +| [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) | `ConnectionManager` singleton (`getConnectionManager`) — cached and tracked connections, `WorkerBridge` instances, closes everything on `app:shutdown` | +| [`src/core/connection/session.ts`](../../src/core/connection/session.ts) | `SESSION_ID_SQL`, `SERVER_CANCEL`, `readSessionId`, `hasServerSideCancel` — per-dialect session-id read and server-side cancel | +| [`src/core/connection/errors.ts`](../../src/core/connection/errors.ts) | `explainConnectionError`, `explainMssqlLoginFailure` — per-dialect error code tables, `DatabaseConnectionError` | +| [`src/core/connection/defaults.ts`](../../src/core/connection/defaults.ts) | `DEFAULT_PORTS`, `PortSchema` shared with `core/config` and `core/settings` | +| `src/core/connection/dialects/*.ts` | Per-dialect connection factories; `mssql.ts` builds TLS/SNI options and tedious login handling, `mssql-limit-plugin.ts` rewrites `LimitNode` to `TopNode` for Kysely's MSSQL compiler | +| [`src/core/db/operations.ts`](../../src/core/db/operations.ts) | `checkDbStatus`, `createDb`, `destroyDb` | +| [`src/core/db/policy.ts`](../../src/core/db/policy.ts) | `assertDbPolicy` — shared destructive-lifecycle gate for `core/db` and `core/teardown` | +| [`src/core/db/dual.ts`](../../src/core/db/dual.ts) | `withDualConnection` — generic two-connection lifecycle used by `transfer` and vault-copy | +| `src/core/db/dialects/*.ts` | Per-dialect `databaseExists`/`createDatabase`/`dropDatabase`/`getSystemDatabase` | +| [`src/core/explore/operations.ts`](../../src/core/explore/operations.ts) | `fetchOverview`, `fetchList`, `fetchDetail`, `fetchRowPeek` | +| [`src/core/explore/peek.ts`](../../src/core/explore/peek.ts) | `peekQuery`, `readPeekRows`, `MAX_PEEK_ROWS` — row-window query builder and cap used by `fetchRowPeek` | +| `src/core/explore/dialects/*.ts` | Per-dialect catalog queries (`information_schema`/`pg_catalog`, `INFORMATION_SCHEMA`, `sys.*`) | +| [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) | `truncateData`, `teardownSchema`, `previewTeardown`, `isNoormTable` | +| `src/core/teardown/dialects/*.ts` | Per-dialect DDL generation; only `mssql.ts` implements `dropCheckConstraints` | +| [`src/core/transfer/planner.ts`](../../src/core/transfer/planner.ts) | `planTransfer` — table metadata, FK dependency graph, topological sort, destination schema probe | +| [`src/core/transfer/executor.ts`](../../src/core/transfer/executor.ts) | `executeTransfer`, `transferTableSameServer`, `transferTableCrossDialect`, `transferTableCrossServer` | +| [`src/core/transfer/same-server.ts`](../../src/core/transfer/same-server.ts) | `isSameServer`, `getDefaultPort` | +| [`src/core/transfer/events.ts`](../../src/core/transfer/events.ts) | `TransferEvents` observer contract | +| `src/core/transfer/dialects/*.ts` | Per-dialect FK toggle, identity-insert toggle, sequence reset, conflict-aware INSERT (no sqlite module) | +| `src/cli/db/*.ts` | `noorm db ` Citty subcommands; `create.ts`/`drop.ts` call `checkDbStatus`/`createDb`/`destroyDb` directly, bypassing the SDK layer | +| `tests/core/{connection,db,explore,teardown,transfer}/`, [`tests/integration/`](../../tests/integration) | Unit coverage per module plus cross-database integration runs | + +## Constraints + +| Constraint | What breaks if ignored | +|---|---| +| `PostgreSQL` is never same-server ([`src/core/transfer/same-server.ts`](../../src/core/transfer/same-server.ts)) | Treating it as same-server would run `INSERT INTO t SELECT ... FROM t`, copying the destination into itself instead of transferring data | +| `readSessionId` requires a positive integer | Skipping the guard would let a driver's unexpected result flow straight into the mysql `kill query ` raw SQL | +| Indexing `SESSION_ID_SQL[dialect]` (no sqlite entry) or `SERVER_CANCEL[dialect]` (no sqlite or mssql entry) without checking for `undefined` first | Throws; the watcher guards both lookups before use (`statement-watcher.ts:176-178`, `226-229`) | +| `explainConnectionError`'s messages never say "does not exist" except for a missing database | The TUI offers to create a database on that exact phrase, and `testConnection`'s server-only fallback keys off it too | +| `truncateData`'s FK-enable phase always executes, even after a disable/truncate failure | Stopping at the first failure would leave FK enforcement off on a dialect like MSSQL, where the per-table `NOCHECK` survives reconnects until manually repaired | +| `teardownSchema` drops FKs, then MSSQL CHECK constraints, then procedures/functions/views, before tables | Dropping tables first fails on MSSQL schema-bound objects, and a CHECK constraint on a scalar UDF blocks dropping that function while its table exists | +| `__noorm_*` prefix matching alone does not exclude noorm's tracking tables on postgres/mssql | On postgres and mssql, noorm's tracking tables live in a `noorm` schema without the `__noorm_` prefix. A prefix-only filter lists them in explore and makes transfer's `listUserTables` ([`src/core/transfer/planner.ts`](../../src/core/transfer/planner.ts)) copy them into the destination; `EXCLUDED_SCHEMAS` (explore) and the `!== 'noorm'` schema filter (transfer) keep them out | +| `policy` is optional on `core/db` and `core/teardown` options (`db/types.ts`, `teardown/types.ts`) | Omitting it skips `assertDbPolicy` entirely (`db/policy.ts:67`); callers that omit it (the SDK `db` namespace, the TUI `Db*Screen` components) gate with `checkConfigPolicy` themselves instead. `core/transfer` has no such gap: its options take `channel?` and every entry point always gates via `assertPolicy(options.channel ?? 'user', ...)` | + +## Coupling + +- **core-runner**: [`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts) imports `SESSION_ID_SQL`, `SERVER_CANCEL`, and `readSessionId` from [`src/core/connection/session.ts`](../../src/core/connection/session.ts) to pin a connection, read its session id, and cancel a running statement's file from a side connection; [`src/core/runner/statement-probes.ts`](../../src/core/runner/statement-probes.ts) polls through its own `STATEMENT_PROBES` table and does not reference `session.ts`. +- **core-identity**: [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts) imports the same `session.ts` exports (plus `hasServerSideCancel`) to cancel a query the SQL terminal is running; [`src/core/vault/copy.ts`](../../src/core/vault/copy.ts) calls `withDualConnection` ([`src/core/db/dual.ts`](../../src/core/db/dual.ts)) to hold source and destination connections open for a vault copy. +- **core-policy**: `assertDbPolicy` ([`src/core/db/policy.ts`](../../src/core/db/policy.ts)), `checkConfigPolicy` ([`src/cli/db/create.ts`](../../src/cli/db/create.ts), [`src/cli/db/drop.ts`](../../src/cli/db/drop.ts)), and `assertPolicy` ([`src/core/transfer/index.ts`](../../src/core/transfer/index.ts)) resolve against `Permission` values (`db:create`, `db:reset`, `db:destroy`, `db:truncate`, `db:teardown`, `transfer:plan`) and the role matrix in [`src/core/policy/matrix.ts`](../../src/core/policy/matrix.ts) and [`src/core/policy/types.ts`](../../src/core/policy/types.ts); [`src/core/explore/operations.ts`](../../src/core/explore/operations.ts) calls `assertPolicy` to gate `fetchRowPeek` on `sql:read`. +- **core-state**: [`src/core/db/operations.ts`](../../src/core/db/operations.ts) and [`src/core/db/dual.ts`](../../src/core/db/dual.ts) call `bootstrapSchema`/`tablesExist`/`ensureSchemaVersion` from [`src/core/version/`](../../src/core/version); [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) subscribes to `app:shutdown` from [`src/core/observer.ts`](../../src/core/observer.ts); connection config types come from [`src/core/config/types.ts`](../../src/core/config/types.ts). +- **core-change**: [`src/core/teardown/operations.ts`](../../src/core/teardown/operations.ts) imports `ChangeHistory`/`ChangeTracker` from [`src/core/change/`](../../src/core/change) to mark changes stale and record a reset event. +- **sdk**: [`src/core/transfer/planner.ts`](../../src/core/transfer/planner.ts) and `executor.ts` depend on `buildDtSchema`, `DtStreamer`, `createKeysetPager`, `queryDatabaseVersion` from [`src/core/dt/`](../../src/core/dt) for cross-dialect type conversion and streaming; [`src/sdk/namespaces/db.ts`](../../src/sdk/namespaces/db.ts), `dt.ts`, `transfer.ts` wrap `core/explore`, `core/teardown`, `core/transfer`, `core/dt` directly, and `core/db` is reached only transitively, through `transfer/index.ts`'s use of `db/dual.ts`. +- **worker-bridge**: [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) tracks `WorkerBridge` instances so they shut down alongside regular connections; [`src/workers/connection.ts`](../../src/workers/connection.ts) imports `core/connection` to own the Kysely instance off the main thread. +- **tui**: [`src/tui/providers/ConnectionProvider.tsx`](../../src/tui/providers/ConnectionProvider.tsx) holds the connection lifecycle at runtime; [`src/tui/hooks/useConnection.ts`](../../src/tui/hooks/useConnection.ts) and `useVaultConnection.ts` import `core/connection` types only, delegating to the provider. [`src/tui/utils/connection.ts`](../../src/tui/utils/connection.ts), `run-context.ts`, `config-validation.ts`, `change-loader.ts` import `core/connection` directly. +- **mcp-rpc**: [`src/rpc/commands/explore.ts`](../../src/rpc/commands/explore.ts) calls into `core/explore` directly. diff --git a/docs/wiki/core-identity.md b/docs/wiki/core-identity.md index af7d2e37..68217b6b 100644 --- a/docs/wiki/core-identity.md +++ b/docs/wiki/core-identity.md @@ -1,107 +1,139 @@ --- type: Domain -description: Cryptographic identity, team vault secrets, structured logging, and ad-hoc SQL terminal execution. +description: Cryptographic identity, team vault secrets, structured logging, and the ad-hoc SQL terminal. +tags: [security, observability] --- # core-identity ## What it does -Manages X25519 cryptographic identity (keypair generation, key storage, CI env bootstrap, audit-identity resolution) used to authenticate users and to encrypt shared secrets. Stores team-shared vault secrets in the database, encrypted with a vault key individually sealed to each user's public key. Captures observer events into a structured, redacted, rotating log file. Executes ad-hoc SQL through Kysely behind an access-policy gate and persists gzip-compressed query history per config. - -## CLI code - -- [`src/core/identity/types.ts`](../../src/core/identity/types.ts) — `Identity` (audit identity: name/email/source), `CryptoIdentity` (keypair + identityHash), `KnownUser`, `SharedConfigPayload` -- [`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts) — `generateKeyPair`, `derivePublicKeyFromPrivate`, `encryptForRecipient`/`decryptWithPrivateKey` (ephemeral X25519 ECDH + HKDF + AES-256-GCM for config sharing), `deriveStateKey`/`encryptState`/`decryptState` (state-file encryption keyed directly off the identity private key) -- [`src/core/identity/hash.ts`](../../src/core/identity/hash.ts) — `computeIdentityHash` (SHA-256 of `email\0name\0machine\0os`), `isValidIdentityHash`, `truncateHash` -- [`src/core/identity/storage.ts`](../../src/core/identity/storage.ts) — reads/writes `~/.noorm/identity.key`, `identity.pub`, `identity.json`; `isValidKeyHex` (88/96 hex-char length check), `validateKeyPermissions` (0600 enforcement, always `true` on win32), `backupKeyPair`; in-memory `setKeyOverride`/`setIdentityOverride` for CI -- [`src/core/identity/factory.ts`](../../src/core/identity/factory.ts) — `detectIdentityDefaults` (git config / OS user), `createCryptoIdentity`, `regenerateKeyPair`, `createIdentityForExistingKeys`, `loadExistingIdentity` -- [`src/core/identity/env.ts`](../../src/core/identity/env.ts) — `loadIdentityFromEnv`; builds a `CryptoIdentity` from `NOORM_IDENTITY_PRIVATE_KEY`/`NOORM_IDENTITY_NAME`/`NOORM_IDENTITY_EMAIL` without touching disk -- [`src/core/identity/resolver.ts`](../../src/core/identity/resolver.ts) — `resolveIdentity` (audit identity priority chain: config override → crypto identity → `NOORM_IDENTITY` env → git → system user), `formatIdentity`/`identityToString` -- [`src/core/identity/provenance.ts`](../../src/core/identity/provenance.ts) — `withAgentProvenance`; appends `(via )` to `executed_by`, truncated to fit the 255-char DB column -- [`src/core/identity/sync.ts`](../../src/core/identity/sync.ts) — `registerIdentity`, `fetchKnownUsers`, `syncIdentity`, `syncIdentityWithConfig`; upserts the current identity into the `identities` table and pulls known users -- [`src/core/identity/index.ts`](../../src/core/identity/index.ts) — barrel export plus `resolveIdentity`'s process-level cache (`cachedIdentity`), `getIdentityForConfig`, `getIdentityWithCrypto`, `waitForIdentityToLoad` -- [`src/core/vault/types.ts`](../../src/core/vault/types.ts) — `EncryptedVaultKey`, `VaultSecret`, `VaultStatus`, `VaultCopyResult`, `VaultPropagationResult`, `PendingVaultUser` -- [`src/core/vault/key.ts`](../../src/core/vault/key.ts) — `generateVaultKey`, `encryptVaultKey`/`decryptVaultKey` (ephemeral X25519 ECDH sealed per-recipient), `encryptSecret`/`decryptSecret` (AES-256-GCM with the vault key) -- [`src/core/vault/storage.ts`](../../src/core/vault/storage.ts) — `initializeVault` (idempotent), `getVaultKey`, `setVaultSecret`/`getVaultSecret`/`getAllVaultSecrets`/`listVaultSecretKeys`/`deleteVaultSecret`/`vaultSecretExists`, `getVaultStatus`; only `initializeVault`/`getVaultKey`/`setVaultSecret`/`listVaultSecretKeys`/`deleteVaultSecret` have `*Checked` policy-gated twins — `getVaultSecret`/`getAllVaultSecrets` are gated indirectly (useless without the key `getVaultKeyChecked` gates, per its doc comment) and are called ungated in production (e.g. [`src/cli/vault/list.ts`](../../src/cli/vault/list.ts)'s `getAllVaultSecrets`); `vaultSecretExists` takes no vault key at all (just checks row existence by key name) and relies instead on a caller-level policy check ([`src/cli/vault/rm.ts`](../../src/cli/vault/rm.ts) gates the surrounding command via `checkVaultPolicy`); `getVaultStatus` has no twin and needs none -- [`src/core/vault/policy.ts`](../../src/core/vault/policy.ts) — `VaultPolicyGate`, `checkVaultPolicy`/`assertVaultPolicy`; wraps `core/policy`'s `checkConfigPolicy`/`assertPolicy` for vault permissions -- [`src/core/vault/propagate.ts`](../../src/core/vault/propagate.ts) — `getUsersWithoutVaultAccess`, `propagateVaultKey`/`propagateVaultKeyTo` (+ `*Checked` twins); per-user failures land in `result.failed` rather than being dropped -- [`src/core/vault/resolve.ts`](../../src/core/vault/resolve.ts) — `resolveVaultKey`, `resolveSecret`/`resolveSecrets` (priority: config-specific local → global local → vault), `buildSecretsContext` (merges vault → global → config-specific for template rendering) -- [`src/core/vault/copy.ts`](../../src/core/vault/copy.ts) — `copyVaultSecrets`; cross-config secret copy over `withDualConnection`, supports `dryRun` and `force` -- [`src/core/vault/events.ts`](../../src/core/vault/events.ts) — `VaultEvents`, merged into `NoormEvents` -- [`src/core/vault/index.ts`](../../src/core/vault/index.ts) — barrel export for the vault module -- [`src/core/logger/types.ts`](../../src/core/logger/types.ts) — `LogLevel`, `EntryLevel`, `LogEntry`, `LoggerConfig`, `DEFAULT_LOGGER_CONFIG` (default file [`.noorm/state/noorm.log`](../../.noorm/state/noorm.log)) -- [`src/core/logger/classifier.ts`](../../src/core/logger/classifier.ts) — `classifyEvent` (regex-pattern event-name → level classification), `shouldLog` -- [`src/core/logger/formatter.ts`](../../src/core/logger/formatter.ts) — `generateMessage` (per-event message templates), `formatEntry`, `serializeEntry`, `sanitizeData` -- [`src/core/logger/color.ts`](../../src/core/logger/color.ts) — `formatColorLine`, `STATUS_ICONS`, `formatDuration`; uses [`src/core/theme.ts`](../../src/core/theme.ts) -- [`src/core/logger/timestamp.ts`](../../src/core/logger/timestamp.ts) — `formatLogTimestamp`/`formatLogTimestampIso`; hand-rolled `Date` formatting to avoid a dayjs dependency on the per-line hot path -- [`src/core/logger/redact.ts`](../../src/core/logger/redact.ts) — `filterData`, `maskValue`, `addMaskedFields`, `redactCredentialsInText` (strips credentials embedded in URIs), `listenForSecrets` (subscribes to `secret:set`/`global-secret:set`) -- [`src/core/logger/rotation.ts`](../../src/core/logger/rotation.ts) — `checkAndRotate`, `parseSize`, `rotateFile`, `cleanupRotatedFiles` -- [`src/core/logger/queue.ts`](../../src/core/logger/queue.ts) — `WriteQueue`; ordered, non-blocking file writes -- [`src/core/logger/logger.ts`](../../src/core/logger/logger.ts) — `Logger` class; subscribes via `observer.queue(/./)`, writes console (JSON/color/plain) and file (always JSON) output, owns the rotation interval and `app:shutdown` cleanup -- [`src/core/logger/init.ts`](../../src/core/logger/init.ts) — `enableAutoLoggerInit`/`disableAutoLoggerInit`/`getInitializedLogger`; defers `Logger` construction until `settings:loaded` fires, forces file logging off (`file: ''`) under `isCi()` -- [`src/core/logger/reader.ts`](../../src/core/logger/reader.ts) — `readLogFile`; parses JSON-Lines log file newest-first, skipping malformed lines -- [`src/core/logger/index.ts`](../../src/core/logger/index.ts) — barrel export for the logger module -- [`src/core/sql-terminal/types.ts`](../../src/core/sql-terminal/types.ts) — `SqlExecutionResult`, `SqlHistoryEntry`, `SqlHistoryFile(Serialized)`, `ClearResult` -- [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts) — `executeRawSqlUnchecked` (ungated Kysely `sql.raw()` execution), `executeRawSql` (classifies the statement via `classifyStatements` and gates it via `assertPolicy` before delegating), `SqlPolicyGate` -- [`src/core/sql-terminal/history.ts`](../../src/core/sql-terminal/history.ts) — `SqlHistoryManager`; per-config history at `.noorm/state/history/.json` plus gzipped per-query results under `.noorm/state/history//` -- [`src/core/sql-terminal/index.ts`](../../src/core/sql-terminal/index.ts) — barrel export; deliberately omits `executeRawSqlUnchecked` so the ungated primitive is never one autocomplete away from a production call site -- [`src/cli/identity/index.ts`](../../src/cli/identity/index.ts) — `noorm identity` command group: `init`, `edit`, `export`, `list` -- [`src/cli/identity/init.ts`](../../src/cli/identity/init.ts) — creates a new identity; `--force --yes` backs up and replaces existing keys and warns that existing `state.enc` is not re-encrypted under the new key -- [`src/cli/identity/edit.ts`](../../src/cli/identity/edit.ts) — updates name/email via `createIdentityForExistingKeys` (recomputes `identityHash`, warns when it changes) -- [`src/cli/identity/export.ts`](../../src/cli/identity/export.ts) — prints the public key for sharing -- [`src/cli/identity/list.ts`](../../src/cli/identity/list.ts) — lists known users synced from connected databases (reads local state, not the vault) -- [`src/cli/secret/index.ts`](../../src/cli/secret/index.ts) — `noorm secret` command group: `list`, `rm`, `set` (config-scoped local secrets, stored in `state.enc`) -- [`src/cli/secret/_policy.ts`](../../src/cli/secret/_policy.ts) — `resolveSecretPolicy`; resolves the target config name and gates it via `checkConfigPolicy`, since `StateManager` itself takes no config object -- [`src/cli/secret/list.ts`](../../src/cli/secret/list.ts), [`src/cli/secret/rm.ts`](../../src/cli/secret/rm.ts), [`src/cli/secret/set.ts`](../../src/cli/secret/set.ts) — list/remove/set a config-scoped secret; `rm`/`set` require `secret:write` and honor `NOORM_YES` via `isYesMode` -- [`src/cli/vault/index.ts`](../../src/cli/vault/index.ts) — `noorm vault` command group: `cp`, `init`, `list`, `propagate`, `rm`, `set` -- [`src/cli/vault/_secret-value.ts`](../../src/cli/vault/_secret-value.ts) — `readSecretValue`; shared `--stdin`-or-positional secret input for `vault set` and `secret set`, strips one trailing newline -- [`src/cli/vault/init.ts`](../../src/cli/vault/init.ts), [`src/cli/vault/list.ts`](../../src/cli/vault/list.ts), [`src/cli/vault/set.ts`](../../src/cli/vault/set.ts), [`src/cli/vault/rm.ts`](../../src/cli/vault/rm.ts) — initialize/list/set/remove vault secrets through `withVaultContext` plus the `*Checked` core entrypoints -- [`src/cli/vault/propagate.ts`](../../src/cli/vault/propagate.ts) — grants vault access to pending identities; shows the pending list before requiring `--yes`, treats any per-user failure as `EXIT.PARTIAL` -- [`src/cli/vault/cp.ts`](../../src/cli/vault/cp.ts) — copies vault secrets between two configs via `copyVaultSecrets`, loading identity/private key directly rather than through `withVaultContext` because the core function manages its own dual connection -- [`src/cli/sql/index.ts`](../../src/cli/sql/index.ts) — `noorm sql` command group: `query`, `history`, `clear`, `repl` -- [`src/cli/sql/_config.ts`](../../src/cli/sql/_config.ts) — `resolveHistoryConfigName`; resolves the config name for history-only commands, decrypting state only when no explicit name or `NOORM_CONFIG` is set -- [`src/cli/sql/query.ts`](../../src/cli/sql/query.ts) — executes one SQL statement (or `--file`) via `executeRawSql`; does not record history -- [`src/cli/sql/history.ts`](../../src/cli/sql/history.ts) — shows persisted history recorded by the interactive terminal only -- [`src/cli/sql/clear.ts`](../../src/cli/sql/clear.ts) — clears history, optionally `--older-than ` -- [`src/cli/sql/repl.ts`](../../src/cli/sql/repl.ts) — launches the Ink TUI directly at the SQL Terminal screen; requires a TTY, rejects `--yes`/`NOORM_YES` - -## Docs - -- [`docs/cli/identity.md`](../cli/identity.md) — `noorm identity` command reference -- [`docs/cli/secret.md`](../cli/secret.md) — `noorm secret` command reference -- [`docs/cli/sql.md`](../cli/sql.md) — `noorm sql` command reference -- [`docs/cli/sql-repl.md`](../cli/sql-repl.md) — `noorm sql repl` command reference -- [`docs/dev/identity.md`](../dev/identity.md) — identity system internals -- [`docs/dev/vault.md`](../dev/vault.md) — vault internals -- [`docs/dev/secrets.md`](../dev/secrets.md) — local secrets internals -- [`docs/dev/logger.md`](../dev/logger.md) — logger internals -- [`docs/dev/sql-terminal.md`](../dev/sql-terminal.md) — SQL terminal internals -- [`docs/guide/environments/vault.md`](../guide/environments/vault.md) — user guide: vault -- [`docs/guide/environments/secrets.md`](../guide/environments/secrets.md) — user guide: local secrets -- [`docs/guide/database/terminal.md`](../guide/database/terminal.md) — user guide: SQL terminal +Every write to the database needs an answer to "who did this", and every shared secret needs an answer to "who can read this". This domain answers both: [`src/core/identity/`](../../src/core/identity) resolves an audit identity (`executed_by` on rows) through a priority chain and, separately, holds the X25519 keypair used as the crypto identity for team-shared config. [`src/core/vault/`](../../src/core/vault) uses that same keypair to seal a database-wide secret so only registered users can unwrap it. [`src/core/logger/`](../../src/core/logger) records what every other domain did as redacted JSON-Lines. [`src/core/sql-terminal/`](../../src/core/sql-terminal) runs ad-hoc SQL through the same policy gate as every other write path. + +The submodules share little code ([`src/core/vault/key.ts`](../../src/core/vault/key.ts) and [`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts) each implement their own `deriveSharedSecret`/`deriveEncryptionKey` pair, and the logger has its own event-handling path), but they group under one page because they back the `noorm identity`, `noorm secret`, `noorm vault`, and `noorm sql` command groups. + +## How it works + +### Audit identity resolution takes the first source present, in priority order + +`resolveIdentity` ([`src/core/identity/resolver.ts`](../../src/core/identity/resolver.ts)) picks the name/email written to `executed_by`. It is a priority chain, not a merge: the first source present wins, and lower sources never mix in. `CryptoIdentity` ([`src/core/identity/types.ts`](../../src/core/identity/types.ts)) holds a public key and an `identityHash`; the matching private key loads separately, through `loadPrivateKey` ([`src/core/identity/storage.ts`](../../src/core/identity/storage.ts)). [`src/core/identity/factory.ts`](../../src/core/identity/factory.ts) creates a `CryptoIdentity`, [`src/core/identity/storage.ts`](../../src/core/identity/storage.ts) loads one from disk, and the object enters the resolution chain as one possible input, `cryptoIdentity`. It is also what [`src/core/vault/`](../../src/core/vault) and config-sharing encryption use instead of the audit identity. + +```mermaid +flowchart TD + A[resolveIdentity] --> B{configIdentity?} + B -->|yes| R1["parseIdentityString - config"] + B -->|no| C{cryptoIdentity?} + C -->|yes| R2[cryptoIdentityToAuditIdentity] + C -->|no| D{NOORM_IDENTITY env?} + D -->|yes| R3["parseIdentityString - env"] + D -->|no| E{skipGit?} + E -->|no| F[getGitIdentity] + F -->|found| R4[getGitIdentity] + F -->|null| R5[getSystemIdentity] + E -->|yes| R5 +``` + +### Vault key sealing repeats one ephemeral-ECDH pattern per recipient + +[`src/core/vault/key.ts`](../../src/core/vault/key.ts)'s `encryptVaultKey` and [`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts)'s `encryptForRecipient` each implement this independently, with a distinct HKDF `info` string per use (`'noorm-vault-key'` vs `'noorm-config-share'`). + +```mermaid +sequenceDiagram + participant Sender + participant Recipient + Sender->>Sender: generateKeyPairSync('x25519') (ephemeral) + Sender->>Sender: diffieHellman(ephemeralPrivate, recipientPubKey) + Sender->>Sender: hkdfSync(sha256, sharedSecret, info) -> 32-byte key + Sender->>Sender: createCipheriv('aes-256-gcm').encrypt(vaultKey) + Sender-->>Recipient: ephemeralPubKey, iv, authTag, ciphertext + Recipient->>Recipient: diffieHellman(recipientPrivateKey, ephemeralPubKey) + Recipient->>Recipient: hkdfSync(sha256, sharedSecret, info) -> same 32-byte key + Recipient->>Recipient: createDecipheriv('aes-256-gcm').decrypt(ciphertext) +``` + +Secret values are encrypted once under the shared vault key (`encryptSecret`); only the vault key itself is sealed per recipient with the pattern above. Propagation cannot be revoked (`src/core/vault/types.ts:83`), so [`src/cli/vault/propagate.ts`](../../src/cli/vault/propagate.ts) shows the operator the identity being granted before sealing. + +### SQL cancellation pins one connection, sends the kill from a second + +`executeRawSql` ([`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts)) classifies the statement and asserts policy before delegating to `executeRawSqlUnchecked`, which calls `runQuery`. `runQuery` wraps the pinned-connection path in `raceAbort` ([`src/core/shared/abort.ts`](../../src/core/shared/abort.ts)), and when a dialect supports a server-side cancel, `runWithServerCancel` pins one Kysely connection so the cancel lands on the right backend, using `SESSION_ID_SQL`/`SERVER_CANCEL`/`readSessionId` imported from [`src/core/connection/session.ts`](../../src/core/connection/session.ts). On abort, `raceAbort` rejects with `OperationAbortedError` (`src/core/shared/abort.ts:95-98`); `executeRawSqlUnchecked` catches it and turns it into a `SqlExecutionResult` with `aborted: 'server-cancel-requested'` or `aborted: 'stopped-waiting'` rather than letting it propagate. A policy denial from `assertPolicy` still throws, so a caller only sees a thrown error before execution starts, never once it is running. + +```mermaid +sequenceDiagram + participant Caller + participant U as executeRawSqlUnchecked + participant R as runQuery / raceAbort + participant S as runWithServerCancel + participant Pool as db (pool) + Caller->>U: query, options + U->>R: runQuery + R->>S: pin db.connection() + S->>S: SESSION_ID_SQL[dialect], readSessionId(rows) + S->>S: arming.armed = true, add onAbort listener + S->>S: sql.raw(query).execute(pinned) + Caller-->>R: abort signal fires + R-->>U: raceAbort rejects OperationAbortedError + S-)Pool: onAbort sends SERVER_CANCEL[dialect](db, sessionId) + U-->>Caller: SqlExecutionResult aborted 'server-cancel-requested' +``` + +If `readSessionId` finds no usable id, or the dialect has no entry in `SERVER_CANCEL` (sqlite, mssql), `arming.armed` stays false and the result reports `aborted: 'stopped-waiting'` instead: the abort only stopped the client from waiting. `abortMessageFor` picks the wording for the UI from `hasServerSideCancel(dialect)` alone (a static per-dialect capability check); only `executeRawSqlUnchecked` checks `arming.armed`, the per-call fact of whether a cancel was sent. + +### Every logged event except the logger's own passes one filter/classify/redact path before console and file output + +`Logger#handleEvent` ([`src/core/logger/logger.ts`](../../src/core/logger/logger.ts)) subscribes to every event (`observer.queue(/./)`) but returns immediately on anything prefixed `logger:`, so the logger never logs itself into a loop. + +```mermaid +flowchart TD + A[observer event] --> L{event starts with 'logger:'?} + L -->|yes| Z1[skipped] + L -->|no| B{shouldLog at config level?} + B -->|no| Z2[dropped] + B -->|yes| C[classifyEvent -> EntryLevel] + C --> D{":complete/:after" and status not success/skipped?} + D -->|yes| E[force level: error] + D -->|no| F2{data.error present?} + E --> F2 + F2 -->|yes| G[force level: error] + F2 -->|no| H[filterData - redact] + G --> H + H --> I[generateMessage - MESSAGE_TEMPLATES] + I --> J["writeConsole: json/color/plain"] + I --> K["writeFile: always JSON, mode 0600"] +``` + +`generateMessage` ([`src/core/logger/formatter.ts`](../../src/core/logger/formatter.ts)) has a template per known event, for example `'vault:propagated'` or `'vault:initialized'`; an event with no template falls back to a generic `key=value` join. + +## Where it lives + +| Path | Covers | +|------|--------| +| [`src/core/identity/factory.ts`](../../src/core/identity/factory.ts), `resolver.ts`, `crypto.ts`, `hash.ts`, `env.ts`, `provenance.ts`, `sync.ts`, `storage.ts` | keypair generation, audit-identity resolution chain, config-sharing/state encryption, `identityHash`, CI env bootstrap, provenance/harness lookup, `identities` table sync, private/public key file I/O | +| [`src/core/vault/key.ts`](../../src/core/vault/key.ts), `storage.ts`, `policy.ts`, `propagate.ts`, `resolve.ts`, `copy.ts` | vault key generation/sealing, secret CRUD, `*Checked` policy wrappers (`storage.ts`, `propagate.ts`: `propagateVaultKeyChecked`, `propagateVaultKeyToChecked`), `checkVaultPolicy`/`assertVaultPolicy`/`VaultPolicyGate` (`policy.ts`), propagation to pending users (`propagate.ts`), secret resolution/merge, cross-config copy | +| [`src/core/logger/logger.ts`](../../src/core/logger/logger.ts), `classifier.ts`, `formatter.ts`, `redact.ts`, `rotation.ts`, `queue.ts`, `reader.ts`, `init.ts` | the `Logger` class, event-name-to-level classification, message templates, field redaction, size/count rotation, ordered file writes, JSON-Lines reader, startup wiring against `Settings` | +| [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts), `history.ts` | `executeRawSql`/`executeRawSqlUnchecked`, abort/server-cancel handling, per-config plain JSON history plus gzipped result files | +| [`src/cli/identity/`](../../src/cli/identity), [`src/cli/secret/`](../../src/cli/secret), [`src/cli/vault/`](../../src/cli/vault), [`src/cli/sql/`](../../src/cli/sql) | `noorm identity`, `noorm vault`, `noorm sql` command groups; [`src/cli/secret/`](../../src/cli/secret) reads and writes config-scoped secrets through core-state's `StateManager` rather than through [`src/core/vault/`](../../src/core/vault) | +| [`tests/core/identity/`](../../tests/core/identity), [`tests/core/vault/`](../../tests/core/vault), [`tests/core/logger/`](../../tests/core/logger), [`tests/core/sql-terminal/`](../../tests/core/sql-terminal) | unit coverage, including dedicated edge-case files (`key-file-corruption.test.ts`, `storage-key-permission-guard.test.ts`, `idempotent-init.test.ts`, `policy-gate.test.ts`, `redact-coverage.test.ts`, `rotation-reopen.test.ts`, `executor-abort.test.ts`) | + +## Constraints + +- Raw vault primitives (`getVaultKey`, `setVaultSecret`, `deleteVaultSecret`, `listVaultSecretKeys`, `propagateVaultKey`, `propagateVaultKeyTo`, `initializeVault`) are ungated; every production caller must use the `*Checked` wrapper. Surfaces holding a config (the CLI) use `*Checked`; the SDK gates through its own `#gate` method ([`src/sdk/namespaces/vault.ts`](../../src/sdk/namespaces/vault.ts)). The TUI vault screens (`VaultSetScreen`, `VaultRemoveScreen`, `VaultInitScreen`) call the raw primitives directly, unchecked; only `VaultScreen.tsx` gates its propagate action through `checkConfigPolicy(..., 'vault:propagate')`. Skipping the gate lets a `viewer` role write the vault. `executeRawSqlUnchecked` is the SQL-terminal equivalent, excluded from [`src/core/sql-terminal/index.ts`](../../src/core/sql-terminal/index.ts)'s barrel export so it is never one autocomplete away from a production call site. +- A propagated vault key cannot be revoked (`src/core/vault/types.ts:83`): once sealed to a recipient's public key, that user keeps decrypting every secret, and no key-rotation path exists. +- `isValidKeyHex` ([`src/core/identity/storage.ts`](../../src/core/identity/storage.ts)) requires 88 (SPKI public) or 96 (PKCS8 private) hex characters. A key that fails validation is a hard error at every write/derive site that reads it: `loadPrivateKey` throws, `setKeyOverride` throws, `deriveStateKey` throws. `loadIdentityFromEnv` is the exception: it returns `null` on an invalid key rather than throwing, since CI bootstrap treats a bad env key as "no override" rather than a fatal error. +- Key file permissions: `~/.noorm/identity.key` is written 0600, `identity.pub` 0644. `loadPrivateKey` throws "Insecure permissions on private key file" when the file's group/other bits are set, which blocks every identity-needing command until the file is `chmod 600`'d. `validateKeyPermissions` checks `mode & 0o077 === 0` and always returns `true` on `win32`, since Windows `stat` doesn't reliably report POSIX modes. +- CI identity bootstrap reads `NOORM_IDENTITY_PRIVATE_KEY`/`NOORM_IDENTITY_NAME`/`NOORM_IDENTITY_EMAIL` once via `loadIdentityFromEnv`, then installs the result through in-memory `setKeyOverride`/`setIdentityOverride` so the rest of the process skips disk reads. `computeIdentityHash` hashes `email + name + publicKey` (with `os: 'env'`) for this case instead of the hostname, so every CI runner sharing the same private key resolves to the same identity. +- `withAgentProvenance` ([`src/core/identity/provenance.ts`](../../src/core/identity/provenance.ts)) appends " (via )" to the audit identity when an agent harness is detected; `src/core/shared/operation-id.ts:146` calls it before every insert, so `executed_by` records the harness alongside the human or system identity. +- SQL history is a plain JSON file per config (`SqlHistoryManager`); result rows are gzip-compressed separately. Both are written 0600 (files) / 0700 (dirs) via `HISTORY_FILE_MODE`/`HISTORY_DIR_MODE`, the same permission discipline as `state.enc`. Dropping those modes makes result rows readable by other local users. `noorm sql query` (headless/CI) never writes history; only `sql repl` and the TUI SQL terminal do. +- `noorm sql repl` requires a TTY and rejects `--yes`/`NOORM_YES` outright, since a REPL is interactive by definition and pointing `--yes` at it would silently do nothing useful. +- Log event classification ([`src/core/logger/classifier.ts`](../../src/core/logger/classifier.ts)) is regex-pattern-based on event-name prefix/suffix, not a registry: a new event namespace defaults to `debug` level unless a suffix pattern or an `INFO_PATTERNS` prefix matches it. +- Vault has no local-disk history file; vault secrets live in DB rows only, unlike SQL history and local secrets (`state.enc`), so a lost vault key cannot be recovered from a local cache. ## Coupling -- core-policy: [`src/core/vault/policy.ts`](../../src/core/vault/policy.ts) and [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts) gate every operation through `assertPolicy`/`checkConfigPolicy`/`classifyStatements` from [`src/core/policy/`](../../src/core/policy); [`src/core/identity/provenance.ts`](../../src/core/identity/provenance.ts) reads `AgentHarness` from [`src/core/policy/harness.ts`](../../src/core/policy/harness.ts). Changes to `Permission`, `ConfigAccess`, or the SQL classifier ripple into all four submodules here. -- core-state: [`src/core/vault/resolve.ts`](../../src/core/vault/resolve.ts) takes a `StateManager` for local-secret resolution; [`src/core/logger/init.ts`](../../src/core/logger/init.ts) waits on the `settings:loaded` event and reads the `Settings` type from [`src/core/settings/`](../../src/core/settings); [`src/core/identity/sync.ts`](../../src/core/identity/sync.ts) calls `tablesExist`/`ensureSchemaVersion` from [`src/core/version/`](../../src/core/version); [`src/cli/sql/_config.ts`](../../src/cli/sql/_config.ts) and [`src/cli/secret/_policy.ts`](../../src/cli/secret/_policy.ts) resolve the active config through `initState`/`getStateManager`. `StateManager`'s own encryption key is derived from the identity private key (`deriveStateKey`), so `core-state` cannot decrypt state until this domain has an identity available. -- core-db: [`src/core/identity/sync.ts`](../../src/core/identity/sync.ts) and [`src/core/vault/copy.ts`](../../src/core/vault/copy.ts) open connections via `createConnection`/`withDualConnection` from [`src/core/connection/`](../../src/core/connection) and [`src/core/db/dual.ts`](../../src/core/db/dual.ts); vault and identity storage share the `NoormDatabase`/`noormDb`/`getNoormTables` helpers in [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts). -- sdk: [`src/sdk/namespaces/vault.ts`](../../src/sdk/namespaces/vault.ts), [`src/sdk/namespaces/db.ts`](../../src/sdk/namespaces/db.ts), [`src/sdk/namespaces/lock.ts`](../../src/sdk/namespaces/lock.ts), [`src/sdk/state.ts`](../../src/sdk/state.ts), [`src/sdk/context.ts`](../../src/sdk/context.ts), and [`src/sdk/noorm-ops.ts`](../../src/sdk/noorm-ops.ts) import `core/identity` and `core/vault` directly and re-export a subset as the public `@noormdev/sdk` surface — renaming or reshaping any `*Checked` export here is a breaking SDK change. -- mcp-rpc: [`src/rpc/commands/query.ts`](../../src/rpc/commands/query.ts) calls `executeRawSql` directly, sharing the same `SqlPolicyGate` contract used by `noorm sql query` and the TUI SQL terminal. -- tui: [`src/tui/screens/identity/`](../../src/tui/screens/identity), [`src/tui/screens/vault/`](../../src/tui/screens/vault), [`src/tui/screens/db/SqlTerminalScreen.tsx`](../../src/tui/screens/db/SqlTerminalScreen.tsx) (and sibling Sql*Screen files), and [`src/tui/components/overlays/LogViewerOverlay.tsx`](../../src/tui/components/overlays/LogViewerOverlay.tsx) import these core modules directly and re-implement the CLI's policy-gate pattern in Ink screens. -- cli (shared plumbing): `withContext`/`withVaultContext`, `outputResult`/`outputError`, `sharedArgs`, `isYesMode`, `resolveChannel` used throughout [`src/cli/identity/`](../../src/cli/identity), [`src/cli/secret/`](../../src/cli/secret), [`src/cli/vault/`](../../src/cli/vault), [`src/cli/sql/`](../../src/cli/sql) live in [`src/cli/_utils.ts`](../../src/cli/_utils.ts); exit codes come from [`src/cli/_exit.ts`](../../src/cli/_exit.ts) — both owned by the `cli` domain, not this one. - -## Conventions worth knowing - -- Policy-gate pattern: raw vault primitives (`getVaultKey`, `setVaultSecret`, `deleteVaultSecret`, `listVaultSecretKeys`, `propagateVaultKey`, `propagateVaultKeyTo`, `initializeVault`) are ungated; every production caller must use the `*Checked` wrapper instead. `executeRawSqlUnchecked` is the SQL-terminal equivalent and is deliberately excluded from [`src/core/sql-terminal/index.ts`](../../src/core/sql-terminal/index.ts)'s barrel export. -- Two distinct identity concepts: `Identity` (audit identity — name/email/source, used for `executed_by` tracking, resolution priority config → crypto → `NOORM_IDENTITY` env → git → system user, process-cached in [`src/core/identity/index.ts`](../../src/core/identity/index.ts) unless overrides are passed) versus `CryptoIdentity` (keypair + `identityHash`, used for vault/config-sharing encryption). -- Ephemeral-key encryption pattern is repeated, not shared: config sharing ([`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts)) and vault key sealing ([`src/core/vault/key.ts`](../../src/core/vault/key.ts)) each implement their own `deriveSharedSecret`/`deriveEncryptionKey` — generate an ephemeral X25519 keypair, ECDH with the recipient's public key, HKDF-SHA256 to a 32-byte key, AES-256-GCM — with a distinct HKDF `info` string per use (`'noorm-config-share'`, `'noorm-state-encryption'`, `'noorm-vault-key'`). -- Key material validation: `isValidKeyHex` requires exactly 88 (SPKI public) or 96 (PKCS8 private) hex characters. A key that fails validation is a hard error everywhere it's read (`loadPrivateKey`, `setKeyOverride`, `deriveStateKey`) — never silently truncated, because `Buffer.from(str, 'hex')` truncates at the first invalid pair and a truncated key still HKDFs to a deterministic, publicly-computable output. -- Key file permissions: `~/.noorm/identity.key` is written 0600, `identity.pub` 0644; `validateKeyPermissions` checks `mode & 0o077 === 0` (rejects any group/other bit) and always returns `true` on `win32`, since Windows `stat` doesn't reliably report POSIX modes. -- CI identity bootstrap: `NOORM_IDENTITY_PRIVATE_KEY`/`NOORM_IDENTITY_NAME`/`NOORM_IDENTITY_EMAIL` are read once by `loadIdentityFromEnv`; the CLI entrypoint installs the result via `setKeyOverride`/`setIdentityOverride`, in-memory overrides that make `loadPrivateKey`/`loadIdentityMetadata` skip disk reads for the rest of the process. `computeIdentityHash` deliberately omits `os.hostname()` for the env-loaded case so every CI runner sharing the same private key resolves to the same identity. -- Secret key names: `SECRET_KEY_PATTERN` in [`src/core/vault/storage.ts`](../../src/core/vault/storage.ts) (`/^[A-Za-z][A-Za-z0-9_]*$/`) is deliberately identical to `StateManager.setSecret`'s validation — both feed the same `$.secrets` template namespace. -- Observer event typing is inconsistent across the four submodules: `identity:*`, `sql-terminal:*`, and `logger:*` events are declared inline in `NoormEvents` ([`src/core/observer.ts`](../../src/core/observer.ts)), while `vault:*` gets its own `VaultEvents` interface ([`src/core/vault/events.ts`](../../src/core/vault/events.ts)) merged into `NoormEvents` — vault is the only one of the four with a dedicated events file. -- Log event classification ([`src/core/logger/classifier.ts`](../../src/core/logger/classifier.ts)) is regex-pattern-based on event-name prefix/suffix, not a registry — a new `xyz:` event namespace defaults to `debug` level unless added to `INFO_PATTERNS`/`WARN_PATTERNS`/`ERROR_PATTERNS`/`DEBUG_PATTERNS`. -- SQL history and results are gzip-compressed (`SqlHistoryManager`, [`src/core/sql-terminal/history.ts`](../../src/core/sql-terminal/history.ts)) and written 0600 (files) / 0700 (dirs) — the same permission discipline as `state.enc`. `sql query` (headless/CI) never writes history; only `sql repl` and the TUI SQL terminal do. Vault has no local-disk history file — vault secrets live in DB rows only. -- `sql history`/`sql clear` resolve their target config via `resolveHistoryConfigName` (explicit flag → `NOORM_CONFIG` → active config), decrypting state only when neither of the first two is set — these are the only two SQL-terminal commands that don't require a database connection. -- Test coverage includes dedicated edge-case files beyond the mirrored happy-path tests: [`tests/core/identity/key-file-corruption.test.ts`](../../tests/core/identity/key-file-corruption.test.ts) and [`tests/core/identity/storage-key-permission-guard.test.ts`](../../tests/core/identity/storage-key-permission-guard.test.ts) target malformed-key rejection and the 0600 permission guard; [`tests/core/vault/idempotent-init.test.ts`](../../tests/core/vault/idempotent-init.test.ts) and [`tests/core/vault/policy-gate.test.ts`](../../tests/core/vault/policy-gate.test.ts) cover `initializeVault`'s idempotency and the `*Checked` gate; [`tests/core/logger/redact-coverage.test.ts`](../../tests/core/logger/redact-coverage.test.ts) and [`tests/core/logger/rotation-reopen.test.ts`](../../tests/core/logger/rotation-reopen.test.ts) target redaction field coverage and the re-open-after-rotation behavior in `logger.ts`. +- **core-policy**: [`src/core/vault/policy.ts`](../../src/core/vault/policy.ts) and [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts) gate every operation through `assertPolicy`/`checkConfigPolicy`/`classifyStatements` from [`src/core/policy/`](../../src/core/policy); [`src/core/identity/provenance.ts`](../../src/core/identity/provenance.ts) reads `AgentHarness` from [`src/core/policy/harness.ts`](../../src/core/policy/harness.ts); `resolveChannel` ([`src/core/policy/channel.ts`](../../src/core/policy/channel.ts), re-exported from [`src/core/policy/index.ts`](../../src/core/policy/index.ts)) resolves the acting channel used by policy checks throughout this domain, called from `src/cli/vault/{init,list,propagate,rm,set}.ts` and [`src/cli/secret/_policy.ts`](../../src/cli/secret/_policy.ts). +- **core-db**: [`src/core/connection/session.ts`](../../src/core/connection/session.ts) supplies `SESSION_ID_SQL`, `SERVER_CANCEL`, `hasServerSideCancel`, and `readSessionId`, which [`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts) imports to pin a connection and cancel a running statement from a second one; the runner's statement watcher shares the same module to poll and cancel a running file the same way. [`src/core/identity/sync.ts`](../../src/core/identity/sync.ts) and [`src/core/vault/copy.ts`](../../src/core/vault/copy.ts) open connections via `createConnection`/`withDualConnection` from [`src/core/connection/`](../../src/core/connection) and [`src/core/db/dual.ts`](../../src/core/db/dual.ts); vault and identity storage share the `NoormDatabase`/`noormDb`/`getNoormTables` helpers, which live in [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts) outside any single owning domain. +- **core-state**: [`src/core/vault/resolve.ts`](../../src/core/vault/resolve.ts) takes a `StateManager` for local-secret resolution; [`src/core/logger/init.ts`](../../src/core/logger/init.ts) waits on `settings:loaded` and reads the `Settings` type from [`src/core/settings/`](../../src/core/settings); [`src/core/identity/sync.ts`](../../src/core/identity/sync.ts) calls `tablesExist`/`ensureSchemaVersion` from [`src/core/version/`](../../src/core/version). `StateManager`'s own encryption key is derived from the identity private key (`deriveStateKey`), so `core-state` cannot decrypt state until this domain has an identity available. [`src/core/logger/logger.ts`](../../src/core/logger/logger.ts) subscribes to every event through `observer.queue(/./)` ([`src/core/observer.ts`](../../src/core/observer.ts)); [`src/core/config/types.ts`](../../src/core/config/types.ts) imports `LogLevel` from [`src/core/logger/types.ts`](../../src/core/logger/types.ts). +- **sdk**: [`src/sdk/namespaces/vault.ts`](../../src/sdk/namespaces/vault.ts) imports `core/vault` directly and wraps each call behind its own `#gate` method rather than the CLI's `*Checked` wrappers; [`src/sdk/index.ts`](../../src/sdk/index.ts) calls `loadIdentityFromEnv`, `setKeyOverride`, `setIdentityOverride`, and `getIdentityForConfig` ungated. [`src/sdk/index.ts`](../../src/sdk/index.ts) re-exports the `Identity` type and the vault result/option types `VaultSecret`, `VaultStatus`, `VaultCopyOptions`, `VaultCopyResult`, `VaultPropagationResult`; `VaultAccessError` is the SDK's own class, defined in [`src/sdk/namespaces/vault.ts`](../../src/sdk/namespaces/vault.ts), not this domain. +- **mcp-rpc**: [`src/rpc/commands/query.ts`](../../src/rpc/commands/query.ts) calls `executeRawSql` directly, sharing the same `SqlPolicyGate` contract used by `noorm sql query` and the TUI SQL terminal. +- **tui**: [`src/tui/screens/identity/`](../../src/tui/screens/identity), [`src/tui/screens/vault/`](../../src/tui/screens/vault), [`src/tui/screens/db/SqlTerminalScreen.tsx`](../../src/tui/screens/db/SqlTerminalScreen.tsx), and [`src/tui/components/overlays/LogViewerOverlay.tsx`](../../src/tui/components/overlays/LogViewerOverlay.tsx) import these core modules directly. Only `VaultScreen.tsx` (`vault:propagate`) and `SqlTerminalScreen.tsx` (via `executeRawSql`) go through a policy check; the other vault screens call raw primitives unchecked. +- **cli**: shared plumbing (`withContext`/`withVaultContext`, `outputResult`/`outputError`, `sharedArgs`, `isYesMode`, exit codes) used throughout [`src/cli/identity/`](../../src/cli/identity), [`src/cli/secret/`](../../src/cli/secret), [`src/cli/vault/`](../../src/cli/vault), [`src/cli/sql/`](../../src/cli/sql) lives in [`src/cli/_utils.ts`](../../src/cli/_utils.ts) and [`src/cli/_exit.ts`](../../src/cli/_exit.ts), owned by the `cli` domain. + diff --git a/docs/wiki/core-runner.md b/docs/wiki/core-runner.md index 5e393323..6b5e893e 100644 --- a/docs/wiki/core-runner.md +++ b/docs/wiki/core-runner.md @@ -1,70 +1,126 @@ --- type: Domain -description: SQL file execution with checksum dedup and Eta templating +description: SQL file execution with checksum dedup, Eta templating, and long-running-statement detection +tags: [sql-execution, core] --- # core-runner ## What it does -- Executes `.sql` and `.sql.tmpl` files against a Kysely connection ([`src/core/runner/runner.ts`](../../src/core/runner/runner.ts)), tracking each run in `__noorm_change__`/`__noorm_executions__` via `Tracker` ([`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts)) so unchanged files are skipped on the next run. -- Renders `.sql.tmpl` files through an Eta-based engine ([`src/core/template/engine.ts`](../../src/core/template/engine.ts)) with auto-loaded data side-cars, inherited `$helpers` files, and built-in helpers (`quote`, `escape`, `include`, `json`, `now`, `uuid`). -- Exposes five execution modes — `runBuild`, `runFile`, `runDir`, `runFiles`, `preview` — plus `checkFilesStatus` for pre-execution status categorization ([`src/core/runner/runner.ts`](../../src/core/runner/runner.ts)). - -## Artifacts - -- [`src/core/runner/runner.ts`](../../src/core/runner/runner.ts) — `runBuild`/`runFile`/`runDir`/`runFiles`/`preview`/`checkFilesStatus`/`discoverFiles`/`executeFiles`; the first six form the policy-gated entrypoint set every SDK/TUI/CLI caller funnels through — `discoverFiles` and `executeFiles` are not policy-gated. -- [`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts) — `Tracker` class: `needsRun`, `needsRunByName`, `createOperation`, `recordExecution`, `createFileRecords`, `updateFileExecution`, `finalizeOperation`, `skipRemainingFiles`, `priorSuccessfulExecutions`. -- [`src/core/runner/checksum.ts`](../../src/core/runner/checksum.ts) — `computeChecksum`, `computeChecksumFromContent`, `computeCombinedChecksum` (SHA-256). -- [`src/core/runner/mssql-batches.ts`](../../src/core/runner/mssql-batches.ts) — `splitMssqlBatches` (splits on line-only `GO`), `executeSqlBody` (dialect dispatch: mssql splits on `GO`, sqlite splits on statement boundaries, postgres/mysql execute the body whole). -- [`src/core/runner/sqlite-statements.ts`](../../src/core/runner/sqlite-statements.ts) — `splitSqliteStatements`, a boundary scanner (not a SQL parser) that tracks string/identifier quoting, comments, and `BEGIN`/`CASE`…`END` trigger bodies to find real statement boundaries. -- [`src/core/runner/types.ts`](../../src/core/runner/types.ts) — `RunOptions`, `RunContext`, `FileResult`, `BatchResult`, `NeedsRunResult`, `FileInput`, `ExecuteFilesOptions`, `FilesStatusResult`, and `DEFAULT_RUN_OPTIONS`. -- [`src/core/runner/index.ts`](../../src/core/runner/index.ts) — public export surface for the domain. -- [`src/core/template/engine.ts`](../../src/core/template/engine.ts) — `processFile`, `processFiles`, `renderTemplate`, `isTemplate`; owns the configured `Eta` instance (custom `{% %}` tags, `$` varName, `autoEscape: false`) and the `-- {% %}` directive-line stripping convention. -- [`src/core/template/context.ts`](../../src/core/template/context.ts) — `buildContext` assembles the `$` template context (helpers, auto-loaded data files, config, secrets, `env`, built-ins); `MissingSecretError` and the `$.secrets` proxy that throws on an unresolved key instead of stringifying `undefined`. -- [`src/core/template/helpers.ts`](../../src/core/template/helpers.ts) — `findHelperFiles`/`loadHelpers` walk from a template's directory up to `projectRoot`, merging `$helpers.{ts,js,mjs}` files root-to-leaf (child overrides parent). -- [`src/core/template/loaders/`](../../src/core/template/loaders) — per-extension data loaders: `json5.ts`, `yaml.ts`, `csv.ts` (lazy-imported), `js.ts` (dynamic import, `Bun.build()` bundling path for compiled binaries), `sql.ts`, `dt.ts` (`.dt`/`.dtz`, not `.dtzx`). `loaders/index.ts` registers extensions and marks `.js`/`.mjs`/`.ts` as `isExecutableExtension`. -- [`src/core/template/utils.ts`](../../src/core/template/utils.ts) — `toContextKey` (filename → camelCase), `sqlEscape`, `sqlQuote` (throws `UndefinedSqlValueError` on `undefined`), `isWithinRoot` (segment-aware path containment), `generateUuid`, `isoNow`. -- [`src/core/template/types.ts`](../../src/core/template/types.ts) — `TemplateContext`, `BuiltInHelpers`, `RenderOptions`, `ProcessResult`, `Loader`/`LoaderRegistry`, `DATA_EXTENSIONS`, `TEMPLATE_EXTENSION` (`.tmpl`), `HELPER_FILENAME` (`$helpers`), `HELPER_EXTENSIONS`. - -## CLI code - -- [`src/cli/run/index.ts`](../../src/cli/run/index.ts) — registers the `run` command group with subcommands `build`, `dir`, `exec`, `file`, `files`, `inspect`, `preview`. -- [`src/cli/run/build.ts`](../../src/cli/run/build.ts) — `run build`; runs `ctx.noorm.run.build`, reports `unmatchedInclude`/`unmatchedExclude` warnings and dry-run tmp/ output. -- [`src/cli/run/dir.ts`](../../src/cli/run/dir.ts) — `run dir `; validates the directory exists, reports `EXIT.USAGE` (not success) when zero SQL files are found. -- [`src/cli/run/exec.ts`](../../src/cli/run/exec.ts) — `run exec `; accepts a directory (delegates to `discoverFiles`) or a glob pattern (expanded via `Bun.Glob` when available, else Node's `fs/promises.glob`). -- [`src/cli/run/file.ts`](../../src/cli/run/file.ts) — `run file `; executes a single file via `ctx.noorm.run.file`. -- [`src/cli/run/files.ts`](../../src/cli/run/files.ts) — `run files --paths `; comma-separated file list via `ctx.noorm.run.files`. -- [`src/cli/run/inspect.ts`](../../src/cli/run/inspect.ts) — `run inspect `; builds the template `$` context without rendering, categorizes entries into data files/helpers/builtins, reports helper load errors and secret counts. -- [`src/cli/run/preview.ts`](../../src/cli/run/preview.ts) — `run preview `; renders a `.sql.tmpl` and writes raw SQL to stdout (or `--json`), without executing. -- [`src/cli/run/_render-secrets.ts`](../../src/cli/run/_render-secrets.ts) — `resolveRenderSecrets` shared by `preview`/`inspect`: probes the vault tier with retry disabled so an offline render degrades to local-only secrets (`vaultProbeFailed`) instead of hanging. - -## Docs - -- [`docs/dev/runner.md`](../dev/runner.md) — runner design notes. -- [`docs/dev/template.md`](../dev/template.md) — template engine design notes. -- [`docs/cli/run.md`](../cli/run.md) — `noorm run` subcommand reference. -- [`docs/guide/sql-files/execution.md`](../guide/sql-files/execution.md) — how execution/change-detection works for end users. -- [`docs/guide/sql-files/organization.md`](../guide/sql-files/organization.md) — file/directory ordering conventions. -- [`docs/guide/sql-files/templates.md`](../guide/sql-files/templates.md) — `.sql.tmpl` authoring guide. +Runs `.sql` and `.sql.tmpl` files against a Kysely connection with checksum-based change detection, so a build command run twice only re-executes what changed. `runBuild`, `runFile`, `runDir`, `runFiles`, `preview`, and `checkFilesStatus` in [`src/core/runner/runner.ts`](../../src/core/runner/runner.ts) are the policy-gated entrypoints. It also renders `.sql.tmpl` files through an Eta-based template engine before execution, and watches every file's SQL for how long it runs, so a slow `CREATE INDEX` and a file stuck behind another session's lock read differently to the caller instead of both looking like "still running." + +## How it works + +### Execution and change detection + +A file's status decides whether it runs, and a run always records the outcome before moving to the next file. + +```mermaid +flowchart TD + A[loadAndRenderFile] --> B[computeChecksumFromContent] + B --> C{Tracker.needsRun} + C -->|"new / changed / failed / stale / force / error"| D[runWatched: executeSqlBody via StatementWatcher] + C -->|unchanged| E[skip: skipReason unchanged] + D --> F[Tracker.updateFileExecution] + E --> F +``` + +`executeSingleFileWithUpdate` ([`src/core/runner/runner.ts`](../../src/core/runner/runner.ts)) loads and renders the file first, then recomputes the checksum from the rendered content, because comparing raw `.sql.tmpl` bytes would re-execute every template on every build. The raw `computeChecksum` only seeds the pending row that `createFileRecords` inserts for every file in the batch before the batch starts. [`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts)'s `Tracker.needsRun` excludes that pending row by its own operation id (`excludeOperationId`), or every file would read as "new" forever. A prior `skipped` row with `skip_reason: 'unchanged'` counts as a valid outcome and falls through to the stale and checksum comparison; any other `skipped` row (a cascade skip after an earlier failure, or a cancelled run) and any `pending` row re-runs with `reason: 'new'`. + +### Long-running statement detection + +`StatementWatcher` ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) pins one file's SQL to one connection, reads that connection's session id, and reports back only if the file is still running past a fixed delay. + +```mermaid +sequenceDiagram + participant R as runner.ts + participant W as StatementWatcher + participant P as pinned connection + participant S as side connection + participant O as observer + R->>W: run(filepath, db, fn) + W->>P: SESSION_ID_SQL + W->>P: fn(conn) runs the file + Note over W: 10s delay passes + W->>S: checkout if not already held + loop every 10s until the file ends + W->>S: STATEMENT_PROBES[dialect](sessionId) + W->>O: observer.emit file:progress + end + P-->>W: file done, timer cleared + R->>W: close() at end of run returns S to the pool +``` + +`run()` uses the executor as is when it is already a transaction (a postgres change), otherwise pins a fresh connection with `.connection()` so the session id read up front is the session the SQL runs on. The side connection is checked out on first need and held until `close()`. If the checkout arrives after the file that asked for it has finished, the watcher returns it and resets `#side`, and the next slow file checks out again. `close()` (called once per run, in a `finally`) releases whatever side connection is held back to the pool. + +`STATEMENT_PROBES` ([`src/core/runner/statement-probes.ts`](../../src/core/runner/statement-probes.ts)) supplies the per-dialect status query: postgres reads `pg_stat_activity`/`pg_blocking_pids()`/`pg_stat_progress_*`, mssql reads `sys.dm_exec_requests`, mysql reads `information_schema.processlist` plus the `sys` lock-wait views and `performance_schema.events_stages_current`. sqlite has no probe (in-process, single connection, nothing outside it to ask), so its `file:progress` events carry elapsed time only. Each piece of a probe (activity, blockers, progress) is attempted independently: a missing view or privilege drops that piece, not the whole report. + +### Cancellation + +Aborting `RunContext.signal` sends the dialect's `SERVER_CANCEL` from the side connection, which is checked out on demand if no report has claimed it yet. Nothing is sent when the dialect has no `SERVER_CANCEL` entry (mssql, sqlite) or the session id is null. + +| Dialect | On abort | +|---------|----------| +| postgres | `pg_cancel_backend(pid)` from the side connection; the file fails, implicit transaction rolls back | +| mysql | `KILL QUERY id` from the side connection | +| mssql | Runs to completion; Kysely's `MssqlDialect` never exposes tedious's `Request`, and `KILL` would end the session, not the request | +| sqlite | Runs to completion; in-process, single connection, no second session to cancel from | + +`SERVER_CANCEL` and `SESSION_ID_SQL` live in [`src/core/connection/session.ts`](../../src/core/connection/session.ts), a core-db artifact shared with the SQL terminal's cancel path. `readSessionId` only accepts a positive integer, which is what keeps the mysql `KILL QUERY ${id}` interpolation (not preparable) safe from an unexpected driver value. The runner stops starting new files on abort, marks the rest skipped, and returns `error: 'Run cancelled'`; a file whose SQL was never sent throws `OperationAbortedError` and is treated as not-started rather than failed. + +## Where it lives + +| Path | Role | +|------|------| +| [`src/core/runner/runner.ts`](../../src/core/runner/runner.ts) | `runBuild`/`runFile`/`runDir`/`runFiles`/`preview`/`checkFilesStatus`/`discoverFiles`/`executeFiles`; all except `discoverFiles`/`executeFiles` are gated. `createWatcher` builds one `StatementWatcher` per run and routes every file's SQL through `runWatched`. | +| [`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts) | `StatementWatcher` class: pins a file's connection, reads its session id, times the 10s delay and 10s report interval via `#report`/`#poll`, checks out and holds the side connection via `#sideConnection`, sends the cancel on abort via `#cancel`. | +| [`src/core/runner/statement-probes.ts`](../../src/core/runner/statement-probes.ts) | `STATEMENT_PROBES` (postgres/mssql/mysql `StatementProbe` functions) and the `StatementStatus`/`BlockingSession`/`OperationProgress` shapes a probe returns. | +| [`src/core/runner/tracker.ts`](../../src/core/runner/tracker.ts) | `Tracker` class: `needsRun`, `needsRunByName`, `createOperation`, `recordExecution`, `createFileRecords`, `updateFileExecution`, `finalizeOperation`, `skipRemainingFiles`, `priorSuccessfulExecutions`. | +| [`src/core/runner/checksum.ts`](../../src/core/runner/checksum.ts) | `computeChecksum`, `computeChecksumFromContent`, `computeCombinedChecksum` (SHA-256). | +| [`src/core/runner/mssql-batches.ts`](../../src/core/runner/mssql-batches.ts) | `executeSqlBody` (dialect dispatch: mssql splits on line-only `GO`, sqlite splits on statement boundaries, postgres/mysql execute the body whole), `splitMssqlBatches`. | +| [`src/core/runner/sqlite-statements.ts`](../../src/core/runner/sqlite-statements.ts) | `splitSqliteStatements`, a boundary scanner (not a SQL parser) tracking string/identifier quoting, comments, and `BEGIN`/`CASE`…`END` trigger bodies. | +| [`src/core/runner/types.ts`](../../src/core/runner/types.ts) | `RunOptions`, `RunContext` (including `signal?: AbortSignal`), `FileResult`, `BatchResult`, `NeedsRunResult`, `FileInput`, `ExecuteFilesOptions`, `FilesStatusResult`, `DEFAULT_RUN_OPTIONS`. | +| [`src/core/runner/index.ts`](../../src/core/runner/index.ts) | Public export surface for the domain. | +| [`src/core/template/engine.ts`](../../src/core/template/engine.ts) | `processFile`, `processFiles`, `renderTemplate`, `isTemplate`; owns the configured `Eta` instance (custom `{% %}` tags, `$` varName, `autoEscape: false`) and the `-- {% %}` directive-line stripping convention. | +| [`src/core/template/context.ts`](../../src/core/template/context.ts) | `buildContext` assembles the `$` template context; `MissingSecretError` and the `$.secrets` proxy that throws on an unresolved key. | +| [`src/core/template/helpers.ts`](../../src/core/template/helpers.ts) | `findHelperFiles`/`loadHelpers` walk from a template's directory up to `projectRoot`, merging `$helpers.{ts,js,mjs}` files root-to-leaf. | +| [`src/core/template/loaders/`](../../src/core/template/loaders) | Per-extension data loaders: `json5.ts`, `yaml.ts`, `csv.ts`, `js.ts` (dynamic import, `Bun.build()` bundling for compiled binaries), `sql.ts`, `dt.ts` (`.dt`/`.dtz`). `loaders/index.ts` registers extensions and marks `.js`/`.mjs`/`.ts` as `isExecutableExtension`. | +| [`src/core/template/utils.ts`](../../src/core/template/utils.ts) | `toContextKey`, `sqlEscape`, `sqlQuote` (throws `UndefinedSqlValueError` on `undefined`), `isWithinRoot` (segment-aware path containment), `generateUuid`, `isoNow`. | +| [`src/core/template/types.ts`](../../src/core/template/types.ts) | `TemplateContext`, `BuiltInHelpers`, `RenderOptions`, `ProcessResult`, `Loader`/`LoaderRegistry`, `DATA_EXTENSIONS`, `TEMPLATE_EXTENSION` (`.tmpl`), `HELPER_FILENAME` (`$helpers`), `HELPER_EXTENSIONS`. | +| [`src/cli/run/index.ts`](../../src/cli/run/index.ts) | Registers the `run` command group: `build`, `dir`, `exec`, `file`, `files`, `inspect`, `preview`. | +| [`src/cli/run/build.ts`](../../src/cli/run/build.ts) | `run build`; runs `ctx.noorm.run.build`, reports `unmatchedInclude`/`unmatchedExclude` warnings and dry-run output. | +| [`src/cli/run/dir.ts`](../../src/cli/run/dir.ts) | `run dir `; `EXIT.USAGE` when zero SQL files are found. | +| [`src/cli/run/exec.ts`](../../src/cli/run/exec.ts) | `run exec `; a directory delegates to `discoverFiles`, a glob expands via `Bun.Glob` or Node's `fs/promises.glob`. | +| [`src/cli/run/file.ts`](../../src/cli/run/file.ts) | `run file `; executes a single file via `ctx.noorm.run.file`. | +| [`src/cli/run/files.ts`](../../src/cli/run/files.ts) | `run files --paths `; comma-separated file list via `ctx.noorm.run.files`. | +| [`src/cli/run/inspect.ts`](../../src/cli/run/inspect.ts) | `run inspect `; builds the template `$` context without rendering, categorizes entries, reports helper load errors and secret counts. | +| [`src/cli/run/preview.ts`](../../src/cli/run/preview.ts) | `run preview `; renders a `.sql.tmpl` and writes raw SQL to stdout or `--json`, without executing. | +| [`src/cli/run/_render-secrets.ts`](../../src/cli/run/_render-secrets.ts) | `resolveRenderSecrets` shared by `preview`/`inspect`: probes the vault tier with retry disabled so an offline render degrades to local-only secrets instead of hanging. | +| [`docs/dev/runner.md`](../dev/runner.md) | Runner design notes, including "Long-Running Statements". | +| [`docs/guide/sql-files/execution.md`](../guide/sql-files/execution.md) | End-user execution/change-detection and long-running-file guide. | +| [`tests/core/runner/`](../../tests/core/runner), [`tests/core/template/`](../../tests/core/template), [`tests/integration/runner/`](../../tests/integration/runner) | Unit and integration coverage, including the `StatementWatcher` tests. | + +## Constraints + +- Dropping `excludeOperationId` from `Tracker.needsRun` makes every file read as "new" forever, since the batch's own `pending` rows would be the latest record. +- Comparing raw `.sql.tmpl` bytes instead of the rendered checksum re-executes every template on every build. +- Skipping `StatementWatcher.close()` leaks any side connection checked out during the run; it never returns to the pool. +- With `connection.pool.max: 1`, the side connection checkout waits `SIDE_CONNECTION_WAIT_MS` (5s) then the watcher gives it up for the rest of the run: reports carry elapsed time only, and cancel can only act between files. +- Behind a transaction-mode pooler (PgBouncer `pool_mode = transaction`, RDS Proxy, Supabase port 6543), the session-id read and the file's SQL can land on different backends, so a report or a cancel can target another client's session. Point noorm at the database directly, or at a session-mode pooler. +- `include()` and the `$helpers` directory walk enforce project-root containment via `isWithinRoot` (segment-aware, not `startsWith`), so a sibling directory like `-evil` cannot be traversed into. +- `$.secrets` is a `Proxy` that throws `MissingSecretError` on an unresolved key instead of resolving to `undefined`; `sqlQuote(undefined)` throws `UndefinedSqlValueError` rather than stringifying to the literal text `undefined`. +- Data-file auto-loading in `buildContext` skips `.js`/`.mjs`/`.ts` side-cars unless the template source textually references the resulting context key, so `preview`/`inspect`/`--dry-run` never execute arbitrary code without the user referencing it. +- MSSQL batch splitting and SQLite statement splitting are line/boundary scanners, not SQL parsers: a `GO` alone on a line inside a string literal or `/* */` block splits the MSSQL batch there ([`src/core/runner/mssql-batches.ts`](../../src/core/runner/mssql-batches.ts)), so the file fails or runs a truncated statement. Postgres and mysql receive the full file body via `sql.raw(...)` with no splitting. +- Dry-run output writes rendered SQL, including every resolved secret in plaintext, to `/tmp/`; files and created directories are owner-only (`0o600`/`0o700`), and `tmp/` is not gitignored by `noorm init`. ## Coupling -- **core-change**: `ChangeTracker` ([`src/core/change/tracker.ts`](../../src/core/change/tracker.ts)) extends `Tracker`, giving it constructor-compatible checksum tracking — but core-change does not call the core runner's `executeFiles` or depend on its `ExecuteFilesOptions` contract. [`src/core/change/executor.ts`](../../src/core/change/executor.ts) defines its own private, same-named `executeFiles` function with an unrelated signature (`ChangeContext`/`Change`/`ChangeFile[]`/`direction`/`checksum`/`force`/`history`/`startTime`), and the need-to-run check for change execution is a separate `needsRun` implementation on `ChangeHistory` ([`src/core/change/history.ts`](../../src/core/change/history.ts)), not the inherited `Tracker.needsRun`. -- **core-policy**: every exported entrypoint (`runBuild`/`runFile`/`runDir`/`runFiles`/`preview`/`checkFilesStatus`) gates through `assertPolicy` from [`src/core/policy/index.ts`](../../src/core/policy/index.ts) against the `run:build`/`run:file`/`run:dir` permissions (matrix: viewer deny, operator confirm, admin allow). Adding a new run entrypoint or changing the permission matrix touches both domains. -- **sdk**: [`src/sdk/namespaces/run.ts`](../../src/sdk/namespaces/run.ts) (`RunNamespace`) wraps 6 of the 8 exported runner functions (`discoverFiles`, `preview`, `runFile`, `runFiles`, `runDir`, `runBuild`) and builds `RunContext` (secrets, dialect, identity, access) for every call; `checkFilesStatus` has no `RunNamespace` wrapper and is called directly by the TUI (`RunDirScreen.tsx`, `RunFileScreen.tsx`), bypassing the SDK layer. [`src/sdk/namespaces/templates.ts`](../../src/sdk/namespaces/templates.ts) wraps the template engine for `ctx.noorm.templates`. -- **tui**: `src/tui/screens/run/*.tsx` (`RunBuildScreen`, `RunDirScreen`, `RunExecScreen`, `RunFileScreen`, `RunInspectScreen`) and [`src/tui/utils/run-context.ts`](../../src/tui/utils/run-context.ts) consume the same core runner/template functions as the CLI and SDK. -- **core-state**: emits `build:start`/`build:complete`, `run:file`/`run:dir`/`run:files`, `file:before`/`file:after`/`file:skip`/`file:dry-run`, `template:render`/`template:load`/`template:helpers`, and `error` events, typed in the shared observer at [`src/core/observer.ts`](../../src/core/observer.ts) (a core-state artifact). -- **core-identity**: `formatIdentity` ([`src/core/identity/resolver.ts`](../../src/core/identity/resolver.ts)) stamps `executedBy` on every tracked operation. -- **sdk**: [`src/core/template/loaders/dt.ts`](../../src/core/template/loaders/dt.ts) reads `.dt`/`.dtz` files via `DtReader` from [`src/core/dt/reader.ts`](../../src/core/dt/reader.ts) (the DT binary format lives in the sdk domain). - -## Conventions worth knowing - -- Checksums are computed from *rendered* content for `.sql.tmpl` files, not raw file bytes — `executeSingleFileWithUpdate` recomputes the checksum after rendering and overwrites the pending row's raw-file checksum, because comparing raw bytes made every template re-execute on every build ([`tests/core/runner/template-dedup.test.ts`](../../tests/core/runner/template-dedup.test.ts)). -- `executeFiles` inserts a `pending` execution row for every file in a batch upfront (before any file runs), so `Tracker.needsRun` must exclude the running operation's own id (`excludeOperationId`) or every file reads as "new" forever. -- Dry-run output writes rendered SQL — including every resolved secret in plaintext — to `/tmp/`, mirroring the source path and stripping `.tmpl`; files and any created directories are written owner-only (`mode: 0o600`/`0o700`), and `tmp/` is not gitignored by `noorm init`. -- `run preview`/`run inspect` reuse the `run:file` permission cell rather than a dedicated permission — both resolve every secret tier into plaintext and can execute `$helpers`/side-car scripts even though nothing is written to the database. -- Data-file auto-loading in `buildContext` skips `.js`/`.mjs`/`.ts` side-cars unless the template source textually references the resulting context key (`$.key` or `$['key']`) — otherwise `preview`/`inspect`/`--dry-run` would execute arbitrary code with no way for the user to know. -- `include()` and the `$helpers` directory walk both enforce project-root containment via `isWithinRoot` (segment-aware, not a bare `startsWith`), so a sibling directory like `-evil` cannot be traversed into. -- `$.secrets` is a `Proxy` that throws `MissingSecretError` on an unresolved key instead of resolving to `undefined` — `sqlQuote(undefined)` also throws `UndefinedSqlValueError` rather than stringifying to the literal text `undefined`. -- MSSQL batch splitting (`splitMssqlBatches`) and SQLite statement splitting (`splitSqliteStatements`) are the only two dialects requiring file-content splitting before execution; postgres and mysql receive the full file body via `sql.raw(...)`. +- **core-db**: `SESSION_ID_SQL`, `SERVER_CANCEL`, `readSessionId` in [`src/core/connection/session.ts`](../../src/core/connection/session.ts) are imported by the runner's `StatementWatcher`; `hasServerSideCancel` from the same file is used by [`src/tui/utils/run-context.ts`](../../src/tui/utils/run-context.ts) (`runCancelMessage`) and the SQL terminal ([`src/core/sql-terminal/executor.ts`](../../src/core/sql-terminal/executor.ts)). +- **core-change**: [`src/core/change/executor.ts`](../../src/core/change/executor.ts) builds its own `StatementWatcher` (one per change) rather than sharing the runner's, and defines a private, same-named `executeFiles` with an unrelated signature; it also imports `processFile`/`isTemplate` from [`src/core/template/`](../../src/core/template). It shares `computeChecksum`, `computeCombinedChecksum`, and the `Tracker` base class with the runner. The change watcher gets no `signal` (`src/core/change/executor.ts:477`): progress reporting only, no cancel. +- **core-policy**: `runBuild` gates on `run:build`; `runFile`, `preview`, and `checkFilesStatus` gate on `run:file`; `runDir` and `runFiles` gate on `run:dir`. `discoverFiles`/`executeFiles` do not gate. +- **core-state**: `file:progress` (and `build:start`/`build:complete`, `run:file`/`run:dir`/`run:files`, `file:before`/`file:after`/`file:skip`/`file:dry-run`, `template:*`, `error`) are typed on the shared observer at [`src/core/observer.ts`](../../src/core/observer.ts). +- **core-identity**: `formatIdentity` stamps `executedBy` on every tracked operation. +- **sdk**: [`src/sdk/namespaces/run.ts`](../../src/sdk/namespaces/run.ts) wraps `runBuild`, `runFile`, `runDir`, `runFiles`, `preview`, and `discoverFiles`; it passes no `signal`, so SDK runs cannot be cancelled. +- **tui**: `src/tui/screens/run/*.tsx` and [`src/tui/utils/run-context.ts`](../../src/tui/utils/run-context.ts) consume the same runner/template functions as the CLI and SDK, and are the only callers that set `RunContext.signal` (`RunBuildScreen.tsx:164`, `RunExecScreen.tsx:159`). `StatementProgress` renders `file:progress` reports across both the run screens and the change screens. The TUI calls `checkFilesStatus` directly. +- **mcp-rpc**: [`src/rpc/commands/run.ts`](../../src/rpc/commands/run.ts) calls `ctx.noorm.run.build`/`ctx.noorm.run.file`, so MCP runs pass no `signal` and cannot be cancelled. diff --git a/docs/wiki/core-state.md b/docs/wiki/core-state.md index 2db3046e..3ffc5729 100644 --- a/docs/wiki/core-state.md +++ b/docs/wiki/core-state.md @@ -1,63 +1,144 @@ --- type: Domain -description: Encrypted project state, settings.yml, config resolution, app lifecycle, and schema/state/settings version migration +description: Encrypted state, settings.yml, config resolution, app lifecycle, schema/state/settings version migration, and the shared event bus +tags: [core, state, config] --- # core-state ## What it does -Persists project data across three files with different trust levels: [`.noorm/state/state.enc`](../../.noorm/state/state.enc) (encrypted configs, secrets, known users), `.noorm/settings.yml` (version-controlled build/stage/rule config), and process-lifetime app startup/shutdown coordination. [`src/core/version/`](../../src/core/version) runs three independent migration tracks (schema, state, settings), each keyed on its own integer version number in `CURRENT_VERSIONS` ([`src/core/version/types.ts`](../../src/core/version/types.ts)), separate from the CLI's semver package version. [`src/core/observer.ts`](../../src/core/observer.ts) is the central `ObserverEngine` instance (`@logosdx/observer`) that every domain in the repo emits events through, and whose `NoormEvents` interface aggregates event types from `settings`, `lifecycle`, `update`, `vault`, `transfer`, `dt`, `logger`, and `teardown`. +Every noorm command needs to know which database it is talking to, whether that config is locked down, and whether the on-disk data it is about to read still matches the shape the running build expects. This domain answers all three, split across storage with different trust levels: `.noorm/state/state.enc` (encrypted configs, secrets, known users), `.noorm/settings.yml` (version-controlled build/stage/rule config), and in-process `LifecycleManager` state (startup/shutdown coordination, not persisted). [`src/core/version/`](../../src/core/version) runs a schema-version migration and a semver migration in a fixed order so a project's on-disk data can move forward without forcing every layer to match the CLI's package version. [`src/core/observer.ts`](../../src/core/observer.ts) is the single `ObserverEngine` every other domain emits through, including the runner's `StatementWatcher`, which emits `file:progress` while a long-running statement is still executing, so a caller with no other signal for "it's still running" gets one. -## Artifacts +## How it works -- [`.noorm/state/state.enc`](../../.noorm/state/state.enc) — AES-256-GCM encrypted JSON ([`src/core/state/manager.ts`](../../src/core/state/manager.ts)), mode `0o600`; holds `configs`, `secrets`, `globalSecrets`, `knownUsers`, `activeConfig`, `version`, `schemaVersion`. -- `.noorm/state/state.enc.bak` — previous-generation backup written by `backupExisting` ([`src/core/state/persistence.ts`](../../src/core/state/persistence.ts)) before every overwrite. -- `.noorm/state/state.enc.lock` — advisory `O_EXCL` lock file ([`src/core/state/persistence.ts`](../../src/core/state/persistence.ts)), 5s acquire timeout, 30s staleness threshold. -- `.noorm/settings.yml` — YAML, version controlled, parsed/written by [`src/core/settings/manager.ts`](../../src/core/settings/manager.ts) via the `yaml` package. -- [`.noorm/`](../../.noorm), [`.noorm/state/`](../../.noorm/state), `.noorm/.gitignore`, `sql/`, `changes/`, and the root [`.gitignore`](../../.gitignore) `# noorm` block — created by `src/core/project-init.ts:performProjectInit`. +### Project initialization creates identity before StateManager is constructed -## Docs +`performProjectInit` ([`src/core/project-init.ts`](../../src/core/project-init.ts)) is the one function both the TUI `InitScreen` and `noorm init` call. It creates identity before constructing either manager because `new StateManager(root, { privateKey })` needs the key `createCryptoIdentity` returns; `SettingsManager` takes no key and does not depend on identity at all. -- [`docs/dev/config.md`](../dev/config.md) — config internals. -- [`docs/dev/settings.md`](../dev/settings.md) — settings file reference. -- [`docs/dev/state.md`](../dev/state.md) — state file internals. -- [`docs/dev/version.md`](../dev/version.md) — version migration internals. -- [`docs/dev/project-discovery.md`](../dev/project-discovery.md) — project root detection. -- [`docs/guide/environments/configs.md`](../guide/environments/configs.md) — user guide: configs. -- [`docs/guide/environments/stages.md`](../guide/environments/stages.md) — user guide: stages. -- [`docs/guide/environments/secrets.md`](../guide/environments/secrets.md) — user guide: config-scoped secrets stored in `state.enc`. +```mermaid +sequenceDiagram + participant Init as performProjectInit + participant FS as filesystem + participant Settings as SettingsManager + participant State as StateManager + participant Singleton as getStateManager(root) + + Init->>FS: mkdir sql/, changes/, .noorm/, .noorm/state/ + Init->>FS: write .noorm/.gitignore + alt identityInfo supplied and no keys exist + Init->>Init: createCryptoIdentity() + end + Init->>Settings: init(force) + Settings->>FS: write settings.yml + Init->>State: new StateManager(root, { privateKey }) + Init->>State: load() + Init->>Singleton: reloadPrivateKey() + Init->>FS: append "# noorm" block to .gitignore +``` + +Identity is optional input, not a side effect. `performProjectInit` only creates or updates identity when the caller passes `identityInfo`; when it is `null`, the function assumes a global identity already exists in `~/.noorm/` and proceeds straight to `SettingsManager.init` and `StateManager.load`. On a new project, `StateManager.load()` does not write `state.enc`: with no file on disk it builds `createEmptyState` in memory and returns (`src/core/state/manager.ts:151-166`), so `reloadPrivateKey()` on the process-wide singleton is what makes the just-created key available for later commands. + +### Loading state runs two migration systems in a fixed order, then repairs access unconditionally + +`StateManager.load()` ([`src/core/state/manager.ts`](../../src/core/state/manager.ts)) decrypts the file, then must reconcile two independent version numbers before the state is usable: the numeric `schemaVersion` field ([`src/core/version/state/index.ts`](../../src/core/version/state/index.ts)) and the semver-keyed `version` string ([`src/core/state/migrations.ts`](../../src/core/state/migrations.ts)). Schema-version migrations run first, on the raw record. The semver path (`migrateState`, [`src/core/state/migrations.ts`](../../src/core/state/migrations.ts)) then spreads every unknown top-level field through unchanged (`...carried`), so a downgrade does not destroy a field only a newer build knows about, and it drops exactly one field on purpose: `identity`, because that key moved to `~/.noorm/` and an old state file's copy of it holds private-key material that must never be re-persisted. + +```mermaid +flowchart TD + A["readFileSync(
state.enc)"] --> B["decrypt(payload,
privateKey)"] + B --> C["JSON.parse ->
stateRecord"] + C --> D["migrateSchemaVersion(
stateRecord)"] + D --> E["migrateState(
schemaMigratedState,
currentVersion)"] + E --> F["repairConfigAccess
per config"] + F --> G{"needsVersionMigration
OR backfilledAccess?"} + G -->|yes| H["#persist()"] + G -->|no| I["state:loaded"] + H --> I +``` + +`repairConfigAccess` ([`src/core/state/access.ts`](../../src/core/state/access.ts)) runs on every config regardless of whether either migration fired, because a hand-edited or corrupted state file can reach this point with a malformed `access` even at the current schema version. It is fail-closed: an unrecognized `access.user` or `access.agent` value falls back to `viewer` (`MOST_RESTRICTIVE_ROLE`), and `access.agent: false` (invisible to the agent channel) survives untouched since it is already the strictest possible value. + +### The runner's `file:before`/`file:after` bracket the watcher's repeating `file:progress` + +[`src/core/observer.ts`](../../src/core/observer.ts) exports a single `ObserverEngine` instance. `NoormEvents` aggregates event-payload types from every domain that emits (`settings`, `update`, `vault`, `transfer`, `dt`), plus inline families such as `file:*`, `state:*`, `config:*`, `app:*`, `version:*` declared in this file, so the type surface of this one file is a dependency for anything in the repo that emits or subscribes. `file:before` and `file:after` are the runner's own events ([`src/core/runner/runner.ts`](../../src/core/runner/runner.ts), `executeSingleFileWithUpdate` and `executeSingleFile`); `StatementWatcher` ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) emits only `file:progress`, from inside its own `#report` loop. + +The watcher waits out the watch delay before its first emission, then repeats on the watch interval until `file:after` closes the bracket. + +```mermaid +sequenceDiagram + participant Runner as runner + participant Watcher as StatementWatcher + participant Obs as observer + participant CLI as CLI/TUI subscriber + + Runner->>Obs: emit("file:before", {...}) + loop past the watch delay, every watch interval + Watcher->>Obs: emit("file:progress", { filepath, elapsedMs, sessionId, status }) + Obs->>CLI: on("file:progress", ...) + end + Runner->>Obs: emit("file:after", { status, durationMs }) +``` + +`sessionId` is the server-side session or process id the watcher reads up front for the pinned connection; it is `null` on dialects with no session-id query (sqlite), when the id read itself fails (`statement-watcher.ts`'s `#readSessionId`: `if (err) return null`), and shows as `pid` in the TUI (`src/tui/components/status/StatementProgress.tsx:78`). Its payload's `status: StatementStatus | null` is `null` when the dialect has no probe or the server could not be asked, so a subscriber cannot treat a `null` status as "still fine": it means "no signal available," and `sessionId` lets a caller correlate repeats of the same in-flight statement. + +## Where it lives + +| Path | What | +|---|---| +| [`src/core/state/manager.ts`](../../src/core/state/manager.ts) | `StateManager` — load/persist, three-way merge reconciliation, config/secret/known-user CRUD | +| [`src/core/state/persistence.ts`](../../src/core/state/persistence.ts) | Atomic writes (`writeFileAtomicSync`), advisory lock (`acquireWriteLock`), backup (`backupExisting`), content fingerprinting | +| [`src/core/state/merge.ts`](../../src/core/state/merge.ts) | `mergeState` — per-field reconciliation rules for concurrent writers | +| [`src/core/state/access.ts`](../../src/core/state/access.ts) | `repairConfigAccess` — fail-closed backfill of a config's `access` field | +| [`src/core/state/migrations.ts`](../../src/core/state/migrations.ts) | Semver-keyed `migrateState`/`needsMigration`, keyed on `State.version` | +| [`src/core/state/encryption/crypto.ts`](../../src/core/state/encryption/crypto.ts), `.../index.ts` | AES-256-GCM encrypt/decrypt over `deriveStateKey` (derivation in [`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts)) | +| [`src/core/state/types.ts`](../../src/core/state/types.ts) | `State`, `EncryptedPayload`, `createEmptyState` | +| [`src/core/state/version.ts`](../../src/core/state/version.ts) | `getPackageVersion` — build-time-injected CLI version used to stamp `state.enc` | +| [`src/core/settings/manager.ts`](../../src/core/settings/manager.ts) | `SettingsManager` — `settings.yml` load/save, `#document` vs. env-overlaid `#settings` split | +| [`src/core/settings/rules.ts`](../../src/core/settings/rules.ts) | `evaluateRules`, `getEffectiveBuildPaths`, `isConfigGuarded` | +| [`src/core/settings/defaults.ts`](../../src/core/settings/defaults.ts) | `DEFAULT_SETTINGS`, `SETTINGS_DIR_PATH`, `createDefaultSettings` | +| [`src/core/settings/events.ts`](../../src/core/settings/events.ts), `.../schema.ts`, `.../types.ts` | `SettingsEvents`, zod schema, `Stage`/`Rule`/`BuildConfig` types | +| [`src/core/config/resolver.ts`](../../src/core/config/resolver.ts) | `resolveConfig` — merges defaults, stage, stored, env, and flags; `applyStageCeiling` | +| [`src/core/config/schema.ts`](../../src/core/config/schema.ts) | `ConfigSchema` (`.transform(withResolvedAccess)`), `DANGEROUS_DB_NAME_CHARS` | +| [`src/core/config/validate.ts`](../../src/core/config/validate.ts) | `validateConfigChecks` — the shared connection/name/database/host-presence check sequence used by [`src/cli/config/validate.ts`](../../src/cli/config/validate.ts) and `ConfigValidateScreen.tsx` | +| [`src/core/config/index.ts`](../../src/core/config/index.ts) | `getEnvConfig` — env-var config accessor | +| [`src/core/config/types.ts`](../../src/core/config/types.ts) | `Config`/`ConfigInput` types | +| [`src/core/lifecycle/manager.ts`](../../src/core/lifecycle/manager.ts) | `LifecycleManager` — phased shutdown (`stopping` → `completing` → `releasing` → `flushing` → `exiting`) | +| [`src/core/lifecycle/handlers.ts`](../../src/core/lifecycle/handlers.ts) | `registerSignalHandlers`, `registerExceptionHandlers` | +| [`src/core/lifecycle/types.ts`](../../src/core/lifecycle/types.ts) | `LifecycleConfig`, `ShutdownPhase`, `AppMode` | +| [`src/core/version/types.ts`](../../src/core/version/types.ts) | `CURRENT_VERSIONS` (`schema: 2`, `state: 3`, `settings: 1`), migration interfaces | +| [`src/core/version/schema/`](../../src/core/version/schema), `.../state/`, `.../settings/` | Numeric migration engines, each a `MIGRATIONS` array plus `up`/`down` steps; `schema/` tracks the DB-side tracking tables | +| [`src/core/project.ts`](../../src/core/project.ts) | `findProjectRoot`, `getOriginalCwd`/`setOriginalCwd` | +| [`src/core/project-init.ts`](../../src/core/project-init.ts) | `performProjectInit` — directory/settings/state/identity bootstrap | +| [`src/core/environment.ts`](../../src/core/environment.ts) | `isCi`, `isDev`, `isDebug`, `isEnvTruthy`, `shouldSkipConfirmations`, `getEnvConfigName` | +| [`src/core/observer.ts`](../../src/core/observer.ts) | `observer` singleton, `NoormEvents` (includes `file:progress`) | +| `.noorm/state/state.enc` | AES-256-GCM encrypted JSON, mode `0o600`; holds `configs`, `secrets`, `globalSecrets`, `knownUsers`, `activeConfig`, `version`, `schemaVersion` | +| `.noorm/state/state.enc.bak` | Previous-generation backup written before every overwrite | +| `.noorm/state/state.enc.lock` | Advisory `O_EXCL` lock file, 5s acquire timeout, 30s staleness threshold | +| `.noorm/settings.yml` | YAML, version controlled, parsed/written via the `yaml` package | +| `tests/core/{state,settings,config,lifecycle,version}/` | Unit coverage for state migration/access/persistence, settings load/save, config resolution, lifecycle phases, and version migration | + +## Constraints + +- `StateManager.load()` never writes on init: with no `state.enc` on disk, `load()` builds `createEmptyState` in memory and returns without calling `#persist()`. A caller that needs the file to exist on disk after init must trigger a later save. +- Migration order is fixed: schema-version migration (`migrateSchemaVersion`, numeric `schemaVersion`) must run before the semver `migrateState` (string `version`), on the raw record. `migrateState` ([`src/core/state/migrations.ts`](../../src/core/state/migrations.ts)) is the semver engine; [`src/core/version/`](../../src/core/version) holds the numeric engines (`schema/`, `state/`, `settings/`). +- Access repair is fail-closed on record-shaped `access`: `repairConfigAccess` can only make a record-shaped config more restrictive on an unrecognized channel value; it never loosens. `access.agent: false` (invisible) is distinct from any role and preserved as-is. A non-object `access` with no `protected` instead resolves to `DEFAULT_ACCESS` (`{ user: 'admin', agent: 'viewer' }`) via `resolveLegacyAccess`; a truthy `protected` maps to `GUARDED_ACCESS`. +- `DEFAULTS` omits `access` on purpose: `resolveConfig`'s merge base leaves `access` out so `parseConfig`'s `resolveLegacyAccess` fallback sees what the merged stored/env/flag input supplied instead of a pre-filled default. Pre-filling `access` in `DEFAULTS` would short-circuit `withResolvedAccess`, so a config that only sets legacy `protected: true` would silently resolve to `DEFAULT_ACCESS` (admin) instead of `GUARDED_ACCESS` (`src/core/config/resolver.ts:67-74`). +- Stage ceilings only clamp down: `applyStageCeiling`/`clampToCeiling` cap a `protected: true` stage's resolved access at `{ user: 'operator', agent: 'viewer' }`; a config already stricter than the ceiling is left alone. +- `#document` and `#settings` never merge back: `SettingsManager` mutators (`setStage`, `addRule`, etc.) read and write `#document` only; the env-overlaid `#settings` view is recomputed via `#refreshResolved` after every load and mutation, so ambient `NOORM_*` shell values never land in the version-controlled `settings.yml`. +- Env overlay reads live, not cached: `allSettingsEnv()` calls `makeNestedConfig` fresh per call with `memoizeOpts: false`, so `process.env` mutations after module load are still visible. +- Lock uses `O_EXCL`, not `flock`: `acquireWriteLock` opens with `wx` because `flock` silently no-ops on some network filesystems; a lock older than 30s is treated as abandoned and removed. +- `IV_LENGTH` is 16 bytes, a deliberate deviation from the NIST-recommended 12, noted in [`src/core/state/encryption/crypto.ts`](../../src/core/state/encryption/crypto.ts). `EncryptedPayload.kdf` is optional; absent means `hkdf-sha256`, so old payloads keep decrypting after a future derivation change. +- Secret keys follow one regex declared twice: `isValidSecretKey` (`/^[A-Za-z][A-Za-z0-9_]*$/`) in [`src/core/state/manager.ts`](../../src/core/state/manager.ts), and the same pattern as `SECRET_KEY_PATTERN` in `src/core/vault/storage.ts:35`. `setSecret` and TUI live-typing validators call one or the other rather than deriving a shared constant. +- `file:progress.status` can be `null`: it means no probe exists for the dialect (sqlite) or the server could not be asked. A subscriber must not read it as "healthy." +- [`.gitignore`](../../.gitignore) append is keyed on the entry string: `performProjectInit` checks for the literal `.noorm/state/` entry, not the `# noorm` header comment, because a [`.gitignore`](../../.gitignore) can carry a `# noorm` header with no `.noorm/state/` entry under it. Existing projects may carry a header-only `# noorm` block, so the append still fires for them. ## Coupling -- Imports `resolveLegacyAccess`, `ConfigAccess`, `Role` from **core-policy** ([`src/core/state/access.ts`](../../src/core/state/access.ts), [`src/core/config/schema.ts`](../../src/core/config/schema.ts), [`src/core/config/resolver.ts`](../../src/core/config/resolver.ts)) to resolve and clamp per-config access roles — data resolution, not enforcement. -- Imports `guarded` from **core-policy** (`src/core/settings/rules.ts:isConfigGuarded`) so rule matching on `match.protected` reflects actual access state rather than a stored flag. -- [`src/core/state/manager.ts`](../../src/core/state/manager.ts) imports `KnownUser` ([`src/core/identity/types.ts`](../../src/core/identity/types.ts)) and `loadPrivateKey` ([`src/core/identity/storage.ts`](../../src/core/identity/storage.ts)) from **core-identity**; [`src/core/state/encryption/crypto.ts`](../../src/core/state/encryption/crypto.ts) imports `deriveStateKey` ([`src/core/identity/crypto.ts`](../../src/core/identity/crypto.ts)) and `isValidKeyHex` ([`src/core/identity/storage.ts`](../../src/core/identity/storage.ts)) — state encryption is keyed off the user's identity private key. -- [`src/core/config/types.ts`](../../src/core/config/types.ts) imports `ConnectionConfig`/`Dialect` from [`src/core/connection/types.ts`](../../src/core/connection/types.ts); [`src/core/config/validate.ts`](../../src/core/config/validate.ts) imports `testConnection` from [`src/core/connection/factory.ts`](../../src/core/connection/factory.ts); [`src/core/config/schema.ts`](../../src/core/config/schema.ts) and [`src/core/settings/schema.ts`](../../src/core/settings/schema.ts) both import `PortSchema` from [`src/core/connection/defaults.ts`](../../src/core/connection/defaults.ts) — all **core-db**. +- [`src/core/state/access.ts`](../../src/core/state/access.ts) and [`src/core/config/schema.ts`](../../src/core/config/schema.ts)/`resolver.ts` import `resolveLegacyAccess`, `ConfigAccess`, `Role` from **core-policy** to resolve and clamp per-config access roles: data resolution, not enforcement. +- `src/core/settings/rules.ts:isConfigGuarded` calls `guarded` from **core-policy** so rule matching on `match.protected` reflects actual access state rather than a stored flag. +- [`src/core/state/manager.ts`](../../src/core/state/manager.ts) imports `KnownUser` ([`src/core/identity/types.ts`](../../src/core/identity/types.ts)) and `loadPrivateKey` ([`src/core/identity/storage.ts`](../../src/core/identity/storage.ts)); [`src/core/state/encryption/crypto.ts`](../../src/core/state/encryption/crypto.ts) imports `deriveStateKey` and `isValidKeyHex`: state encryption is keyed off the user's identity private key (**core-identity**). +- [`src/core/config/types.ts`](../../src/core/config/types.ts) imports `ConnectionConfig`/`Dialect` from [`src/core/connection/types.ts`](../../src/core/connection/types.ts); [`src/core/config/validate.ts`](../../src/core/config/validate.ts) imports `testConnection` from [`src/core/connection/factory.ts`](../../src/core/connection/factory.ts); [`src/core/config/schema.ts`](../../src/core/config/schema.ts) and [`src/core/settings/schema.ts`](../../src/core/settings/schema.ts) both import `PortSchema` from [`src/core/connection/defaults.ts`](../../src/core/connection/defaults.ts), all **core-db**. - [`src/core/config/types.ts`](../../src/core/config/types.ts) imports `LogLevel` from [`src/core/logger/types.ts`](../../src/core/logger/types.ts) (**core-identity**). -- [`src/core/lifecycle/manager.ts`](../../src/core/lifecycle/manager.ts) registers a default shutdown resource that calls `getConnectionManager().closeAll()` from [`src/core/connection/manager.ts`](../../src/core/connection/manager.ts) (**core-db**). +- [`src/core/lifecycle/manager.ts`](../../src/core/lifecycle/manager.ts) registers a default shutdown resource that calls `getConnectionManager().closeAll()` (**core-db**). - [`src/core/version/schema/index.ts`](../../src/core/version/schema/index.ts) calls `waitForIdentityToLoad` from [`src/core/identity/index.ts`](../../src/core/identity/index.ts) (**core-identity**) after bootstrapping or migrating tracking tables. -- [`src/core/observer.ts`](../../src/core/observer.ts)'s `NoormEvents` interface extends `SettingsEvents` (this domain) and imports event-payload types from [`src/core/update/`](../../src/core/update), [`src/core/vault/events.ts`](../../src/core/vault/events.ts), [`src/core/transfer/events.ts`](../../src/core/transfer/events.ts), [`src/core/dt/events.ts`](../../src/core/dt/events.ts), [`src/core/logger/types.ts`](../../src/core/logger/types.ts), [`src/core/teardown/types.ts`](../../src/core/teardown/types.ts) — every domain that emits events depends on this file, and this file's type surface depends on those domains' event shapes. - -## Conventions worth knowing - -- `StateManager` ([`src/core/state/manager.ts`](../../src/core/state/manager.ts)) keeps three snapshots: `#state` (working copy), `#baseline` (state as loaded, cloned), and `#diskFingerprint` (SHA-256 of the last-read/written raw file contents). Every `#persist()` re-reads the file, compares its fingerprint to `#diskFingerprint`, and if changed, three-way merges via `mergeState` ([`src/core/state/merge.ts`](../../src/core/state/merge.ts)) using `#baseline` to distinguish "we never touched this key" from "we deleted this key". -- `mergeState`'s per-field rules: `schemaVersion` takes `Math.max(ours, theirs)` (never steps backward), `activeConfig` keeps `ours` only if it changed from baseline, `secrets` is merged two levels deep (`mergeSecrets`) so a concurrent `secret set` on a sibling key isn't dropped, and any top-level field not in `KNOWN_FIELDS` (written by a newer build this one doesn't model) is carried through under the same equals-baseline rule. -- `StateManager.load()` runs two independent migration systems in a fixed order: `migrateSchemaVersion` ([`src/core/version/state/index.ts`](../../src/core/version/state/index.ts), keyed on the `schemaVersion` int field) runs first on the raw untyped record, then the package-semver `migrateState` ([`src/core/state/migrations.ts`](../../src/core/state/migrations.ts), keyed on the `version` string field) runs second. Reversing the order would drop `schemaVersion` and anything else the semver migration doesn't know about, since it only knows `State`'s seven top-level fields. -- After both migrations, `StateManager.load()` calls `repairConfigAccess` ([`src/core/state/access.ts`](../../src/core/state/access.ts)) on every config's `access` field unconditionally — the single point that backfills a missing/malformed `access`, so no downstream consumer (`setConfig`, `listConfigs`, `guarded`) needs its own fallback. It is fail-closed: an unrecognized shape can only make a config *more* restrictive (falls back to `MOST_RESTRICTIVE_ROLE = 'viewer'`), never less. -- `src/core/state/migrations.ts:migrateState` deliberately drops the legacy `identity` field (moved to `~/.noorm/`) rather than carrying it through, so a private key once stored in `state.enc` doesn't get re-persisted forever. Everything else unknown is spread through rather than rebuilt from an allowlist. -- `isValidSecretKey`/`InvalidSecretKeyError` ([`src/core/state/manager.ts`](../../src/core/state/manager.ts)) are the single declaration of the secret-key identifier regex (`/^[A-Za-z][A-Za-z0-9_]*$/`); `setSecret` enforces it, and TUI live-typing validators call the same predicate instead of hand-copying the pattern. -- `writeFileAtomicSync` ([`src/core/state/persistence.ts`](../../src/core/state/persistence.ts)) stages to a sibling `...tmp` file, `fsyncSync`s it, then `renameSync`s over the target, and fsyncs the containing directory afterward (best-effort). `acquireWriteLock` uses `open(..., 'wx')` (`O_EXCL`) rather than `flock` because `flock` silently no-ops on some network filesystems. -- `EncryptedPayload.kdf` is optional; absent means `hkdf-sha256` (the only derivation ever shipped) so old payloads keep decrypting if a future build changes the derivation. `IV_LENGTH` is 16 bytes, not the NIST-recommended 12 — a deliberate deviation noted in [`src/core/state/encryption/crypto.ts`](../../src/core/state/encryption/crypto.ts). -- `SettingsManager` ([`src/core/settings/manager.ts`](../../src/core/settings/manager.ts)) keeps `#document` (exactly what's on/will be written to disk) separate from `#settings` (`#document` merged with a `NOORM_*` env-var overlay via `allSettingsEnv`/`makeNestedConfig`). Stage/rule mutators (`setStage`, `addRule`, etc.) read and write `#document`, never `#settings` — merging the env overlay into the persisted document would commit ambient shell values (vault tokens, DB passwords) into version-controlled `settings.yml`. -- `allSettingsEnv()` ([`src/core/settings/manager.ts`](../../src/core/settings/manager.ts)) calls `makeNestedConfig` fresh per-call; `getEnvConfig()`'s `makeNestedConfig` call is at module scope, but `memoizeOpts: false` keeps its returned `allConfigs()` accessor reading `process.env` live on every invocation — both avoid the same env-snapshot staleness by different means, sidestepping a Bun test-runner edge case where a module-scope closure over `process.env` misses later mutations. -- `resolveConfig` ([`src/core/config/resolver.ts`](../../src/core/config/resolver.ts)) merges five layers in ascending priority — `DEFAULTS` → stage defaults → stored config → env vars → CLI flags — via `merge`/`clone` from `@logosdx/utils`, then runs `parseConfig` and `applyStageCeiling`. `DEFAULTS` deliberately omits `access` so a merged-in stored/env/flag `access` (or legacy `protected`) is what `parseConfig`'s `resolveLegacyAccess` fallback actually sees. -- `applyStageCeiling` ([`src/core/config/resolver.ts`](../../src/core/config/resolver.ts)) only clamps access *down*: a stage with `defaults.protected: true` caps resolved access at `{ user: 'operator', agent: 'viewer' }` (`PROTECTED_STAGE_CEILING`) via `roleRank`/`clampToCeiling`, but never loosens a config that was already stricter than the ceiling. -- `ConfigSchema` ([`src/core/config/schema.ts`](../../src/core/config/schema.ts)) is a `.transform(withResolvedAccess)` — parsing a config always resolves `access` from either the new `access` field or the legacy `protected` boolean, and never echoes `protected` back into the output; `access` is the only stored source of truth after parse. -- Database names are validated against `DANGEROUS_DB_NAME_CHARS` ([`src/core/config/schema.ts`](../../src/core/config/schema.ts)) — quotes, backticks, brackets, semicolons, and control characters — because `database` is interpolated into raw DDL (`CREATE`/`DROP DATABASE`) as a quoted dialect-specific identifier. SQLite is exempt since its `database` is a file path. -- `LifecycleManager` ([`src/core/lifecycle/manager.ts`](../../src/core/lifecycle/manager.ts)) runs shutdown in a fixed phase order (`stopping` → `completing` → `releasing` → `flushing` → `exiting`), each phase running its registered `LifecycleResource`s sorted by `priority` under a per-phase timeout via `runWithTimeout`; a second `SIGINT`/`SIGTERM`/`SIGHUP` during `shutting_down` forces `process.exit(128 + signalCode)` rather than re-entering shutdown. -- [`src/core/version/`](../../src/core/version) has three parallel, near-identical migration engines (schema in `version/schema/`, state in `version/state/`, settings in `version/settings/`), each with its own `MIGRATIONS` array, `getXVersion`, `checkXVersion`, and `migrateX` following the same `up(data)`/`down(data)` migration-interface shape from [`src/core/version/types.ts`](../../src/core/version/types.ts). -- Schema migration v2 ([`src/core/version/schema/migrations/v2.ts`](../../src/core/version/schema/migrations/v2.ts)) moves the six tracking tables (`__noorm_version__`, `__noorm_change__`, `__noorm_executions__`, `__noorm_lock__`, `__noorm_identities__`, `__noorm_vault__`) into a dedicated `noorm` schema with the prefix stripped, for postgres and mssql only — it's a no-op on mysql/sqlite (no schema support, prefixed names stay). It's idempotent: each step (create schema → drop FK → drop indexes → move tables → rename → recreate FK → recreate indexes) checks each table's current location first, so a previously-interrupted run resumes correctly. [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts)'s `getNoormTables(dialect)` (returns clean names for pg/mssql, prefixed for mysql/sqlite) and `noormDb(db, dialect)` (wraps with `.withSchema('noorm')` for pg/mssql, passes through otherwise) are the dialect-aware accessors every caller should use now; the un-dialected `NOORM_TABLES` constant is `@deprecated` — it only maps to prefixed names and produces wrong SQL against `noormDb()` on pg/mssql. [`src/core/version/schema/index.ts`](../../src/core/version/schema/index.ts)'s `tablesExist`/`getSchemaVersion` both check the legacy prefixed location first, then fall back to the schema-qualified location via `noormDb`/`getNoormTables` for pg/mssql. -- [`src/core/version/state/migrations/v3.ts`](../../src/core/version/state/migrations/v3.ts) renames `access.mcp` → `access.agent` (the channel key), reusing `repairConfigAccess` so an unrecognized value downgrades to `viewer` rather than being dropped — the same fail-closed rule v2 established. -- `findProjectRoot` ([`src/core/project.ts`](../../src/core/project.ts)) walks up from cwd looking for a [`.noorm`](../../.noorm) directory, stopping at (and not treating as a project) the user's home directory — `~/.noorm/` is global identity storage, not a project. `getOriginalCwd`/`setOriginalCwd` capture the pre-chdir cwd exactly once (first call wins) so commands like `init --here` can still reach it after `initProjectContext` has already `chdir`'d. -- `performProjectInit` ([`src/core/project-init.ts`](../../src/core/project-init.ts)) only creates/updates identity when `identityInfo` is non-null; when null, it assumes a global identity already exists in `~/.noorm/`. It keys the [`.gitignore`](../../.gitignore) append-check on the literal entry string [`.noorm/state/`](../../.noorm/state) rather than the `# noorm` header comment, because earlier versions wrote the header with nothing under it. +- [`src/core/observer.ts`](../../src/core/observer.ts)'s `NoormEvents` imports event-payload types from [`src/core/vault/events.ts`](../../src/core/vault/events.ts) and [`src/core/logger/types.ts`](../../src/core/logger/types.ts) (**core-identity**), [`src/core/transfer/events.ts`](../../src/core/transfer/events.ts) and [`src/core/teardown/types.ts`](../../src/core/teardown/types.ts) (**core-db**), [`src/core/dt/events.ts`](../../src/core/dt/events.ts) (**sdk**), and `StatementStatus` from [`src/core/runner/statement-probes.ts`](../../src/core/runner/statement-probes.ts) (**core-runner**, which also emits `file:before`/`file:after` through this observer). [`src/core/update/`](../../src/core/update) has no owning domain, and its event-payload types are imported here too. Every domain that emits events depends on this file, and this file's type surface depends on those domains' event shapes. diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 599aa3a8..75401db0 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -5,7 +5,7 @@ description: Bun workspace monorepo — noorm, a database schema/change manager --- repo -816ddb2050f6e33ddaad8a73c64bf08c228266b1 +c980267dfaef74c705bfdf8339f8ac0bb3ec89eb 1 # Project signals @@ -70,14 +70,14 @@ CI gate: lint → typecheck → build → 5 test groups → 3 example jobs. Inte | Domain | Repo paths | One-liner | Detail | |--------|------------|-----------|--------| | core-change | [`src/core/change/`](../../src/core/change), [`src/cli/change/`](../../src/cli/change), [`tests/core/change/`](../../tests/core/change) | Versioned DB changes: scaffold, parse, execute, revert, history (timestamps hydrated as UTC on pg/mysql) | [`docs/wiki/core-change.md`](core-change.md) | -| core-runner | [`src/core/runner/`](../../src/core/runner), [`src/core/template/`](../../src/core/template), [`src/cli/run/`](../../src/cli/run), [`tests/core/runner/`](../../tests/core/runner), [`tests/core/template/`](../../tests/core/template) | SQL file execution with checksum dedup and Eta templating | [`docs/wiki/core-runner.md`](core-runner.md) | -| core-db | [`src/core/db/`](../../src/core/db), [`src/core/connection/`](../../src/core/connection), [`src/core/explore/`](../../src/core/explore), [`src/core/teardown/`](../../src/core/teardown), [`src/core/transfer/`](../../src/core/transfer), [`src/cli/db/`](../../src/cli/db), [`tests/core/connection/`](../../tests/core/connection), [`tests/core/explore/`](../../tests/core/explore), [`tests/core/teardown/`](../../tests/core/teardown), [`tests/core/transfer/`](../../tests/core/transfer), [`tests/integration/`](../../tests/integration) | DB lifecycle: create/drop, explore schema, teardown, cross-DB transfer | [`docs/wiki/core-db.md`](core-db.md) | -| core-state | [`src/core/state/`](../../src/core/state), [`src/core/settings/`](../../src/core/settings), [`src/core/config/`](../../src/core/config), [`src/core/lifecycle/`](../../src/core/lifecycle), [`src/core/version/`](../../src/core/version), [`src/core/project.ts`](../../src/core/project.ts), [`src/core/project-init.ts`](../../src/core/project-init.ts), [`src/core/environment.ts`](../../src/core/environment.ts), [`src/core/observer.ts`](../../src/core/observer.ts), [`tests/core/state/`](../../tests/core/state), [`tests/core/settings/`](../../tests/core/settings), [`tests/core/config/`](../../tests/core/config), [`tests/core/lifecycle/`](../../tests/core/lifecycle), [`tests/core/version/`](../../tests/core/version) | Encrypted state, settings.yml, config resolution, lifecycle, version migration | [`docs/wiki/core-state.md`](core-state.md) | +| core-runner | [`src/core/runner/`](../../src/core/runner), [`src/core/template/`](../../src/core/template), [`src/cli/run/`](../../src/cli/run), [`tests/core/runner/`](../../tests/core/runner), [`tests/core/template/`](../../tests/core/template) | SQL file execution with checksum dedup, Eta templating, and long-running statement progress and cancel | [`docs/wiki/core-runner.md`](core-runner.md) | +| core-db | [`src/core/db/`](../../src/core/db), [`src/core/connection/`](../../src/core/connection), [`src/core/explore/`](../../src/core/explore), [`src/core/teardown/`](../../src/core/teardown), [`src/core/transfer/`](../../src/core/transfer), [`src/cli/db/`](../../src/cli/db), [`tests/core/connection/`](../../tests/core/connection), [`tests/core/db/`](../../tests/core/db), [`tests/core/explore/`](../../tests/core/explore), [`tests/core/teardown/`](../../tests/core/teardown), [`tests/core/transfer/`](../../tests/core/transfer), [`tests/integration/`](../../tests/integration) | DB lifecycle: create/drop, connections and session helpers, explore schema, teardown, cross-DB transfer | [`docs/wiki/core-db.md`](core-db.md) | +| core-state | [`src/core/state/`](../../src/core/state), [`src/core/settings/`](../../src/core/settings), [`src/core/config/`](../../src/core/config), [`src/core/lifecycle/`](../../src/core/lifecycle), [`src/core/version/`](../../src/core/version), [`src/core/project.ts`](../../src/core/project.ts), [`src/core/project-init.ts`](../../src/core/project-init.ts), [`src/core/environment.ts`](../../src/core/environment.ts), [`src/core/observer.ts`](../../src/core/observer.ts), [`tests/core/state/`](../../tests/core/state), [`tests/core/settings/`](../../tests/core/settings), [`tests/core/config/`](../../tests/core/config), [`tests/core/lifecycle/`](../../tests/core/lifecycle), [`tests/core/version/`](../../tests/core/version) | Encrypted state, settings.yml, config resolution, lifecycle, version migration, observer events | [`docs/wiki/core-state.md`](core-state.md) | | core-identity | [`src/core/identity/`](../../src/core/identity), [`src/core/vault/`](../../src/core/vault), [`src/core/logger/`](../../src/core/logger), [`src/core/sql-terminal/`](../../src/core/sql-terminal), [`src/cli/identity/`](../../src/cli/identity), [`src/cli/secret/`](../../src/cli/secret), [`src/cli/vault/`](../../src/cli/vault), [`src/cli/sql/`](../../src/cli/sql), [`tests/core/identity/`](../../tests/core/identity), [`tests/core/vault/`](../../tests/core/vault), [`tests/core/logger/`](../../tests/core/logger), [`tests/core/sql-terminal/`](../../tests/core/sql-terminal) | Identity keypairs, vault secrets, structured logger, SQL terminal history | [`docs/wiki/core-identity.md`](core-identity.md) | | core-policy | [`src/core/policy/`](../../src/core/policy), [`tests/core/policy/`](../../tests/core/policy) | Access-control policy: role×permission matrix, SQL statement classifier, legacy `protected`→`access` migration | [`docs/wiki/core-policy.md`](core-policy.md) | | sdk | [`src/sdk/`](../../src/sdk), [`src/core/dt/`](../../src/core/dt), [`packages/sdk/`](../../packages/sdk), [`tests/sdk/`](../../tests/sdk), [`tests/integration/sdk/`](../../tests/integration/sdk) | Programmatic API (`createContext`) + DT binary serialization format | [`docs/wiki/sdk.md`](sdk.md) | -| cli | [`src/cli/`](../../src/cli), [`packages/cli/`](../../packages/cli), [`skills/noorm/`](../../skills/noorm), [`tests/cli/`](../../tests/cli) | Citty CLI with 12 domain-owning command groups + 6 leaf commands, headless mode, binary distribution | [`docs/wiki/cli.md`](cli.md) | -| tui | [`src/tui/`](../../src/tui), [`.claude/rules/tui-development.md`](../../.claude/rules/tui-development.md), [`tests/cli/components/`](../../tests/cli/components), [`tests/cli/hooks/`](../../tests/cli/hooks), [`tests/cli/screens/`](../../tests/cli/screens) | Ink/React TUI with focus manager, keyboard routing, ~94 registered screens | [`docs/wiki/tui.md`](tui.md) | +| cli | [`src/cli/`](../../src/cli), [`packages/cli/`](../../packages/cli), [`skills/noorm/`](../../skills/noorm), [`tests/cli/`](../../tests/cli) | Citty CLI command groups and leaf commands, headless mode, binary distribution | [`docs/wiki/cli.md`](cli.md) | +| tui | [`src/tui/`](../../src/tui), [`.claude/rules/tui-development.md`](../../.claude/rules/tui-development.md), [`tests/cli/components/`](../../tests/cli/components), [`tests/cli/hooks/`](../../tests/cli/hooks), [`tests/cli/screens/`](../../tests/cli/screens) | Ink/React TUI with focus manager, keyboard routing, registered screens | [`docs/wiki/tui.md`](tui.md) | | mcp-rpc | [`src/mcp/`](../../src/mcp), [`src/rpc/`](../../src/rpc), [`src/cli/mcp/`](../../src/cli/mcp), [`tests/core/mcp/`](../../tests/core/mcp), [`tests/core/rpc/`](../../tests/core/rpc) | MCP server over stdio wrapping flat RPC command registry, permission-gated dispatch | [`docs/wiki/mcp-rpc.md`](mcp-rpc.md) | | worker-bridge | [`src/core/worker-bridge/`](../../src/core/worker-bridge), [`src/workers/`](../../src/workers), [`tests/core/worker-bridge/`](../../tests/core/worker-bridge), [`tests/workers/`](../../tests/workers) | Hub-and-spoke worker threads for DT serialization and DB connection worker | [`docs/wiki/worker-bridge.md`](worker-bridge.md) | | infra | [`.github/`](../../.github), [`scripts/`](../../scripts), [`examples/`](../../examples), [`docs/`](..), `tsup.*.config.ts`, [`docker-compose.test.yml`](../../docker-compose.test.yml), [`bunfig.toml`](../../bunfig.toml) | CI, build pipeline, binary release, example projects, VitePress docs (incl. [`docs/tapes/`](../tapes) VHS demo recordings, [`docs/modeling/`](../modeling)) | [`docs/wiki/infra.md`](infra.md) | @@ -86,7 +86,7 @@ CI gate: lint → typecheck → build → 5 test groups → 3 example jobs. Inte **Test layout:** Tests mirror [`src/`](../../src) under [`tests/`](../../tests). [`tests/utils/`](../../tests/utils) holds shared DB helpers. [`tests/fixtures/`](../../tests/fixtures) has SQL fixtures per dialect. [`tests/integration/`](../../tests/integration) requires live databases. [`tests/global-setup.ts`](../../tests/global-setup.ts) / [`tests/global-teardown.ts`](../../tests/global-teardown.ts) coordinate integration DB bootstrap. -**Test isolation (updated 2026-08):** the previously-documented contamination source — `src/core/config/index.ts:34` calling `makeNestedConfig(process.env, …)` at module scope — does not reproduce; the call passes `memoizeOpts: false`, so lookups re-read `process.env` live rather than snapshotting at import (see [`docs/wiki/core-state.md`](core-state.md)'s Conventions section for the mechanism). The real isolation driver, confirmed in root [`CLAUDE.md`](../../CLAUDE.md), is that Bun's `mock.module` registry is process-global and never restores — see the Build/test/lint section above for the CI-group rationale. +**Test isolation:** Bun's `mock.module` registry is process-global and never restores, which is why CI runs the isolated test groups listed under Build/test/lint. [`src/core/config/index.ts`](../../src/core/config/index.ts) calls `makeNestedConfig` with `memoizeOpts: false`, so config lookups re-read `process.env` and are not a contamination source (see [`docs/wiki/core-state.md`](core-state.md)). **Convention pointers:** [`.claude/rules/typescript.md`](../../.claude/rules/typescript.md) (4-block function structure, `attempt` over try-catch), [`.claude/rules/tui-development.md`](../../.claude/rules/tui-development.md) (focus system, Ink layout), [`.claude/rules/testing.md`](../../.claude/rules/testing.md) (test naming, coverage), [`.claude/rules/documentation.md`](../../.claude/rules/documentation.md) (three-pillar structure). @@ -94,10 +94,8 @@ CI gate: lint → typecheck → build → 5 test groups → 3 example jobs. Inte **Access-control policy (2026-07, config-access-roles feature):** `Config.protected: boolean` was replaced by `Config.access: ConfigAccess` (per-channel `user`/`agent` roles), enforced through the `core-policy` domain. `src/core/config/protection.ts` and `src/rpc/protection.ts` were both deleted — their rule-checking is absorbed into `core/policy`. The runner/change/transfer/sql-terminal executors gate at their core seam via `assertPolicy`, so SDK/CLI/TUI/MCP callers all inherit one enforcement path. `StateManager.load()` runs the schemaVersion-keyed migration (`core/version/state/`, v2 maps `protected`→`access`, v3 maps `access.mcp`→`access.agent`) ahead of the pre-existing package-semver migration. -**Schema migration v2 (noorm schema isolation):** [`src/core/version/schema/migrations/v2.ts`](../../src/core/version/schema/migrations/v2.ts) moves the six tracking tables into a dedicated `noorm` schema on postgres/mssql (prefix stripped: `change`, not `__noorm_change__`); a no-op on mysql/sqlite, which keep the `__noorm_*__` prefixed names. [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts)'s `getNoormTables(dialect)`/`noormDb(db, dialect)` are the dialect-aware accessors; the un-dialected `NOORM_TABLES` constant is `@deprecated`. See [`docs/wiki/core-state.md`](core-state.md)'s Conventions section. +**Tracking-table schema:** on postgres and mssql the tracking tables live in a dedicated `noorm` schema without the `__noorm_` prefix ([`src/core/version/schema/migrations/v2.ts`](../../src/core/version/schema/migrations/v2.ts)); mysql and sqlite keep the `__noorm_*__` names. `getNoormTables(dialect)` and `noormDb(db, dialect)` in [`src/core/shared/tables.ts`](../../src/core/shared/tables.ts) are the dialect-aware accessors; `NOORM_TABLES` is `@deprecated`. -**Recent change (2026-08, local):** [`src/core/change/history.ts`](../../src/core/change/history.ts) now hydrates `executed_at` as UTC on pg/mysql — `pg`/`mysql2` were parsing noorm's naive UTC text through the host's local timezone, so a change applied moments ago could render as "in 4 hours" on a UTC-negative host. mssql is deliberately left alone (tedious was not measured). See [`docs/wiki/core-change.md`](core-change.md)'s Conventions section. - -**Unresolved review findings (informational):** `core-change.md` and `core-identity.md` each required a targeted post-review correction beyond the standard reviewer loop (a dead-code/out-of-domain doc claim in core-change's Docs section, and a gzip/permission-model misattribution to vault in core-identity's Conventions section) — both were fixed directly rather than re-dispatching a 4th sub-agent iteration; content is now accurate as of this refresh. +**Change timestamps:** [`src/core/change/history.ts`](../../src/core/change/history.ts) hydrates `executed_at` as UTC on pg and mysql; mssql is not hydrated (see [`docs/wiki/core-change.md`](core-change.md) Constraints). **Deterministic substrate:** [`docs/wiki/scan.md`](scan.md) (regenerated this refresh via `atomic signals scan`) diff --git a/docs/wiki/scan.md b/docs/wiki/scan.md index 45228416..c980267d 100644 --- a/docs/wiki/scan.md +++ b/docs/wiki/scan.md @@ -2,27 +2,76 @@ ## Tree +``` ├── .agents/ (1) │ └── skills/ (1) │ └── opentui/ (2) -│ ├── references/ (0 files, 8 dirs) +│ ├── references/ (8) +│ │ ├── animation/ (1 file, 0 dirs) +│ │ ├── components/ (5 files, 0 dirs) +│ │ ├── core/ (5 files, 0 dirs) +│ │ ├── keyboard/ (1 file, 0 dirs) +│ │ ├── layout/ (2 files, 0 dirs) +│ │ ├── react/ (5 files, 0 dirs) +│ │ ├── solid/ (5 files, 0 dirs) +│ │ └── testing/ (1 file, 0 dirs) │ └── SKILL.md (a62967f, 195L, 7253ch, 7427B) -├── .changeset/ (3) +├── .changeset/ (9) │ ├── README.md (bf33c79, 8L, 510ch, 510B) +│ ├── binary-no-dotenv-autoload.md (c6f0527, 5L, 383ch, 383B) │ ├── config.json (64bb386, 11L, 307ch, 307B) -│ └── sdk-with-schema.md (027fff0, 9L, 468ch, 472B) +│ ├── connection-error-reasons.md (d86f802, 6L, 539ch, 539B) +│ ├── mssql-unprivileged-logins.md (df9693f, 6L, 489ch, 489B) +│ ├── slow-statement-watcher.md (7fa4155, 6L, 639ch, 639B) +│ ├── tui-explore-duplicate-keys.md (80009cf, 6L, 445ch, 445B) +│ ├── tui-progress-bars.md (5dba30d, 5L, 167ch, 167B) +│ └── tui-typing-blocks-shortcuts.md (c3a693c, 5L, 356ch, 356B) ├── .claude/ (3) │ ├── rules/ (4) -│ │ ├── documentation.md (c0abbb0, 45L, 1419ch, 1421B) -│ │ ├── testing.md (3c3b98d, 58L, 1070ch, 1070B) -│ │ ├── tui-development.md (753e068, 208L, 5427ch, 5431B) -│ │ └── typescript.md (1515159, 308L, 8136ch, 8164B) +│ │ ├── documentation.md (72f6102, 33L, 1577ch, 1577B) +│ │ ├── testing.md (87fc604, 95L, 3931ch, 3933B) +│ │ ├── tui-development.md (07906c0, 422L, 18517ch, 18527B) +│ │ └── typescript.md (b106eec, 194L, 6750ch, 6758B) │ ├── skills/ (2) │ │ ├── noorm-design/ (7) -│ │ │ ├── assets/ (5 files, 1 dir) -│ │ │ ├── preview/ (25 files, 0 dirs) -│ │ │ ├── ui_kits/ (0 files, 2 dirs) -│ │ │ ├── uploads/ (1 file, 0 dirs) +│ │ │ ├── assets/ (6) +│ │ │ │ ├── icons/ (23 files, 0 dirs) +│ │ │ │ ├── noorm-logo-dark.svg (263c2a9, 6L, 324ch, 324B) +│ │ │ │ ├── noorm-logo.svg (4410a17, 7L, 327ch, 327B) +│ │ │ │ ├── noorm-mark-light.svg (d1c9ccb, 7L, 426ch, 426B) +│ │ │ │ ├── noorm-mark.svg (e497d1a, 8L, 413ch, 413B) +│ │ │ │ └── source-palette.png (2bcd9d2, 148L, 48403ch, 49839B) +│ │ │ ├── preview/ (25) +│ │ │ │ ├── _card.css (e460eeb, 85L, 1826ch, 1828B) +│ │ │ │ ├── brand-iconography.html (2e2dad4, 38L, 4392ch, 4401B) +│ │ │ │ ├── brand-mark.html (dd38b1b, 39L, 3273ch, 3278B) +│ │ │ │ ├── brand-wordmark-dark.html (103aa0b, 25L, 1348ch, 1352B) +│ │ │ │ ├── brand-wordmark.html (af8f6eb, 24L, 1243ch, 1247B) +│ │ │ │ ├── color-bronze.html (819bc48, 31L, 2035ch, 2042B) +│ │ │ │ ├── color-ember.html (f5282fa, 31L, 2064ch, 2072B) +│ │ │ │ ├── color-neutrals.html (2ac965f, 51L, 3414ch, 3423B) +│ │ │ │ ├── color-seed-palette.html (ea3f1dd, 29L, 1722ch, 1728B) +│ │ │ │ ├── color-semantic-dark.html (3047ebf, 41L, 2926ch, 2928B) +│ │ │ │ ├── color-semantic-light.html (376c47e, 40L, 2913ch, 2915B) +│ │ │ │ ├── color-status.html (c2a38a5, 36L, 2600ch, 2603B) +│ │ │ │ ├── components-badges.html (88d2fc2, 45L, 2746ch, 2751B) +│ │ │ │ ├── components-buttons.html (0b2d44e, 56L, 3041ch, 3050B) +│ │ │ │ ├── components-callouts.html (ea17c2a, 39L, 3210ch, 3216B) +│ │ │ │ ├── components-cards.html (8dbad00, 47L, 2687ch, 2698B) +│ │ │ │ ├── components-code-block.html (2ae6519, 43L, 2582ch, 2591B) +│ │ │ │ ├── components-form.html (9e60df1, 44L, 3442ch, 3452B) +│ │ │ │ ├── spacing-motion.html (36d9a91, 35L, 2884ch, 2900B) +│ │ │ │ ├── spacing-radii.html (ac55eb6, 35L, 2783ch, 2791B) +│ │ │ │ ├── spacing-scale.html (3260f23, 30L, 2302ch, 2305B) +│ │ │ │ ├── spacing-shadows.html (02af78e, 31L, 2118ch, 2125B) +│ │ │ │ ├── type-family.html (5e0085b, 37L, 1990ch, 1996B) +│ │ │ │ ├── type-mono-usage.html (edbae12, 52L, 2829ch, 2838B) +│ │ │ │ └── type-scale.html (3e97fdf, 27L, 2543ch, 2558B) +│ │ │ ├── ui_kits/ (2) +│ │ │ │ ├── cli-tui/ (6 files, 0 dirs) +│ │ │ │ └── docs-site/ (9 files, 0 dirs) +│ │ │ ├── uploads/ (1) +│ │ │ │ └── Screenshot 2026-05-09 at 5.09.53 PM.png (2bcd9d2, 148L, 48403ch, 49839B) │ │ │ ├── README.md (0afc77c, 302L, 13445ch, 13558B) │ │ │ ├── SKILL.md (0b38f2d, 29L, 1885ch, 1893B) │ │ │ └── colors_and_type.css (9e653e9, 403L, 12946ch, 13014B) @@ -55,30 +104,31 @@ │ │ ├── settings-secret.md (99e8f29, 23L, 664ch, 668B) │ │ ├── sql-repl.md (20a6f63, 28L, 799ch, 803B) │ │ └── sql.md (52e5a3c, 77L, 3342ch, 3374B) -│ ├── design/ (4) +│ ├── design/ (5) │ │ ├── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ │ ├── config-access-roles.md (bd73baa, 116L, 6566ch, 6670B) │ │ ├── sdk-with-schema.md (37b324b, 117L, 7194ch, 7234B) +│ │ ├── update-download-progress.md (2106019, 47L, 3002ch, 3002B) │ │ └── v1-49-54-cli-field-defects.md (6f051d8, 242L, 12448ch, 12534B) │ ├── dev/ (25) │ │ ├── change.md (89e1ed4, 556L, 19149ch, 19231B) │ │ ├── ci.md (53c262d, 221L, 8752ch, 8772B) │ │ ├── config-sharing.md (36147c5, 280L, 10834ch, 10884B) -│ │ ├── config.md (adc97e5, 451L, 14243ch, 14293B) +│ │ ├── config.md (d1992d9, 459L, 14716ch, 14768B) │ │ ├── datamodel.md (93e585e, 1105L, 36367ch, 36527B) -│ │ ├── explore.md (84d5cf5, 360L, 10711ch, 10815B) +│ │ ├── explore.md (024a815, 361L, 10798ch, 10902B) │ │ ├── headless.md (8731e15, 839L, 23000ch, 23212B) │ │ ├── identity.md (6124955, 429L, 16703ch, 16747B) │ │ ├── index.md (cf0dae0, 44L, 1558ch, 1558B) -│ │ ├── ink-cheatsheet.md (772d4ba, 1446L, 30362ch, 30390B) -│ │ ├── ink-testing-library-cheatsheet.md (f6bd3cd, 763L, 16841ch, 16849B) +│ │ ├── ink-cheatsheet.md (4a06b26, 1942L, 57168ch, 57240B) +│ │ ├── ink-testing-library-cheatsheet.md (cac36cd, 774L, 18009ch, 18017B) │ │ ├── lock.md (2096e54, 344L, 10829ch, 10875B) │ │ ├── logger.md (e297dba, 599L, 21334ch, 22867B) │ │ ├── project-discovery.md (ac60c94, 150L, 5071ch, 5083B) -│ │ ├── runner.md (aefbef0, 543L, 21385ch, 21423B) +│ │ ├── runner.md (61cd053, 614L, 26307ch, 26345B) │ │ ├── sdk.md (b538f8f, 1159L, 31592ch, 31668B) │ │ ├── secrets.md (47a8ca1, 321L, 11587ch, 11665B) -│ │ ├── settings.md (173bfc7, 800L, 21459ch, 21481B) +│ │ ├── settings.md (eacd272, 854L, 23279ch, 23301B) │ │ ├── sql-terminal.md (1cba9d3, 340L, 10785ch, 12173B) │ │ ├── state.md (93a30bc, 403L, 11675ch, 11719B) │ │ ├── teardown.md (eef86b1, 427L, 16037ch, 16087B) @@ -104,7 +154,7 @@ │ │ │ ├── create.md (061870f, 169L, 6551ch, 6573B) │ │ │ ├── explore.md (db56d57, 521L, 20944ch, 25422B) │ │ │ ├── teardown.md (5919704, 397L, 14078ch, 14084B) -│ │ │ ├── terminal.md (1b4047d, 243L, 10565ch, 12325B) +│ │ │ ├── terminal.md (6883cd8, 269L, 11978ch, 13738B) │ │ │ └── transfer.md (7e28d77, 388L, 12244ch, 12270B) │ │ ├── environments/ (4) │ │ │ ├── configs.md (abc5ba8, 383L, 15598ch, 15638B) @@ -112,33 +162,45 @@ │ │ │ ├── stages.md (14f1041, 252L, 8279ch, 8283B) │ │ │ └── vault.md (b1af624, 258L, 8719ch, 9267B) │ │ ├── sql-files/ (3) -│ │ │ ├── execution.md (068713f, 256L, 10297ch, 10439B) +│ │ │ ├── execution.md (5bc154f, 265L, 11279ch, 11421B) │ │ │ ├── organization.md (5febfd5, 351L, 9226ch, 9692B) │ │ │ └── templates.md (c8f060b, 508L, 16419ch, 16547B) │ │ ├── deployment.md (0e61d4d, 190L, 8503ch, 8530B) │ │ ├── relational-design.md (b606099, 299L, 14581ch, 14597B) -│ │ └── troubleshooting.md (594e50f, 123L, 4356ch, 4370B) +│ │ └── troubleshooting.md (64d8466, 146L, 5771ch, 5785B) │ ├── modeling/ (8) │ │ ├── best-practices.md (88a154a, 149L, 10034ch, 10038B) │ │ ├── branding.md (bf235bc, 110L, 4486ch, 4487B) -│ │ ├── data-flows.md (17d326a, 181L, 10245ch, 10249B) -│ │ ├── entities.md (4344438, 357L, 17707ch, 17721B) -│ │ ├── index.md (a702199, 127L, 9116ch, 9116B) -│ │ ├── installation.md (e7d7912, 163L, 7135ch, 7143B) -│ │ ├── modeling-skill.md (55d7a6d, 86L, 5201ch, 5201B) -│ │ └── reverse-engineering.md (a220459, 189L, 13026ch, 13028B) +│ │ ├── data-flows.md (700e476, 181L, 10254ch, 10258B) +│ │ ├── entities.md (b4e9a1b, 357L, 17710ch, 17724B) +│ │ ├── index.md (0f5a2e8, 127L, 9119ch, 9119B) +│ │ ├── installation.md (845c455, 163L, 7147ch, 7155B) +│ │ ├── modeling-skill.md (c7d7e19, 86L, 5231ch, 5231B) +│ │ └── reverse-engineering.md (069ebf9, 189L, 13038ch, 13040B) │ ├── models/ (3) │ │ ├── polymorphic/ (3) -│ │ │ ├── data/ (0 files, 2 dirs) -│ │ │ ├── groups/ (2 files, 0 dirs) +│ │ │ ├── data/ (2) +│ │ │ │ ├── content/ (4 files, 0 dirs) +│ │ │ │ └── identity/ (3 files, 0 dirs) +│ │ │ ├── groups/ (2) +│ │ │ │ ├── content.md (5a8b1aa, 6L, 104ch, 104B) +│ │ │ │ └── identity.md (b0104ba, 6L, 108ch, 108B) │ │ │ └── ignatius.yml (61d527e, 8L, 250ch, 252B) │ │ ├── social/ (3) -│ │ │ ├── data/ (0 files, 3 dirs) -│ │ │ ├── groups/ (3 files, 0 dirs) +│ │ │ ├── data/ (3) +│ │ │ │ ├── content/ (13 files, 0 dirs) +│ │ │ │ ├── identity/ (3 files, 0 dirs) +│ │ │ │ └── tagging/ (4 files, 0 dirs) +│ │ │ ├── groups/ (3) +│ │ │ │ ├── content.md (9605963, 6L, 115ch, 115B) +│ │ │ │ ├── identity.md (893db84, 6L, 75ch, 75B) +│ │ │ │ └── tagging.md (94a6b12, 6L, 91ch, 91B) │ │ │ └── ignatius.yml (ac15e3e, 12L, 275ch, 277B) │ │ └── todo-list/ (3) -│ │ ├── data/ (0 files, 1 dir) -│ │ ├── groups/ (1 file, 0 dirs) +│ │ ├── data/ (1) +│ │ │ └── core/ (3 files, 0 dirs) +│ │ ├── groups/ (1) +│ │ │ └── core.md (b6f6e38, 6L, 91ch, 91B) │ │ └── ignatius.yml (f8945e4, 8L, 220ch, 222B) │ ├── public/ (5) │ │ ├── icons/ (10) @@ -152,15 +214,36 @@ │ │ │ ├── terminal.svg (5102e98, 1L, 613ch, 613B) │ │ │ ├── toolbox.svg (7cc582d, 1L, 373ch, 373B) │ │ │ └── users.svg (7c6b4e2, 1L, 1099ch, 1099B) -│ │ ├── image/ (8) -│ │ │ ├── tui/ (18 files, 0 dirs) +│ │ ├── image/ (9) +│ │ │ ├── tui/ (20) +│ │ │ │ ├── change-history.png (4e6ddc4, 320L, 99358ch, 103277B) +│ │ │ │ ├── changes-list.png (1a5f3a9, 226L, 81769ch, 85137B) +│ │ │ │ ├── config-add.png (daed998, 348L, 96689ch, 100728B) +│ │ │ │ ├── config-list.png (7adec22, 134L, 43834ch, 45499B) +│ │ │ │ ├── database-menu.png (de846a8, 448L, 128366ch, 133720B) +│ │ │ │ ├── explore-overview.png (bd19ade, 219L, 69498ch, 72473B) +│ │ │ │ ├── explore-row-peek.png (bbd2f99, 494L, 180570ch, 188475B) +│ │ │ │ ├── explore-row-view.png (c5b3aba, 199L, 77146ch, 80268B) +│ │ │ │ ├── explore-table-detail.png (5f8ca08, 430L, 121817ch, 126796B) +│ │ │ │ ├── explore-tables.png (460a007, 177L, 55408ch, 57669B) +│ │ │ │ ├── home.png (f0bf584, 420L, 134065ch, 139457B) +│ │ │ │ ├── identity.png (162254c, 213L, 77614ch, 80640B) +│ │ │ │ ├── lock.png (abab999, 308L, 87494ch, 91300B) +│ │ │ │ ├── log-viewer.png (5066453, 561L, 192639ch, 200763B) +│ │ │ │ ├── more-menu.png (a1059ac, 274L, 67882ch, 70671B) +│ │ │ │ ├── run-menu.png (39e72c1, 249L, 98608ch, 103320B) +│ │ │ │ ├── secrets.png (ea815bd, 107L, 45327ch, 47249B) +│ │ │ │ ├── settings.png (c5ed7b2, 404L, 125506ch, 130509B) +│ │ │ │ ├── sql-terminal.png (a7195f5, 160L, 46568ch, 48460B) +│ │ │ │ └── vault.png (6a5d703, 84L, 39941ch, 41834B) │ │ │ ├── 0105.gif (fa282b2, 31773L, 7759553ch, 8187851B) -│ │ │ ├── build-and-change.gif (5d4590a, 2404L, 360320ch, 369821B) +│ │ │ ├── build-and-change.gif (e90d150, 1334L, 219029ch, 225025B) │ │ │ ├── install.gif (d932cef, 1163L, 187594ch, 193254B) │ │ │ ├── logo.png (f273558, 100L, 44738ch, 46953B) │ │ │ ├── logo.svg (8d46c28, 6L, 2529ch, 2529B) │ │ │ ├── og.png (fa3d4ae, 206L, 57780ch, 60183B) -│ │ │ └── tui.gif (8fd1011, 10802L, 1639046ch, 1686253B) +│ │ │ ├── tui-rows.gif (9389b46, 4231L, 724149ch, 744438B) +│ │ │ └── tui.gif (24fe56d, 11710L, 1679451ch, 1728445B) │ │ ├── models/ (3) │ │ │ ├── polymorphic.html (a2c06a9, 629L, 4272673ch, 4277149B) │ │ │ ├── social.html (02baebf, 629L, 4287168ch, 4291644B) @@ -171,139 +254,546 @@ │ │ └── install.sh (0cc90a2, 116L, 2925ch, 2925B) │ ├── reference/ (1) │ │ └── sdk.md (be03cd8, 1705L, 63186ch, 63417B) -│ ├── spec/ (5) +│ ├── spec/ (6) │ │ ├── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ │ ├── config-access-roles.md (40ef290, 162L, 21805ch, 21929B) │ │ ├── sdk-with-schema.md (593829a, 174L, 13384ch, 13504B) +│ │ ├── update-download-progress.md (85fec84, 152L, 9599ch, 9677B) │ │ ├── v1-45-rewind-tiebreak.md (0e35550, 61L, 5465ch, 5507B) │ │ └── v1-49-54-cli-field-defects.md (438757f, 374L, 23347ch, 23495B) │ ├── superpowers/ (1) │ │ └── specs/ (1) │ │ └── 2026-04-19-cli-ci-identity-design.md (6c9cc80, 938L, 32762ch, 32918B) -│ ├── tapes/ (10) -│ │ ├── demo-project/ (2) -│ │ │ ├── changes/ (0 files, 2 dirs) -│ │ │ └── sql/ (0 files, 2 dirs) +│ ├── tapes/ (11) +│ │ ├── demo-project/ (3) +│ │ │ ├── changes/ (2) +│ │ │ │ ├── 2026-02-01-add-task-priority/ (1 file, 2 dirs) +│ │ │ │ └── 2026-02-08-priority-in-view/ (0 files, 2 dirs) +│ │ │ ├── sql/ (2) +│ │ │ │ ├── 01_tables/ (3 files, 0 dirs) +│ │ │ │ └── 02_views/ (1 file, 0 dirs) +│ │ │ └── seed.sql (3b7f18a, 49L, 3690ch, 3690B) │ │ ├── 01-install.tape (6fdf961, 65L, 2242ch, 2248B) │ │ ├── 02-build-and-change.tape (a8cf0c5, 56L, 1699ch, 1701B) -│ │ ├── 03-tui.tape (72050f6, 193L, 5312ch, 5910B) -│ │ ├── 04-screenshots.tape (b68ad2f, 181L, 3706ch, 4460B) -│ │ ├── README.md (a23b108, 185L, 8560ch, 8618B) +│ │ ├── 03-tui.tape (ba2d886, 231L, 6425ch, 7023B) +│ │ ├── 04-screenshots.tape (c21fa9a, 213L, 4798ch, 5560B) +│ │ ├── 05-rows.tape (f178a55, 108L, 3394ch, 3800B) +│ │ ├── README.md (c6f7ce3, 234L, 11257ch, 11321B) │ │ ├── env-scrub.sh (5a1ec5b, 20L, 930ch, 930B) -│ │ ├── sandbox.sh (d593e66, 153L, 5529ch, 5547B) -│ │ ├── shots.sh (56e7b59, 58L, 1701ch, 1701B) +│ │ ├── sandbox.sh (6679418, 168L, 6349ch, 6367B) +│ │ ├── shots.sh (2e05acc, 63L, 1971ch, 1975B) │ │ └── theme.tape (3ad3d47, 40L, 2063ch, 2071B) │ ├── bun.lockb (34225b2, 190L, 125711ch, 126677B) │ ├── headless.md (a76c0a6, 1870L, 55232ch, 55376B) │ ├── index.md (376b0eb, 166L, 6438ch, 6452B) │ ├── package.json (b4778f0, 24L, 657ch, 657B) -│ ├── tui.md (0952b10, 316L, 9428ch, 9968B) +│ ├── tui.md (da1aad9, 537L, 18451ch, 19040B) │ └── why-noorm.md (dc7e978, 109L, 7078ch, 7078B) ├── examples/ (3) │ ├── llm-memory-db-mssql/ (16) │ │ ├── .cursor/ (1) -│ │ │ └── rules/ (1 file, 0 dirs) +│ │ │ └── rules/ (1) +│ │ │ └── use-bun-instead-of-node-vite-npm-pnpm.mdc (1f39d31, 111L, 2676ch, 2676B) │ │ ├── .noorm/ (2) │ │ │ ├── .gitignore (14188a3, 1L, 7ch, 7B) │ │ │ └── settings.yml (295c27f, 77L, 1800ch, 1800B) │ │ ├── changes/ (1) -│ │ │ └── 2026-05-10-add-memory-tag-color/ (1 file, 2 dirs) +│ │ │ └── 2026-05-10-add-memory-tag-color/ (3) +│ │ │ ├── change/ (2 files, 0 dirs) +│ │ │ ├── revert/ (2 files, 0 dirs) +│ │ │ └── changelog.md (beb0543, 19L, 753ch, 757B) │ │ ├── sql/ (11) -│ │ │ ├── 00_types/ (4 files, 0 dirs) -│ │ │ ├── 01_reference/ (10 files, 0 dirs) -│ │ │ ├── 02_tables/ (9 files, 0 dirs) -│ │ │ ├── 03_validators/ (5 files, 0 dirs) -│ │ │ ├── 04_subtypes/ (8 files, 0 dirs) -│ │ │ ├── 05_binary/ (11 files, 0 dirs) -│ │ │ ├── 06_seeds/ (11 files, 0 dirs) -│ │ │ ├── 07_functions/ (8 files, 0 dirs) -│ │ │ ├── 08_views/ (18 files, 0 dirs) -│ │ │ ├── 09_procedures/ (73 files, 0 dirs) +│ │ │ ├── 00_types/ (4) +│ │ │ │ ├── 01_TagAttachmentInput.sql (10b2e32, 4L, 112ch, 112B) +│ │ │ │ ├── 02_MemoryIdSet.sql (36fc146, 3L, 75ch, 75B) +│ │ │ │ ├── 03_TagIdSet.sql (cf56077, 3L, 69ch, 69B) +│ │ │ │ └── 04_TaskDependencyInput.sql (fa17fa6, 8L, 293ch, 293B) +│ │ │ ├── 01_reference/ (10) +│ │ │ │ ├── 01_RelevanceStatus.sql (2a73c72, 4L, 156ch, 156B) +│ │ │ │ ├── 02_TrackingStatus.sql (1ddd81d, 4L, 152ch, 152B) +│ │ │ │ ├── 03_NoteType.sql (59a6185, 4L, 128ch, 128B) +│ │ │ │ ├── 04_MemoryDomain.sql (9fe44b4, 4L, 130ch, 130B) +│ │ │ │ ├── 05_MemoryCategory.sql (ac18220, 4L, 138ch, 138B) +│ │ │ │ ├── 06_DependencyVerb.sql (741b5cd, 4L, 152ch, 152B) +│ │ │ │ ├── 07_MemoryRelationVerb.sql (2a11a3f, 5L, 196ch, 196B) +│ │ │ │ ├── 08_StateTransitionType.sql (ecc86ac, 4L, 174ch, 174B) +│ │ │ │ ├── 09_TrackingStatus_Allowed.sql (9c2f628, 7L, 487ch, 487B) +│ │ │ │ └── 10_RelevanceStatus_Allowed.sql (9a85e9f, 7L, 495ch, 495B) +│ │ │ ├── 02_tables/ (9) +│ │ │ │ ├── 01_Agent.sql (bced7b0, 8L, 363ch, 363B) +│ │ │ │ ├── 02_Project.sql (3db9972, 16L, 831ch, 831B) +│ │ │ │ ├── 03_Note.sql (3a60c72, 26L, 1396ch, 1396B) +│ │ │ │ ├── 04_Tag.sql (f6a796d, 20L, 1039ch, 1039B) +│ │ │ │ ├── 05_Memory.sql (324256b, 36L, 1930ch, 1930B) +│ │ │ │ ├── 06_Artifact.sql (6ee78e1, 24L, 1317ch, 1317B) +│ │ │ │ ├── 07_Milestone.sql (77217ad, 27L, 1497ch, 1497B) +│ │ │ │ ├── 08_StateTransition.sql (6bbb5f5, 22L, 1326ch, 1328B) +│ │ │ │ └── 09_Task.sql (133b75b, 25L, 1354ch, 1354B) +│ │ │ ├── 03_validators/ (5) +│ │ │ │ ├── 01_fn_NoteIsOfType.sql (aee75b9, 24L, 927ch, 927B) +│ │ │ │ ├── 02_fn_StateTransitionIsOfType.sql (99d4ffb, 26L, 1123ch, 1123B) +│ │ │ │ ├── 02b_fn_StateTransitionIsMilestoneAxis.sql (8847036, 27L, 1062ch, 1062B) +│ │ │ │ ├── 03_fn_IsTrackingTransitionAllowed.sql (8f336ca, 25L, 871ch, 871B) +│ │ │ │ └── 04_fn_IsRelevanceTransitionAllowed.sql (1fa8d32, 25L, 895ch, 895B) +│ │ │ ├── 04_subtypes/ (8) +│ │ │ │ ├── 01_Project_Note.sql (d30c42c, 24L, 1377ch, 1377B) +│ │ │ │ ├── 02_Milestone_Note.sql (babc5d7, 23L, 1304ch, 1304B) +│ │ │ │ ├── 03_Task_Note.sql (41fc735, 24L, 1310ch, 1310B) +│ │ │ │ ├── 04_Milestone_StateTransition.sql (28a115d, 23L, 1483ch, 1483B) +│ │ │ │ ├── 05_Task_StateTransition.sql (155b466, 23L, 1369ch, 1369B) +│ │ │ │ ├── 06_Memory_StateTransition.sql (cddf656, 22L, 1312ch, 1312B) +│ │ │ │ ├── 07_Note_StateTransition.sql (5ba5552, 22L, 1286ch, 1286B) +│ │ │ │ └── 08_Artifact_StateTransition.sql (debab8b, 22L, 1338ch, 1338B) +│ │ │ ├── 05_binary/ (11) +│ │ │ │ ├── 01_Project_Tag.sql (7345ad0, 17L, 917ch, 917B) +│ │ │ │ ├── 02_Memory_Tag.sql (44bb0c1, 17L, 906ch, 906B) +│ │ │ │ ├── 03_Artifact_Tag.sql (063e437, 15L, 786ch, 786B) +│ │ │ │ ├── 04_Milestone_Tag.sql (8e12e49, 16L, 878ch, 878B) +│ │ │ │ ├── 05_Task_Tag.sql (d71a6b7, 18L, 1002ch, 1002B) +│ │ │ │ ├── 06_Project_Memory.sql (1ab895e, 16L, 887ch, 887B) +│ │ │ │ ├── 07_Project_Milestone.sql (b889956, 16L, 924ch, 924B) +│ │ │ │ ├── 08_Related_Memory.sql (d8c3296, 24L, 1437ch, 1437B) +│ │ │ │ ├── 09_Milestone_Artifact.sql (de95f78, 17L, 1000ch, 1000B) +│ │ │ │ ├── 10_Task_Artifact.sql (56a051e, 18L, 1033ch, 1033B) +│ │ │ │ └── 11_Task_Dependency.sql (5153e83, 26L, 1529ch, 1529B) +│ │ │ ├── 06_seeds/ (11) +│ │ │ │ ├── 01_RelevanceStatus.sql.tmpl (432dea2, 7L, 419ch, 419B) +│ │ │ │ ├── 02_TrackingStatus.sql.tmpl (c611a4f, 7L, 454ch, 454B) +│ │ │ │ ├── 03_NoteType.sql.tmpl (cfde88b, 7L, 390ch, 390B) +│ │ │ │ ├── 04_MemoryDomain.sql.tmpl (7e04020, 7L, 409ch, 409B) +│ │ │ │ ├── 05_MemoryCategory.sql.tmpl (060ff5e, 7L, 419ch, 419B) +│ │ │ │ ├── 06_DependencyVerb.sql.tmpl (545133b, 7L, 445ch, 445B) +│ │ │ │ ├── 07_MemoryRelationVerb.sql.tmpl (4f2934c, 9L, 843ch, 843B) +│ │ │ │ ├── 08_StateTransitionType.sql.tmpl (631f7d7, 8L, 561ch, 561B) +│ │ │ │ ├── 09_TrackingStatus_Allowed.sql.tmpl (9e10dea, 12L, 1164ch, 1164B) +│ │ │ │ ├── 10_RelevanceStatus_Allowed.sql.tmpl (b8a72c9, 12L, 991ch, 991B) +│ │ │ │ └── 11_Sentinels.sql.tmpl (e6831d9, 40L, 1617ch, 1619B) +│ │ │ ├── 07_functions/ (8) +│ │ │ │ ├── 01_fn_NextTaskNo.sql (7c622b5, 19L, 693ch, 693B) +│ │ │ │ ├── 02_fn_IsActive.sql (b948251, 12L, 534ch, 534B) +│ │ │ │ ├── 03_fn_IsOpen.sql (b4335a1, 16L, 628ch, 628B) +│ │ │ │ ├── 04_fn_MemoryConfidence.sql (34fa9e2, 23L, 931ch, 931B) +│ │ │ │ ├── 05_fn_TaskDependencyWouldCycle.sql (974307d, 55L, 1939ch, 1939B) +│ │ │ │ ├── 06_fn_MemoryRank.sql (729e2e8, 61L, 2163ch, 2163B) +│ │ │ │ ├── 07_fn_NoteSubtypeCount.sql (9351101, 25L, 985ch, 985B) +│ │ │ │ └── 08_fn_NoteMatchesSubtype.sql (b0a4812, 38L, 1257ch, 1257B) +│ │ │ ├── 08_views/ (18) +│ │ │ │ ├── 01_vw_Tag.sql (70b7c53, 93L, 3480ch, 3480B) +│ │ │ │ ├── 02_vw_Artifact.sql (f3943f9, 45L, 1686ch, 1686B) +│ │ │ │ ├── 03_vw_Note.sql (99c7aea, 30L, 1324ch, 1324B) +│ │ │ │ ├── 04_vw_Related_Memory.sql (9d98250, 34L, 1372ch, 1372B) +│ │ │ │ ├── 05_vw_Memory.sql (c84d01e, 27L, 851ch, 851B) +│ │ │ │ ├── 06_vw_Milestone_Stats.sql (0082e26, 85L, 3590ch, 3590B) +│ │ │ │ ├── 07_vw_Active_Memory.sql (a2fd261, 10L, 436ch, 436B) +│ │ │ │ ├── 08_vw_Active_Note.sql (ff5e0fb, 19L, 598ch, 598B) +│ │ │ │ ├── 09_vw_Active_Artifact.sql (2584e91, 20L, 636ch, 636B) +│ │ │ │ ├── 10_vw_Active_Milestone.sql (fc28ea0, 20L, 644ch, 644B) +│ │ │ │ ├── 11_vw_Deleted_Memory.sql (920ef86, 12L, 563ch, 563B) +│ │ │ │ ├── 12_vw_Deleted_Note.sql (92c5d1b, 19L, 602ch, 602B) +│ │ │ │ ├── 13_vw_Deleted_Artifact.sql (2765162, 20L, 640ch, 640B) +│ │ │ │ ├── 14_vw_Deleted_Milestone.sql (0ffe1f6, 20L, 648ch, 648B) +│ │ │ │ ├── 15_vw_Task_Backlog.sql (792d9e7, 37L, 1462ch, 1462B) +│ │ │ │ ├── 16_vw_StateTransition.sql (46915f7, 39L, 2014ch, 2014B) +│ │ │ │ ├── 17_vw_Recent_Activity.sql (93c2379, 209L, 6653ch, 6653B) +│ │ │ │ └── 18_vw_Agent_Activity.sql (64dca31, 40L, 2502ch, 2502B) +│ │ │ ├── 09_procedures/ (73) +│ │ │ │ ├── 01_agent_01_sp_Agent_Create.sql (16abddc, 23L, 803ch, 809B) +│ │ │ │ ├── 01_agent_02_sp_Agent_Update.sql (4222c7d, 20L, 505ch, 505B) +│ │ │ │ ├── 01_agent_03_sp_Agent_Delete.sql (35e8ae1, 24L, 1046ch, 1046B) +│ │ │ │ ├── 02_project_01_sp_Project_Create.sql (6d52b14, 30L, 1127ch, 1133B) +│ │ │ │ ├── 02_project_02_sp_Project_Update.sql (d7ae1dc, 26L, 747ch, 747B) +│ │ │ │ ├── 02_project_03_sp_Project_Delete.sql (c05c371, 53L, 1850ch, 1850B) +│ │ │ │ ├── 03_memory_01_sp_Memory_Create.sql (2dc3d30, 60L, 2605ch, 2627B) +│ │ │ │ ├── 03_memory_02_sp_Memory_Update.sql (f1ddba1, 38L, 1294ch, 1294B) +│ │ │ │ ├── 03_memory_03_sp_Memory_SetRelevance.sql (611738e, 46L, 1523ch, 1523B) +│ │ │ │ ├── 03_memory_04_sp_Memory_Delete.sql (565f947, 13L, 370ch, 370B) +│ │ │ │ ├── 03_memory_05_sp_Memory_Restore.sql (527c070, 13L, 370ch, 370B) +│ │ │ │ ├── 03_memory_06_sp_Memory_Touch.sql (20b35b3, 14L, 426ch, 426B) +│ │ │ │ ├── 03_memory_07_sp_Memory_Relate.sql (a7a64f5, 35L, 1033ch, 1033B) +│ │ │ │ ├── 03_memory_08_sp_Memory_Unrelate.sql (f9890dc, 12L, 328ch, 328B) +│ │ │ │ ├── 03_memory_09_sp_Memory_Consolidate.sql (06d9b43, 73L, 2865ch, 2865B) +│ │ │ │ ├── 03_memory_10_sp_Memory_Attach_Project.sql (18e949b, 17L, 454ch, 454B) +│ │ │ │ ├── 03_memory_11_sp_Memory_Detach_Project.sql (188564d, 12L, 307ch, 307B) +│ │ │ │ ├── 04_milestone_01_sp_Milestone_Create.sql (9fc5e7e, 36L, 1675ch, 1693B) +│ │ │ │ ├── 04_milestone_02_sp_Milestone_Update.sql (4ee39fb, 16L, 448ch, 448B) +│ │ │ │ ├── 04_milestone_03_sp_Milestone_SetTracking.sql (1a80d4d, 49L, 1580ch, 1580B) +│ │ │ │ ├── 04_milestone_04_sp_Milestone_SetRelevance.sql (7a624f9, 49L, 1595ch, 1595B) +│ │ │ │ ├── 04_milestone_05_sp_Milestone_Delete.sql (3b66538, 53L, 1731ch, 1731B) +│ │ │ │ ├── 04_milestone_06_sp_Milestone_Restore.sql (082d62e, 14L, 389ch, 389B) +│ │ │ │ ├── 04_milestone_07_sp_Milestone_Close.sql (cbe2df9, 49L, 1702ch, 1702B) +│ │ │ │ ├── 04_milestone_08_sp_Milestone_Attach_Project.sql (215a04c, 16L, 437ch, 437B) +│ │ │ │ ├── 04_milestone_09_sp_Milestone_Detach_Project.sql (4301efe, 11L, 272ch, 272B) +│ │ │ │ ├── 05_task_01_sp_Task_Create.sql (a76166a, 30L, 1278ch, 1286B) +│ │ │ │ ├── 05_task_02_sp_Task_Update.sql (871fb68, 18L, 501ch, 501B) +│ │ │ │ ├── 05_task_03_sp_Task_SetTracking.sql (c5962e9, 52L, 1697ch, 1697B) +│ │ │ │ ├── 05_task_04_sp_Task_Delete.sql (a5a3556, 50L, 1569ch, 1569B) +│ │ │ │ ├── 06_note_01_sp_Note_Create_Project.sql (4cef77b, 45L, 1929ch, 1943B) +│ │ │ │ ├── 06_note_02_sp_Note_Create_Milestone.sql (d787a51, 26L, 802ch, 802B) +│ │ │ │ ├── 06_note_03_sp_Note_Create_Task.sql (2672ec9, 27L, 832ch, 832B) +│ │ │ │ ├── 06_note_04_sp_Note_Update.sql (3b23b2b, 15L, 423ch, 423B) +│ │ │ │ ├── 06_note_05_sp_Note_SetRelevance.sql (a0671b3, 46L, 1497ch, 1497B) +│ │ │ │ ├── 06_note_06_sp_Note_Delete.sql (92cf79e, 13L, 361ch, 361B) +│ │ │ │ ├── 06_note_07_sp_Note_Restore.sql (cebc414, 13L, 361ch, 361B) +│ │ │ │ ├── 07_tag_01_sp_Tag_Create.sql (c9cb5c6, 43L, 1708ch, 1718B) +│ │ │ │ ├── 07_tag_02_sp_Tag_Update.sql (f1ef770, 16L, 427ch, 427B) +│ │ │ │ ├── 07_tag_03_sp_Tag_Delete.sql (fbb0ba9, 9L, 235ch, 235B) +│ │ │ │ ├── 07_tag_04_sp_Tag_Attach_Project.sql (9d529b0, 20L, 581ch, 583B) +│ │ │ │ ├── 07_tag_05_sp_Tag_Detach_Project.sql (085be53, 11L, 246ch, 246B) +│ │ │ │ ├── 07_tag_06_sp_Tag_Attach_Memory.sql (62e54c5, 20L, 570ch, 572B) +│ │ │ │ ├── 07_tag_07_sp_Tag_Detach_Memory.sql (a90f20c, 11L, 239ch, 239B) +│ │ │ │ ├── 07_tag_08_sp_Tag_Attach_Artifact.sql (7927a8f, 20L, 592ch, 594B) +│ │ │ │ ├── 07_tag_09_sp_Tag_Detach_Artifact.sql (53caf5a, 11L, 253ch, 253B) +│ │ │ │ ├── 07_tag_10_sp_Tag_Attach_Milestone.sql (57e629b, 20L, 603ch, 605B) +│ │ │ │ ├── 07_tag_11_sp_Tag_Detach_Milestone.sql (2e9afa7, 11L, 260ch, 260B) +│ │ │ │ ├── 07_tag_12_sp_Tag_Attach_Task.sql (56cf133, 22L, 715ch, 717B) +│ │ │ │ ├── 07_tag_13_sp_Tag_Detach_Task.sql (98b5293, 13L, 313ch, 313B) +│ │ │ │ ├── 07_tag_14_sp_Tag_Merge.sql (56146fc, 102L, 3769ch, 3771B) +│ │ │ │ ├── 08_artifact_01_sp_Artifact_Create.sql (a698873, 37L, 1586ch, 1604B) +│ │ │ │ ├── 08_artifact_02_sp_Artifact_Update.sql (5f02c6b, 18L, 525ch, 525B) +│ │ │ │ ├── 08_artifact_03_sp_Artifact_SetRelevance.sql (c6e22b8, 46L, 1549ch, 1549B) +│ │ │ │ ├── 08_artifact_04_sp_Artifact_Delete.sql (3d4809c, 13L, 382ch, 382B) +│ │ │ │ ├── 08_artifact_05_sp_Artifact_Restore.sql (c5c404d, 13L, 382ch, 382B) +│ │ │ │ ├── 08_artifact_06_sp_Artifact_Attach_Milestone.sql (21d4b31, 17L, 486ch, 486B) +│ │ │ │ ├── 08_artifact_07_sp_Artifact_Detach_Milestone.sql (a50a3f4, 12L, 327ch, 327B) +│ │ │ │ ├── 08_artifact_08_sp_Artifact_Attach_Task.sql (e98b307, 20L, 566ch, 566B) +│ │ │ │ ├── 08_artifact_09_sp_Artifact_Detach_Task.sql (1ab70c0, 14L, 380ch, 380B) +│ │ │ │ ├── 09_task_dependency_01_sp_Task_Depend.sql (455bfc3, 63L, 2241ch, 2245B) +│ │ │ │ ├── 09_task_dependency_02_sp_Task_Undepend.sql (c48c2ba, 16L, 481ch, 481B) +│ │ │ │ ├── 10_reference_01_sp_Ref_Create_MemoryDomain.sql (096b206, 23L, 906ch, 906B) +│ │ │ │ ├── 10_reference_02_sp_Ref_Create_MemoryCategory.sql (6984d80, 11L, 303ch, 303B) +│ │ │ │ ├── 10_reference_03_sp_Ref_Create_DependencyVerb.sql (96d53dd, 13L, 352ch, 352B) +│ │ │ │ ├── 10_reference_04_sp_Ref_Delete_MemoryDomain.sql (08c356b, 15L, 413ch, 415B) +│ │ │ │ ├── 10_reference_05_sp_Ref_Delete_MemoryCategory.sql (45dafb0, 15L, 431ch, 433B) +│ │ │ │ ├── 10_reference_06_sp_Ref_Delete_DependencyVerb.sql (e02684b, 17L, 505ch, 507B) +│ │ │ │ ├── 11_cleanup.sql (a54ee52, 232L, 11588ch, 11590B) +│ │ │ │ ├── 12_bulk_tvp_01_sp_Tag_Bulk_Attach_Memory.sql (e7ef1dd, 30L, 1318ch, 1324B) +│ │ │ │ ├── 12_bulk_tvp_02_sp_Memory_Bulk_Touch.sql (c0dd55e, 14L, 498ch, 498B) +│ │ │ │ ├── 12_bulk_tvp_03_sp_Task_Bulk_Depend.sql (06b648b, 64L, 2145ch, 2145B) +│ │ │ │ └── 13_tvf.sql (4d362f7, 41L, 1445ch, 1447B) │ │ │ └── $helpers.ts (2df21d3, 58L, 1722ch, 1722B) │ │ ├── src/ (11) -│ │ │ ├── agent/ (5 files, 0 dirs) -│ │ │ ├── artifact/ (5 files, 0 dirs) -│ │ │ ├── audit/ (3 files, 0 dirs) -│ │ │ ├── core/ (3 files, 0 dirs) -│ │ │ ├── memory/ (5 files, 0 dirs) -│ │ │ ├── milestone/ (5 files, 0 dirs) -│ │ │ ├── note/ (5 files, 0 dirs) -│ │ │ ├── project/ (5 files, 0 dirs) -│ │ │ ├── tag/ (5 files, 0 dirs) -│ │ │ ├── task/ (5 files, 0 dirs) +│ │ │ ├── agent/ (5) +│ │ │ │ ├── commands.ts (1197058, 63L, 1651ch, 1653B) +│ │ │ │ ├── index.ts (795706a, 17L, 461ch, 461B) +│ │ │ │ ├── queries.ts (fae4479, 38L, 932ch, 932B) +│ │ │ │ ├── schema.ts (d8c3a1e, 26L, 903ch, 903B) +│ │ │ │ └── types.ts (7626475, 37L, 942ch, 942B) +│ │ │ ├── artifact/ (5) +│ │ │ │ ├── commands.ts (97117c8, 156L, 4581ch, 4591B) +│ │ │ │ ├── index.ts (e7d4014, 30L, 788ch, 788B) +│ │ │ │ ├── queries.ts (098590a, 41L, 1170ch, 1170B) +│ │ │ │ ├── schema.ts (d1c7700, 78L, 3109ch, 3111B) +│ │ │ │ └── types.ts (05ba5ff, 135L, 3767ch, 3771B) +│ │ │ ├── audit/ (3) +│ │ │ │ ├── index.ts (eafb57a, 20L, 543ch, 545B) +│ │ │ │ ├── queries.ts (926a594, 165L, 5059ch, 5063B) +│ │ │ │ └── types.ts (f766ddc, 55L, 1949ch, 1955B) +│ │ │ ├── core/ (3) +│ │ │ │ ├── context.ts (362812b, 23L, 705ch, 705B) +│ │ │ │ ├── repo.ts (bff75e5, 22L, 591ch, 591B) +│ │ │ │ └── types.ts (a35b84c, 185L, 4971ch, 4975B) +│ │ │ ├── memory/ (5) +│ │ │ │ ├── commands.ts (cc373aa, 214L, 6233ch, 6241B) +│ │ │ │ ├── index.ts (db18571, 35L, 821ch, 821B) +│ │ │ │ ├── queries.ts (da84bc0, 106L, 2866ch, 2872B) +│ │ │ │ ├── schema.ts (08b8333, 111L, 4156ch, 4158B) +│ │ │ │ └── types.ts (700f6c7, 227L, 5954ch, 5956B) +│ │ │ ├── milestone/ (5) +│ │ │ │ ├── commands.ts (7bb780d, 163L, 5026ch, 5036B) +│ │ │ │ ├── index.ts (52484a5, 30L, 792ch, 792B) +│ │ │ │ ├── queries.ts (5d08a25, 55L, 1636ch, 1638B) +│ │ │ │ ├── schema.ts (a74562c, 77L, 3035ch, 3037B) +│ │ │ │ └── types.ts (c6eb97f, 154L, 4248ch, 4250B) +│ │ │ ├── note/ (5) +│ │ │ │ ├── commands.ts (581a39c, 158L, 4422ch, 4422B) +│ │ │ │ ├── index.ts (d724525, 30L, 694ch, 694B) +│ │ │ │ ├── queries.ts (ee764f7, 62L, 1579ch, 1579B) +│ │ │ │ ├── schema.ts (676fa1c, 62L, 2268ch, 2268B) +│ │ │ │ └── types.ts (9876ab9, 174L, 4753ch, 4761B) +│ │ │ ├── project/ (5) +│ │ │ │ ├── commands.ts (33fd537, 70L, 1965ch, 1967B) +│ │ │ │ ├── index.ts (f685a92, 21L, 505ch, 505B) +│ │ │ │ ├── queries.ts (0971d98, 39L, 990ch, 990B) +│ │ │ │ ├── schema.ts (2e2d09b, 33L, 1239ch, 1241B) +│ │ │ │ └── types.ts (ec3dd22, 62L, 1546ch, 1546B) +│ │ │ ├── tag/ (5) +│ │ │ │ ├── commands.ts (2c26931, 245L, 6870ch, 6870B) +│ │ │ │ ├── index.ts (d18e1e8, 35L, 859ch, 859B) +│ │ │ │ ├── queries.ts (97d54a7, 87L, 2416ch, 2416B) +│ │ │ │ ├── schema.ts (27f1a9a, 98L, 3749ch, 3749B) +│ │ │ │ └── types.ts (99d6d3b, 168L, 4070ch, 4072B) +│ │ │ ├── task/ (5) +│ │ │ │ ├── commands.ts (4b670e9, 150L, 4545ch, 4551B) +│ │ │ │ ├── index.ts (d5445d6, 31L, 691ch, 691B) +│ │ │ │ ├── queries.ts (2199234, 122L, 3706ch, 3710B) +│ │ │ │ ├── schema.ts (6bfdf94, 91L, 3359ch, 3361B) +│ │ │ │ └── types.ts (9736df8, 191L, 5044ch, 5048B) │ │ │ └── index.ts (2fe92c8, 55L, 2610ch, 2612B) │ │ ├── tests/ (4) -│ │ │ ├── domain/ (8 files, 0 dirs) -│ │ │ ├── helpers/ (1 file, 0 dirs) -│ │ │ ├── integration/ (5 files, 0 dirs) -│ │ │ └── sql/ (11 files, 0 dirs) +│ │ │ ├── domain/ (8) +│ │ │ │ ├── artifact.test.ts (0678a7b, 253L, 7337ch, 7339B) +│ │ │ │ ├── audit.test.ts (904d2e6, 309L, 9270ch, 9272B) +│ │ │ │ ├── consolidate-and-merge.test.ts (7382d89, 303L, 10036ch, 10038B) +│ │ │ │ ├── memory.test.ts (0801c41, 332L, 9719ch, 9721B) +│ │ │ │ ├── milestone.test.ts (1bfcb35, 355L, 11189ch, 11193B) +│ │ │ │ ├── note.test.ts (fb51d9a, 242L, 7020ch, 7022B) +│ │ │ │ ├── tag.test.ts (f0ceb45, 427L, 13038ch, 13040B) +│ │ │ │ └── task.test.ts (c6c235c, 483L, 14684ch, 14688B) +│ │ │ ├── helpers/ (1) +│ │ │ │ └── test-context.ts (8a09cad, 144L, 4521ch, 4527B) +│ │ │ ├── integration/ (5) +│ │ │ │ ├── impersonation.test.ts (de573cd, 143L, 5156ch, 5162B) +│ │ │ │ ├── lock.test.ts (626f40b, 196L, 7032ch, 7042B) +│ │ │ │ ├── mcp-discovery.test.ts (e0ebd82, 281L, 9493ch, 9513B) +│ │ │ │ ├── observer.test.ts (24006fb, 221L, 7331ch, 7341B) +│ │ │ │ └── vault.test.ts (11a8e96, 179L, 6065ch, 6083B) +│ │ │ └── sql/ (11) +│ │ │ ├── artifact.test.ts (be9a91c, 315L, 8804ch, 8806B) +│ │ │ ├── functions.test.ts (a21fc8d, 377L, 10800ch, 10804B) +│ │ │ ├── memory.test.ts (32d39fc, 569L, 16838ch, 16842B) +│ │ │ ├── milestone.test.ts (5c00166, 326L, 9816ch, 9820B) +│ │ │ ├── note.test.ts (277f3f4, 419L, 11535ch, 11537B) +│ │ │ ├── state-transition.test.ts (61634b9, 383L, 11618ch, 11624B) +│ │ │ ├── tag-tvp-edge-cases.test.ts (9fbf577, 239L, 6714ch, 6723B) +│ │ │ ├── tag.test.ts (e0238a0, 526L, 15650ch, 15650B) +│ │ │ ├── task.test.ts (0e6595f, 349L, 10534ch, 10534B) +│ │ │ ├── tvf.test.ts (c3bc47a, 182L, 4803ch, 4807B) +│ │ │ └── views.test.ts (0ce91fc, 746L, 22141ch, 22887B) │ │ ├── .gitignore (ccb61cd, 40L, 446ch, 446B) │ │ ├── .mcp.json (14f011f, 11L, 174ch, 174B) -│ │ ├── CHANGELOG.md (55fb4ca, 101L, 2269ch, 2269B) +│ │ ├── CHANGELOG.md (4f6a37d, 127L, 2547ch, 2547B) │ │ ├── CLAUDE.md (1f39d31, 111L, 2676ch, 2676B) │ │ ├── README.md (c228f5b, 121L, 6833ch, 6869B) │ │ ├── REPORT.md (4f3efcd, 161L, 12957ch, 13004B) │ │ ├── mcp-config.json (14f011f, 11L, 174ch, 174B) │ │ ├── mssql-problems.md (5083524, 328L, 23834ch, 23934B) -│ │ ├── package.json (484acdb, 23L, 623ch, 623B) +│ │ ├── package.json (09c895f, 23L, 623ch, 623B) │ │ └── tsconfig.json (7be3bae, 27L, 736ch, 736B) │ ├── llm-memory-db-pg/ (17) │ │ ├── .cursor/ (1) -│ │ │ └── rules/ (1 file, 0 dirs) +│ │ │ └── rules/ (1) +│ │ │ └── use-bun-instead-of-node-vite-npm-pnpm.mdc (1f39d31, 111L, 2676ch, 2676B) │ │ ├── .noorm/ (2) │ │ │ ├── .gitignore (14188a3, 1L, 7ch, 7B) │ │ │ └── settings.yml (015d7e5, 50L, 1111ch, 1111B) │ │ ├── changes/ (2) -│ │ │ ├── 2026-05-10-add-memory-tag-color/ (1 file, 2 dirs) +│ │ │ ├── 2026-05-10-add-memory-tag-color/ (3) +│ │ │ │ ├── change/ (2 files, 0 dirs) +│ │ │ │ ├── revert/ (2 files, 0 dirs) +│ │ │ │ └── changelog.md (c7ef82a, 20L, 946ch, 954B) │ │ │ └── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ │ ├── sql/ (12) -│ │ │ ├── 00_types/ (1 file, 0 dirs) -│ │ │ ├── 01_reference/ (10 files, 0 dirs) -│ │ │ ├── 02_tables/ (9 files, 0 dirs) -│ │ │ ├── 03_subtypes/ (8 files, 0 dirs) -│ │ │ ├── 04_binary/ (11 files, 0 dirs) -│ │ │ ├── 05_seeds/ (5 files, 0 dirs) -│ │ │ ├── 06_functions/ (10 files, 0 dirs) -│ │ │ ├── 07_views/ (18 files, 0 dirs) -│ │ │ ├── 08_procedures/ (0 files, 10 dirs) -│ │ │ ├── 09_triggers/ (2 files, 0 dirs) +│ │ │ ├── 00_types/ (1) +│ │ │ │ └── .gitkeep (e3b0c44, 0L, 0ch, 0B) +│ │ │ ├── 01_reference/ (10) +│ │ │ │ ├── 01_relevance_status.sql (602fbbd, 8L, 286ch, 286B) +│ │ │ │ ├── 02_tracking_status.sql (30e90a5, 8L, 283ch, 283B) +│ │ │ │ ├── 03_note_type.sql (68f08e5, 8L, 265ch, 265B) +│ │ │ │ ├── 04_memory_domain.sql (b1b76b5, 8L, 270ch, 270B) +│ │ │ │ ├── 05_memory_category.sql (57e1950, 8L, 276ch, 276B) +│ │ │ │ ├── 06_dependency_verb.sql (119b0b9, 8L, 283ch, 283B) +│ │ │ │ ├── 07_memory_relation_verb.sql (7b3e37a, 9L, 334ch, 334B) +│ │ │ │ ├── 08_state_transition_type.sql (5fd32a5, 8L, 307ch, 307B) +│ │ │ │ ├── 09_tracking_status_allowed.sql (fb8aeb8, 11L, 542ch, 542B) +│ │ │ │ └── 10_relevance_status_allowed.sql (6c2ef55, 11L, 548ch, 548B) +│ │ │ ├── 02_tables/ (9) +│ │ │ │ ├── 01_agent.sql (43fcf11, 11L, 480ch, 480B) +│ │ │ │ ├── 02_project.sql (57a95af, 18L, 881ch, 881B) +│ │ │ │ ├── 03_note.sql (e23d9be, 24L, 1403ch, 1403B) +│ │ │ │ ├── 04_tag.sql (9969e60, 19L, 963ch, 963B) +│ │ │ │ ├── 05_memory.sql (b5dd536, 34L, 2075ch, 2075B) +│ │ │ │ ├── 06_artifact.sql (e00e680, 23L, 1354ch, 1354B) +│ │ │ │ ├── 07_milestone.sql (1e4e9ca, 25L, 1577ch, 1577B) +│ │ │ │ ├── 08_state_transition.sql (f0305f6, 25L, 1539ch, 1541B) +│ │ │ │ └── 09_task.sql (7048d72, 25L, 1428ch, 1428B) +│ │ │ ├── 03_subtypes/ (8) +│ │ │ │ ├── 01_project_note.sql (59db285, 14L, 675ch, 675B) +│ │ │ │ ├── 02_milestone_note.sql (05b0743, 14L, 701ch, 701B) +│ │ │ │ ├── 03_task_note.sql (38b6c68, 15L, 711ch, 711B) +│ │ │ │ ├── 04_milestone_state_transition.sql (afa21a3, 14L, 818ch, 818B) +│ │ │ │ ├── 05_task_state_transition.sql (80517e8, 15L, 829ch, 829B) +│ │ │ │ ├── 06_memory_state_transition.sql (b6929df, 14L, 779ch, 779B) +│ │ │ │ ├── 07_note_state_transition.sql (d830640, 14L, 753ch, 753B) +│ │ │ │ └── 08_artifact_state_transition.sql (7e00661, 14L, 805ch, 805B) +│ │ │ ├── 04_binary/ (11) +│ │ │ │ ├── 01_project_tag.sql (3d87ac3, 13L, 640ch, 640B) +│ │ │ │ ├── 02_memory_tag.sql (3aab87e, 13L, 626ch, 626B) +│ │ │ │ ├── 03_artifact_tag.sql (576a702, 13L, 654ch, 654B) +│ │ │ │ ├── 04_milestone_tag.sql (d0ff467, 13L, 668ch, 668B) +│ │ │ │ ├── 05_task_tag.sql (dacc072, 14L, 702ch, 702B) +│ │ │ │ ├── 06_project_memory.sql (f720b9a, 13L, 674ch, 674B) +│ │ │ │ ├── 07_project_milestone.sql (02f1456, 13L, 716ch, 716B) +│ │ │ │ ├── 08_related_memory.sql (4dd1d59, 17L, 1051ch, 1051B) +│ │ │ │ ├── 09_milestone_artifact.sql (0af8ea0, 13L, 726ch, 726B) +│ │ │ │ ├── 10_task_artifact.sql (df93cf6, 14L, 743ch, 743B) +│ │ │ │ └── 11_task_dependency.sql (7efbe70, 19L, 1221ch, 1221B) +│ │ │ ├── 05_seeds/ (5) +│ │ │ │ ├── 01_reference_values.sql.tmpl (17c9084, 35L, 1630ch, 1630B) +│ │ │ │ ├── 02_transitions_allowed.sql.tmpl (9a2913a, 11L, 630ch, 630B) +│ │ │ │ ├── 03_sentinel_rows.sql.tmpl (9d8f4be, 19L, 1140ch, 1140B) +│ │ │ │ ├── reference_values.yml (c64adae, 78L, 1404ch, 1404B) +│ │ │ │ └── transitions_allowed.yml (1af4d0c, 64L, 1435ch, 1435B) +│ │ │ ├── 06_functions/ (10) +│ │ │ │ ├── 01_fn_next_task_no.sql (50bb032, 14L, 419ch, 419B) +│ │ │ │ ├── 02_fn_note_subtype_count.sql (7d6c6b4, 16L, 750ch, 750B) +│ │ │ │ ├── 03_fn_note_matches_subtype.sql (a6afc7c, 21L, 911ch, 911B) +│ │ │ │ ├── 04_fn_is_active.sql (6077554, 11L, 327ch, 327B) +│ │ │ │ ├── 05_fn_is_open.sql (26fa57f, 12L, 390ch, 390B) +│ │ │ │ ├── 06_fn_memory_confidence.sql (ae180c8, 27L, 805ch, 807B) +│ │ │ │ ├── 07_fn_is_tracking_transition_allowed.sql (4ba70e4, 20L, 589ch, 589B) +│ │ │ │ ├── 08_fn_is_relevance_transition_allowed.sql (c9db56e, 20L, 613ch, 613B) +│ │ │ │ ├── 09_fn_task_dependency_would_cycle.sql (51f827f, 40L, 1171ch, 1173B) +│ │ │ │ └── 10_fn_memory_rank.sql (42816f5, 67L, 2176ch, 2180B) +│ │ │ ├── 07_views/ (18) +│ │ │ │ ├── 01_vw_tag.sql (725eb5b, 97L, 2966ch, 2966B) +│ │ │ │ ├── 02_vw_artifact.sql (d4c1f6f, 41L, 1301ch, 1303B) +│ │ │ │ ├── 03_vw_note.sql (1954486, 27L, 999ch, 1001B) +│ │ │ │ ├── 04_vw_related_memory.sql (bb4c23a, 30L, 1086ch, 1088B) +│ │ │ │ ├── 05_vw_milestone_stats.sql (6264b85, 93L, 3194ch, 3194B) +│ │ │ │ ├── 06_vw_memory.sql (079720e, 29L, 790ch, 790B) +│ │ │ │ ├── 07_vw_active_memory.sql (e33004d, 29L, 804ch, 804B) +│ │ │ │ ├── 08_vw_active_note.sql (1782c54, 21L, 578ch, 580B) +│ │ │ │ ├── 09_vw_active_artifact.sql (f54b1c0, 21L, 547ch, 547B) +│ │ │ │ ├── 10_vw_active_milestone.sql (fc2ac2e, 21L, 523ch, 523B) +│ │ │ │ ├── 11_vw_deleted_memory.sql (17c5828, 30L, 906ch, 908B) +│ │ │ │ ├── 12_vw_deleted_note.sql (1bbeb90, 20L, 475ch, 475B) +│ │ │ │ ├── 13_vw_deleted_artifact.sql (db47434, 21L, 519ch, 519B) +│ │ │ │ ├── 14_vw_deleted_milestone.sql (e022e0d, 21L, 528ch, 528B) +│ │ │ │ ├── 15_vw_task_backlog.sql (02267e5, 35L, 1220ch, 1220B) +│ │ │ │ ├── 16_vw_state_transition.sql (c988e3f, 30L, 1441ch, 1441B) +│ │ │ │ ├── 17_vw_recent_activity.sql (5a5bbf0, 217L, 6253ch, 6253B) +│ │ │ │ └── 18_vw_agent_activity.sql (30f41da, 57L, 2360ch, 2362B) +│ │ │ ├── 08_procedures/ (10) +│ │ │ │ ├── agent/ (3 files, 0 dirs) +│ │ │ │ ├── artifact/ (9 files, 0 dirs) +│ │ │ │ ├── cleanup/ (1 file, 0 dirs) +│ │ │ │ ├── memory/ (11 files, 0 dirs) +│ │ │ │ ├── milestone/ (9 files, 0 dirs) +│ │ │ │ ├── note/ (7 files, 0 dirs) +│ │ │ │ ├── project/ (3 files, 0 dirs) +│ │ │ │ ├── ref/ (6 files, 0 dirs) +│ │ │ │ ├── tag/ (14 files, 0 dirs) +│ │ │ │ └── task/ (6 files, 0 dirs) +│ │ │ ├── 09_triggers/ (2) +│ │ │ │ ├── 01_note_exclusivity_trigger.sql (e86ad3a, 132L, 5699ch, 5703B) +│ │ │ │ └── 02_state_transition_exclusivity_trigger.sql (840e660, 164L, 8657ch, 8661B) │ │ │ ├── $helpers.ts (7bec322, 38L, 982ch, 982B) │ │ │ └── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ │ ├── src/ (11) -│ │ │ ├── agent/ (5 files, 0 dirs) -│ │ │ ├── artifact/ (5 files, 0 dirs) -│ │ │ ├── audit/ (4 files, 0 dirs) -│ │ │ ├── core/ (4 files, 0 dirs) -│ │ │ ├── memory/ (5 files, 0 dirs) -│ │ │ ├── milestone/ (5 files, 0 dirs) -│ │ │ ├── note/ (5 files, 0 dirs) -│ │ │ ├── project/ (5 files, 0 dirs) -│ │ │ ├── tag/ (5 files, 0 dirs) -│ │ │ ├── task/ (5 files, 0 dirs) +│ │ │ ├── agent/ (5) +│ │ │ │ ├── commands.ts (7a1f686, 105L, 2955ch, 2957B) +│ │ │ │ ├── index.ts (3b8791e, 10L, 235ch, 235B) +│ │ │ │ ├── queries.ts (1ef5bd2, 96L, 2909ch, 2921B) +│ │ │ │ ├── schema.ts (6109b9b, 57L, 1808ch, 1814B) +│ │ │ │ └── types.ts (c7d0385, 71L, 2016ch, 2024B) +│ │ │ ├── artifact/ (5) +│ │ │ │ ├── commands.ts (21ef505, 273L, 8303ch, 8315B) +│ │ │ │ ├── index.ts (1d71cfd, 15L, 463ch, 463B) +│ │ │ │ ├── queries.ts (09ff50d, 137L, 4050ch, 4050B) +│ │ │ │ ├── schema.ts (e2a5b72, 210L, 6579ch, 6593B) +│ │ │ │ └── types.ts (717e911, 111L, 3430ch, 3432B) +│ │ │ ├── audit/ (4) +│ │ │ │ ├── index.ts (f79d579, 14L, 305ch, 305B) +│ │ │ │ ├── queries.ts (79ab9eb, 328L, 9701ch, 9719B) +│ │ │ │ ├── schema.ts (1f6bb89, 155L, 4576ch, 4582B) +│ │ │ │ └── types.ts (e3e9f23, 34L, 1203ch, 1205B) +│ │ │ ├── core/ (4) +│ │ │ │ ├── context.ts (1cf7b98, 37L, 1002ch, 1004B) +│ │ │ │ ├── index.ts (654d15d, 65L, 1323ch, 1323B) +│ │ │ │ ├── repo.ts (ee8b3c8, 34L, 821ch, 821B) +│ │ │ │ └── types.ts (5e6f7e5, 576L, 14359ch, 14953B) +│ │ │ ├── memory/ (5) +│ │ │ │ ├── commands.ts (469df41, 353L, 10552ch, 10568B) +│ │ │ │ ├── index.ts (c5dfda3, 18L, 488ch, 488B) +│ │ │ │ ├── queries.ts (6276a6a, 278L, 8110ch, 8130B) +│ │ │ │ ├── schema.ts (74fd7ef, 318L, 10035ch, 10055B) +│ │ │ │ └── types.ts (ba60ea9, 220L, 6441ch, 6455B) +│ │ │ ├── milestone/ (5) +│ │ │ │ ├── commands.ts (8e4fc23, 264L, 8164ch, 8180B) +│ │ │ │ ├── index.ts (80cbda7, 15L, 453ch, 453B) +│ │ │ │ ├── queries.ts (38c0dce, 171L, 5117ch, 5127B) +│ │ │ │ ├── schema.ts (a540330, 231L, 7328ch, 7348B) +│ │ │ │ └── types.ts (b5000bf, 114L, 3484ch, 3488B) +│ │ │ ├── note/ (5) +│ │ │ │ ├── commands.ts (9c7d00e, 263L, 7999ch, 8003B) +│ │ │ │ ├── index.ts (7e405d4, 13L, 351ch, 351B) +│ │ │ │ ├── queries.ts (c28cffb, 207L, 5531ch, 5533B) +│ │ │ │ ├── schema.ts (25b4040, 190L, 6011ch, 6017B) +│ │ │ │ └── types.ts (02402c3, 126L, 3815ch, 3817B) +│ │ │ ├── project/ (5) +│ │ │ │ ├── commands.ts (788e2c3, 111L, 3355ch, 3355B) +│ │ │ │ ├── index.ts (2f10233, 8L, 247ch, 247B) +│ │ │ │ ├── queries.ts (d627270, 98L, 2802ch, 2806B) +│ │ │ │ ├── schema.ts (7072c77, 67L, 2166ch, 2172B) +│ │ │ │ └── types.ts (e7195f8, 76L, 2149ch, 2149B) +│ │ │ ├── tag/ (5) +│ │ │ │ ├── commands.ts (47ca7df, 361L, 10180ch, 10196B) +│ │ │ │ ├── index.ts (c815305, 21L, 538ch, 538B) +│ │ │ │ ├── queries.ts (52a8a33, 232L, 6432ch, 6436B) +│ │ │ │ ├── schema.ts (7b6aecd, 205L, 6867ch, 7027B) +│ │ │ │ └── types.ts (3dd4812, 134L, 3990ch, 3998B) +│ │ │ ├── task/ (5) +│ │ │ │ ├── commands.ts (6c4956d, 218L, 7011ch, 7017B) +│ │ │ │ ├── index.ts (61dde84, 13L, 334ch, 334B) +│ │ │ │ ├── queries.ts (ab3e7f0, 212L, 6708ch, 6722B) +│ │ │ │ ├── schema.ts (bb43e49, 223L, 7276ch, 7292B) +│ │ │ │ └── types.ts (eeb5b68, 152L, 4771ch, 4783B) │ │ │ └── index.ts (746abeb, 57L, 2636ch, 2640B) │ │ ├── tests/ (5) -│ │ │ ├── domain/ (8 files, 0 dirs) -│ │ │ ├── helpers/ (1 file, 0 dirs) -│ │ │ ├── integration/ (4 files, 0 dirs) -│ │ │ ├── sql/ (13 files, 0 dirs) +│ │ │ ├── domain/ (8) +│ │ │ │ ├── 01_memory.test.ts (22159c9, 288L, 8726ch, 8728B) +│ │ │ │ ├── 02_note.test.ts (b63ffeb, 203L, 5560ch, 5560B) +│ │ │ │ ├── 03_tag.test.ts (7e45330, 164L, 4159ch, 4159B) +│ │ │ │ ├── 04_artifact.test.ts (e5ab384, 192L, 5402ch, 5402B) +│ │ │ │ ├── 05_milestone.test.ts (5eae375, 178L, 5057ch, 5061B) +│ │ │ │ ├── 06_task.test.ts (0c2b04c, 245L, 7056ch, 7060B) +│ │ │ │ ├── 07_consolidate_and_merge.test.ts (2e10041, 147L, 5546ch, 5546B) +│ │ │ │ └── 08_audit.test.ts (d4be495, 160L, 4768ch, 4768B) +│ │ │ ├── helpers/ (1) +│ │ │ │ └── test-context.ts (29c33b3, 75L, 2618ch, 2626B) +│ │ │ ├── integration/ (4) +│ │ │ │ ├── 01_observer.test.ts (df358d0, 194L, 6459ch, 6463B) +│ │ │ │ ├── 02_lock.test.ts (bbe592c, 181L, 6566ch, 6578B) +│ │ │ │ ├── 03_vault.test.ts (2c4856a, 109L, 2938ch, 2940B) +│ │ │ │ └── 04_impersonation.test.ts (9f75aa3, 173L, 7028ch, 7038B) +│ │ │ ├── sql/ (13) +│ │ │ │ ├── 00_agent.test.ts (3d6d2f3, 235L, 7605ch, 7605B) +│ │ │ │ ├── 00b_project.test.ts (c86d286, 273L, 9158ch, 9158B) +│ │ │ │ ├── 01_memory.test.ts (395a354, 425L, 13542ch, 13542B) +│ │ │ │ ├── 02_note.test.ts (f4e2a5d, 284L, 9275ch, 9275B) +│ │ │ │ ├── 03_tag.test.ts (532cfcb, 402L, 12049ch, 12049B) +│ │ │ │ ├── 04_artifact.test.ts (e51c017, 250L, 7493ch, 7493B) +│ │ │ │ ├── 05_milestone.test.ts (02536ee, 325L, 10280ch, 10282B) +│ │ │ │ ├── 06_task.test.ts (6c7d356, 276L, 9119ch, 9121B) +│ │ │ │ ├── 07_state_transition.test.ts (385c11c, 368L, 12746ch, 12746B) +│ │ │ │ ├── 08_views.test.ts (5e43537, 978L, 35518ch, 35524B) +│ │ │ │ ├── 09_functions.test.ts (c8cfc93, 541L, 16533ch, 16535B) +│ │ │ │ ├── 10_cleanup.test.ts (3df0597, 276L, 8981ch, 8983B) +│ │ │ │ └── 11_ref.test.ts (f5c0402, 307L, 8851ch, 8855B) │ │ │ └── mcp-discovery.test.ts (02d4035, 765L, 24559ch, 24603B) │ │ ├── .gitignore (a94396a, 36L, 397ch, 397B) │ │ ├── .mcp.json (14f011f, 11L, 174ch, 174B) -│ │ ├── CHANGELOG.md (3ac7a7b, 101L, 2266ch, 2266B) +│ │ ├── CHANGELOG.md (2ff7198, 127L, 2544ch, 2544B) │ │ ├── CLAUDE.md (1f39d31, 111L, 2676ch, 2676B) │ │ ├── README.md (b97f95b, 205L, 10359ch, 10665B) │ │ ├── REPORT-PHASE-1.md (59b7d41, 103L, 9610ch, 9670B) │ │ ├── REPORT.md (98ee180, 140L, 17043ch, 17113B) │ │ ├── mcp-config.json (14f011f, 11L, 174ch, 174B) -│ │ ├── package.json (129231a, 23L, 662ch, 662B) +│ │ ├── package.json (2eae516, 23L, 662ch, 662B) │ │ ├── postgres-problems.md (1cbb5b5, 200L, 13674ch, 13799B) │ │ └── tsconfig.json (4dc04b1, 30L, 735ch, 735B) │ └── todo-db/ (10) │ ├── .noorm/ (1) │ │ └── settings.yml (edb7584, 18L, 270ch, 270B) │ ├── changes/ (3) -│ │ ├── 2026-01-04-add-functions-and-views/ (1 file, 2 dirs) -│ │ ├── 2026-02-15-soft-delete-metadata-and-bulk-ops/ (1 file, 2 dirs) +│ │ ├── 2026-01-04-add-functions-and-views/ (3) +│ │ │ ├── change/ (2 files, 0 dirs) +│ │ │ ├── revert/ (2 files, 0 dirs) +│ │ │ └── changelog.md (03568a1, 14L, 215ch, 215B) +│ │ ├── 2026-02-15-soft-delete-metadata-and-bulk-ops/ (3) +│ │ │ ├── change/ (4 files, 0 dirs) +│ │ │ ├── revert/ (3 files, 0 dirs) +│ │ │ └── changelog.md (b7dbad8, 47L, 2425ch, 2447B) │ │ └── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ ├── export/ (11) │ │ ├── category.dt (5d47f0a, 1L, 295ch, 295B) @@ -318,43 +808,87 @@ │ │ ├── user_tag.dt (bd4000f, 1L, 256ch, 256B) │ │ └── version.dt (cd68ba3, 1L, 54ch, 54B) │ ├── sql/ (5) -│ │ ├── 00_tables/ (7 files, 0 dirs) -│ │ ├── 01_views/ (0 files, 4 dirs) -│ │ ├── 02_functions/ (0 files, 6 dirs) -│ │ ├── 10_seeds/ (5 files, 1 dir) +│ │ ├── 00_tables/ (7) +│ │ │ ├── 00_types.sql (73e9f27, 21L, 852ch, 852B) +│ │ │ ├── 01_core.sql (8536cbf, 26L, 1157ch, 1157B) +│ │ │ ├── 02_organization.sql (56b4c9d, 32L, 1146ch, 1146B) +│ │ │ ├── 03_task.sql (36c361c, 85L, 3355ch, 3355B) +│ │ │ ├── 04_association.sql (98171c6, 89L, 3248ch, 3248B) +│ │ │ ├── 05_cron.sql (2ef3d19, 79L, 3423ch, 3425B) +│ │ │ └── 06_feature_flag.sql (60c04f7, 25L, 1233ch, 1233B) +│ │ ├── 01_views/ (4) +│ │ │ ├── tag/ (2 files, 0 dirs) +│ │ │ ├── todo/ (4 files, 0 dirs) +│ │ │ ├── todo_item/ (2 files, 0 dirs) +│ │ │ └── user/ (3 files, 0 dirs) +│ │ ├── 02_functions/ (6) +│ │ │ ├── category/ (5 files, 0 dirs) +│ │ │ ├── tag/ (6 files, 0 dirs) +│ │ │ ├── tagging/ (6 files, 0 dirs) +│ │ │ ├── todo/ (9 files, 0 dirs) +│ │ │ ├── todo_item/ (6 files, 0 dirs) +│ │ │ └── user/ (7 files, 0 dirs) +│ │ ├── 10_seeds/ (6) +│ │ │ ├── cron/ (3 files, 0 dirs) +│ │ │ ├── feature_flags.sql.tmpl (ec09cf0, 36L, 1526ch, 1526B) +│ │ │ ├── feature_flags.yml (ccfc197, 35L, 1314ch, 1316B) +│ │ │ ├── roles.json (67afb68, 14L, 223ch, 223B) +│ │ │ ├── seed_data.sql.tmpl (587d315, 23L, 722ch, 722B) +│ │ │ └── users.yml (171c41e, 14L, 254ch, 254B) │ │ └── .gitkeep (e3b0c44, 0L, 0ch, 0B) │ ├── tests/ (7) -│ │ ├── _helpers/ (4 files, 0 dirs) -│ │ ├── edge/ (1 file, 0 dirs) -│ │ ├── functions/ (7 files, 0 dirs) -│ │ ├── seeds/ (1 file, 0 dirs) -│ │ ├── tables/ (6 files, 0 dirs) -│ │ ├── views/ (2 files, 0 dirs) +│ │ ├── _helpers/ (4) +│ │ │ ├── context.ts (3bd7b67, 88L, 2747ch, 2749B) +│ │ │ ├── project.ts (55c50ae, 62L, 1909ch, 1913B) +│ │ │ ├── schema.ts (9466228, 428L, 13450ch, 13458B) +│ │ │ └── setup.ts (b109f03, 229L, 6151ch, 6165B) +│ │ ├── edge/ (1) +│ │ │ └── failure_modes.test.ts (b86cc8c, 491L, 15541ch, 16767B) +│ │ ├── functions/ (7) +│ │ │ ├── bulk_create_tags.test.ts (e095252, 154L, 4545ch, 4551B) +│ │ │ ├── complete_todo.test.ts (e5ca355, 217L, 6574ch, 6578B) +│ │ │ ├── list_categories_tags.test.ts (b748cae, 122L, 3233ch, 3233B) +│ │ │ ├── list_users.test.ts (31335c7, 128L, 3557ch, 3557B) +│ │ │ ├── search_todos.test.ts (aed8d6d, 297L, 9208ch, 9212B) +│ │ │ ├── soft_delete_user.test.ts (33e9012, 140L, 4095ch, 4097B) +│ │ │ └── todo_crud.test.ts (70312b1, 252L, 8387ch, 8399B) +│ │ ├── seeds/ (1) +│ │ │ └── cron_and_flags.test.ts (8b73fc1, 140L, 4802ch, 4822B) +│ │ ├── tables/ (6) +│ │ │ ├── category.test.ts (1b6aa87, 160L, 4271ch, 4273B) +│ │ │ ├── jsonb_metadata.test.ts (5fdfabf, 231L, 6982ch, 6986B) +│ │ │ ├── tag.test.ts (ec9341d, 145L, 3620ch, 3622B) +│ │ │ ├── todo.test.ts (7039eab, 290L, 8495ch, 8497B) +│ │ │ ├── todo_item.test.ts (586470a, 266L, 8881ch, 8883B) +│ │ │ └── user.test.ts (da0ab19, 149L, 4373ch, 4375B) +│ │ ├── views/ (2) +│ │ │ ├── active_users.test.ts (a606b79, 144L, 4060ch, 4062B) +│ │ │ └── todos_with_details.test.ts (76bc932, 215L, 6840ch, 6842B) │ │ └── preload.ts (0d97cd6, 25L, 658ch, 660B) │ ├── .gitignore (81531bd, 5L, 57ch, 57B) -│ ├── CHANGELOG.md (ce8eb26, 107L, 2326ch, 2326B) +│ ├── CHANGELOG.md (e765a76, 133L, 2604ch, 2604B) │ ├── bunfig.toml (e10e7cb, 5L, 89ch, 89B) -│ ├── package.json (37c659a, 22L, 573ch, 573B) +│ ├── package.json (bc715ae, 22L, 573ch, 573B) │ └── tsconfig.json (efeae48, 17L, 484ch, 484B) ├── packages/ (2) │ ├── cli/ (7) │ │ ├── scripts/ (1) │ │ │ └── postinstall.js (b82655f, 411L, 12511ch, 12513B) -│ │ ├── CHANGELOG.md (1fd031e, 1499L, 106203ch, 106721B) +│ │ ├── CHANGELOG.md (ceae00f, 1603L, 112727ch, 113265B) │ │ ├── LICENSE (cfc7749, 202L, 11358ch, 11358B) │ │ ├── NOTICE (d464ce1, 2L, 43ch, 43B) │ │ ├── README.md (5ec4b13, 65L, 1728ch, 1734B) │ │ ├── noorm.js (e3d76e8, 41L, 1041ch, 1043B) -│ │ └── package.json (26c70c5, 32L, 563ch, 563B) +│ │ └── package.json (5b81351, 32L, 563ch, 563B) │ └── sdk/ (5) -│ ├── CHANGELOG.md (d978dc8, 1199L, 78024ch, 78462B) +│ ├── CHANGELOG.md (146ccc1, 1223L, 78839ch, 79281B) │ ├── LICENSE (cfc7749, 202L, 11358ch, 11358B) │ ├── NOTICE (5efbb3e, 2L, 43ch, 43B) │ ├── README.md (4cd7cfa, 104L, 3525ch, 3541B) -│ └── package.json (cbc8fef, 62L, 1135ch, 1135B) +│ └── package.json (85ffb22, 62L, 1135ch, 1135B) ├── scripts/ (5) │ ├── Dockerfile (5fe0d7a, 51L, 1595ch, 1595B) -│ ├── build-binary.mjs (f598b0c, 37L, 1284ch, 1288B) +│ ├── build-binary.mjs (3003f29, 41L, 1627ch, 1631B) │ ├── build.mjs (303daea, 40L, 1519ch, 1519B) │ ├── check-flag-placement.sh (3796ab7, 66L, 3247ch, 3261B) │ └── ralph-wiggum.sh (7182e0a, 319L, 8973ch, 8973B) @@ -383,7 +917,10 @@ │ │ │ ├── rm.ts (d398cab, 158L, 5016ch, 5018B) │ │ │ └── run.ts (4a3a859, 135L, 3794ch, 3796B) │ │ ├── ci/ (4) -│ │ │ ├── identity/ (3 files, 0 dirs) +│ │ │ ├── identity/ (3) +│ │ │ │ ├── enroll.ts (70b3a23, 368L, 14062ch, 14074B) +│ │ │ │ ├── index.ts (9f3d6bc, 18L, 355ch, 357B) +│ │ │ │ └── new.ts (90de2fe, 113L, 3490ch, 3492B) │ │ │ ├── index.ts (edb5806, 26L, 673ch, 679B) │ │ │ ├── init.ts (fd60bc5, 296L, 10286ch, 10294B) │ │ │ └── secrets.ts (9b159a4, 235L, 6521ch, 6525B) @@ -479,12 +1016,12 @@ │ │ ├── index.ts (2e302ae, 391L, 13727ch, 13745B) │ │ ├── info.ts (fa5e2b1, 384L, 11512ch, 11516B) │ │ ├── init.ts (e04cc7c, 177L, 5303ch, 5305B) -│ │ ├── ui.ts (28a7a9c, 70L, 1869ch, 1875B) -│ │ ├── update.ts (13469b2, 219L, 6573ch, 6581B) +│ │ ├── ui.ts (6d369c6, 75L, 2214ch, 2220B) +│ │ ├── update.ts (2233042, 218L, 6633ch, 6641B) │ │ └── version.ts (ca59073, 273L, 6880ch, 6890B) │ ├── core/ (30) │ │ ├── change/ (9) -│ │ │ ├── executor.ts (1e7f9af, 1331L, 36554ch, 38038B) +│ │ │ ├── executor.ts (948525e, 1340L, 36909ch, 38393B) │ │ │ ├── history.ts (1ef7798, 1266L, 38071ch, 39941B) │ │ │ ├── index.ts (b0a2184, 131L, 3071ch, 4779B) │ │ │ ├── manager.ts (05fe831, 653L, 17287ch, 19599B) @@ -496,18 +1033,32 @@ │ │ ├── config/ (5) │ │ │ ├── index.ts (3d6e44e, 127L, 3484ch, 3484B) │ │ │ ├── resolver.ts (f2cd9c7, 505L, 13582ch, 13590B) -│ │ │ ├── schema.ts (79f5b64, 346L, 9526ch, 10020B) +│ │ │ ├── schema.ts (106c8ce, 348L, 9654ch, 10148B) │ │ │ ├── types.ts (24b482c, 143L, 3480ch, 3728B) │ │ │ └── validate.ts (7ebdb26, 89L, 2464ch, 2464B) -│ │ ├── connection/ (6) -│ │ │ ├── dialects/ (7 files, 0 dirs) -│ │ │ ├── defaults.ts (42ecd2d, 27L, 685ch, 685B) -│ │ │ ├── factory.ts (1388e22, 311L, 9476ch, 9482B) -│ │ │ ├── index.ts (fb5e6ea, 10L, 387ch, 387B) +│ │ ├── connection/ (8) +│ │ │ ├── dialects/ (7) +│ │ │ │ ├── bun-sqlite.d.ts (a6ea1dd, 29L, 800ch, 800B) +│ │ │ │ ├── mssql-limit-plugin.ts (3568b0f, 118L, 3209ch, 3211B) +│ │ │ │ ├── mssql.ts (aa770db, 250L, 8265ch, 8269B) +│ │ │ │ ├── mysql.ts (63e5948, 54L, 1538ch, 1538B) +│ │ │ │ ├── postgres.ts (3f11d03, 57L, 1536ch, 1536B) +│ │ │ │ ├── sqlite-bun.ts (86656b8, 121L, 2857ch, 2859B) +│ │ │ │ └── sqlite.ts (722e606, 39L, 986ch, 986B) +│ │ │ ├── defaults.ts (9d83755, 64L, 2026ch, 2030B) +│ │ │ ├── errors.ts (fb14a6c, 418L, 14420ch, 14420B) +│ │ │ ├── factory.ts (c745810, 471L, 15327ch, 15337B) +│ │ │ ├── index.ts (5c22501, 11L, 508ch, 508B) │ │ │ ├── manager.ts (8152aa7, 348L, 8120ch, 8120B) -│ │ │ └── types.ts (b0950c0, 86L, 1911ch, 1911B) +│ │ │ ├── session.ts (883d47c, 87L, 3230ch, 3230B) +│ │ │ └── types.ts (a5e604a, 96L, 2311ch, 2313B) │ │ ├── db/ (6) -│ │ │ ├── dialects/ (5 files, 0 dirs) +│ │ │ ├── dialects/ (5) +│ │ │ │ ├── index.ts (8d46a23, 37L, 1079ch, 1079B) +│ │ │ │ ├── mssql.ts (204bc6a, 137L, 3820ch, 3824B) +│ │ │ │ ├── mysql.ts (0fb202c, 111L, 2802ch, 2806B) +│ │ │ │ ├── postgres.ts (cad492f, 123L, 3213ch, 3217B) +│ │ │ │ └── sqlite.ts (62cd22b, 65L, 1507ch, 1507B) │ │ │ ├── dual.ts (baaddd4, 174L, 4914ch, 4914B) │ │ │ ├── index.ts (7a8c9d2, 54L, 1314ch, 1314B) │ │ │ ├── operations.ts (0ea9ee9, 286L, 7664ch, 7668B) @@ -517,7 +1068,11 @@ │ │ │ ├── index.ts (66ccc4c, 22L, 409ch, 409B) │ │ │ └── operations.ts (25178e2, 548L, 15077ch, 15821B) │ │ ├── dt/ (17) -│ │ │ ├── dialects/ (4 files, 0 dirs) +│ │ │ ├── dialects/ (4) +│ │ │ │ ├── index.ts (7a32ad3, 86L, 2361ch, 2363B) +│ │ │ │ ├── mssql.ts (0e4b51a, 155L, 4972ch, 4978B) +│ │ │ │ ├── mysql.ts (8455eb2, 151L, 4706ch, 4712B) +│ │ │ │ └── postgres.ts (597dd33, 125L, 4438ch, 4446B) │ │ │ ├── constants.ts (289c9e4, 104L, 3005ch, 3011B) │ │ │ ├── crypto.ts (7586f37, 123L, 3783ch, 3787B) │ │ │ ├── deserialize.ts (8a485d0, 408L, 10546ch, 10552B) @@ -534,11 +1089,17 @@ │ │ │ ├── types.ts (e11c4e4, 413L, 9366ch, 9366B) │ │ │ ├── version.ts (6a13dc2, 216L, 5118ch, 5128B) │ │ │ └── writer.ts (3525333, 233L, 5667ch, 5667B) -│ │ ├── explore/ (4) -│ │ │ ├── dialects/ (5 files, 0 dirs) -│ │ │ ├── index.ts (59b350c, 52L, 1222ch, 1222B) -│ │ │ ├── operations.ts (85ab6af, 468L, 11091ch, 11097B) -│ │ │ └── types.ts (2ceb9ec, 394L, 8217ch, 8219B) +│ │ ├── explore/ (5) +│ │ │ ├── dialects/ (5) +│ │ │ │ ├── index.ts (7ded821, 48L, 1293ch, 1293B) +│ │ │ │ ├── mssql.ts (06521a0, 901L, 28119ch, 28119B) +│ │ │ │ ├── mysql.ts (4d4ffe7, 789L, 23830ch, 23830B) +│ │ │ │ ├── postgres.ts (16828ab, 960L, 29902ch, 29904B) +│ │ │ │ └── sqlite.ts (b94af06, 504L, 13335ch, 13335B) +│ │ │ ├── index.ts (e518ce0, 57L, 1387ch, 1387B) +│ │ │ ├── operations.ts (a75134d, 660L, 17637ch, 17649B) +│ │ │ ├── peek.ts (8ac0882, 196L, 7324ch, 7336B) +│ │ │ └── types.ts (7e18207, 438L, 9815ch, 9817B) │ │ ├── identity/ (10) │ │ │ ├── crypto.ts (d7c3c7e, 372L, 10932ch, 10938B) │ │ │ ├── env.ts (b4c5d5f, 120L, 3024ch, 3032B) @@ -563,7 +1124,7 @@ │ │ ├── logger/ (12) │ │ │ ├── classifier.ts (4fa4422, 179L, 3747ch, 3747B) │ │ │ ├── color.ts (1dd10fa, 192L, 4033ch, 4045B) -│ │ │ ├── formatter.ts (07ecfee, 450L, 12942ch, 12944B) +│ │ │ ├── formatter.ts (d61874e, 451L, 13052ch, 13054B) │ │ │ ├── index.ts (9e238ba, 69L, 1613ch, 1613B) │ │ │ ├── init.ts (38fbacb, 164L, 4588ch, 4590B) │ │ │ ├── logger.ts (258a011, 859L, 21046ch, 21782B) @@ -582,36 +1143,42 @@ │ │ │ ├── legacy-access.ts (1cb43c7, 53L, 2292ch, 2300B) │ │ │ ├── matrix.ts (f323c96, 49L, 2634ch, 2639B) │ │ │ └── types.ts (5b1fc76, 82L, 2738ch, 2744B) -│ │ ├── runner/ (7) +│ │ ├── runner/ (9) │ │ │ ├── checksum.ts (d21f46d, 99L, 2650ch, 2650B) -│ │ │ ├── index.ts (5156823, 55L, 1020ch, 1020B) +│ │ │ ├── index.ts (ff52cef, 58L, 1106ch, 1106B) │ │ │ ├── mssql-batches.ts (3a929cd, 180L, 5670ch, 5684B) -│ │ │ ├── runner.ts (9ad0774, 1574L, 42676ch, 44396B) +│ │ │ ├── runner.ts (ee1701e, 1681L, 45992ch, 47712B) │ │ │ ├── sqlite-statements.ts (74e41da, 252L, 6407ch, 6419B) +│ │ │ ├── statement-probes.ts (7c20808, 335L, 10843ch, 10843B) +│ │ │ ├── statement-watcher.ts (1ff301a, 341L, 10073ch, 10073B) │ │ │ ├── tracker.ts (8a25bc7, 743L, 23769ch, 24003B) -│ │ │ └── types.ts (f467c87, 436L, 11424ch, 13384B) +│ │ │ └── types.ts (da308ee, 443L, 11695ch, 13655B) │ │ ├── settings/ (7) -│ │ │ ├── defaults.ts (3fef652, 117L, 2748ch, 2748B) +│ │ │ ├── defaults.ts (fc071d6, 143L, 3677ch, 3679B) │ │ │ ├── events.ts (f8588b1, 108L, 2257ch, 2257B) -│ │ │ ├── index.ts (f096e30, 78L, 1597ch, 1597B) +│ │ │ ├── index.ts (3b06d77, 82L, 1677ch, 1677B) │ │ │ ├── manager.ts (76e5e37, 1107L, 25307ch, 28003B) │ │ │ ├── rules.ts (52445be, 288L, 6906ch, 6906B) -│ │ │ ├── schema.ts (9df771c, 335L, 9305ch, 11383B) -│ │ │ └── types.ts (fda4f91, 310L, 7363ch, 7367B) -│ │ ├── shared/ (6) +│ │ │ ├── schema.ts (fbb1ab8, 354L, 9982ch, 12304B) +│ │ │ └── types.ts (6cbf075, 341L, 8248ch, 8254B) +│ │ ├── shared/ (8) +│ │ │ ├── abort.ts (999c9a6, 121L, 3387ch, 3389B) │ │ │ ├── dialect-quoting.ts (8f028eb, 66L, 1860ch, 1866B) │ │ │ ├── errors.ts (9cb1445, 221L, 6038ch, 6528B) │ │ │ ├── files.ts (aa00f6e, 170L, 5252ch, 5258B) -│ │ │ ├── index.ts (53c6a75, 65L, 1468ch, 1468B) +│ │ │ ├── index.ts (c6bfa1d, 71L, 1623ch, 1623B) +│ │ │ ├── mask.ts (0e78786, 88L, 3439ch, 3447B) │ │ │ ├── operation-id.ts (7feb6c9, 197L, 6858ch, 6874B) │ │ │ └── tables.ts (06b23dd, 487L, 12754ch, 14716B) │ │ ├── sql-terminal/ (4) -│ │ │ ├── executor.ts (9c75cbc, 164L, 4951ch, 4955B) +│ │ │ ├── executor.ts (1a634a5, 334L, 10634ch, 10642B) │ │ │ ├── history.ts (edcbcea, 428L, 11003ch, 11007B) -│ │ │ ├── index.ts (23c3d3d, 17L, 557ch, 557B) -│ │ │ └── types.ts (cb0a264, 123L, 2519ch, 2519B) +│ │ │ ├── index.ts (7d8900f, 17L, 614ch, 614B) +│ │ │ └── types.ts (140337f, 135L, 3122ch, 3122B) │ │ ├── state/ (9) -│ │ │ ├── encryption/ (2 files, 0 dirs) +│ │ │ ├── encryption/ (2) +│ │ │ │ ├── crypto.ts (999bb51, 142L, 4273ch, 4283B) +│ │ │ │ └── index.ts (5cc17b1, 4L, 86ch, 86B) │ │ │ ├── access.ts (fd7a5b4, 76L, 2616ch, 2624B) │ │ │ ├── index.ts (4d767a8, 72L, 1531ch, 1531B) │ │ │ ├── manager.ts (f37ca3c, 940L, 25032ch, 26500B) @@ -621,12 +1188,24 @@ │ │ │ ├── types.ts (1e7c0fc, 103L, 3015ch, 3017B) │ │ │ └── version.ts (11a6a0b, 26L, 649ch, 649B) │ │ ├── teardown/ (4) -│ │ │ ├── dialects/ (5 files, 0 dirs) +│ │ │ ├── dialects/ (5) +│ │ │ │ ├── index.ts (43ced94, 56L, 1523ch, 1523B) +│ │ │ │ ├── mssql.ts (b3e06b2, 139L, 4942ch, 4944B) +│ │ │ │ ├── mysql.ts (d1db886, 79L, 2001ch, 2003B) +│ │ │ │ ├── postgres.ts (b52aa51, 81L, 2276ch, 2278B) +│ │ │ │ └── sqlite.ts (6d02570, 89L, 2401ch, 2403B) │ │ │ ├── index.ts (74cbb84, 27L, 549ch, 549B) │ │ │ ├── operations.ts (36a9316, 688L, 20730ch, 20774B) │ │ │ └── types.ts (375b750, 327L, 8970ch, 8976B) │ │ ├── template/ (7) -│ │ │ ├── loaders/ (7 files, 0 dirs) +│ │ │ ├── loaders/ (7) +│ │ │ │ ├── csv.ts (08bb46a, 34L, 824ch, 826B) +│ │ │ │ ├── dt.ts (61c5c22, 70L, 1649ch, 1651B) +│ │ │ │ ├── index.ts (eb3366c, 195L, 5080ch, 5574B) +│ │ │ │ ├── js.ts (654303e, 94L, 2798ch, 2798B) +│ │ │ │ ├── json5.ts (25eb9c3, 33L, 737ch, 737B) +│ │ │ │ ├── sql.ts (5d8fa34, 25L, 541ch, 541B) +│ │ │ │ └── yaml.ts (de2612e, 28L, 578ch, 578B) │ │ │ ├── context.ts (991d1f2, 390L, 11848ch, 11864B) │ │ │ ├── engine.ts (5eb6f57, 214L, 6145ch, 6155B) │ │ │ ├── helpers.ts (dc37e61, 185L, 4817ch, 4845B) @@ -634,7 +1213,12 @@ │ │ │ ├── types.ts (eeb0f2a, 214L, 4307ch, 4307B) │ │ │ └── utils.ts (66c4ca1, 187L, 4773ch, 4817B) │ │ ├── transfer/ (7) -│ │ │ ├── dialects/ (5 files, 0 dirs) +│ │ │ ├── dialects/ (5) +│ │ │ │ ├── index.ts (749b6d3, 63L, 1755ch, 1755B) +│ │ │ │ ├── mssql.ts (ef5ed14, 203L, 5962ch, 5962B) +│ │ │ │ ├── mysql.ts (b939c8a, 140L, 3831ch, 3831B) +│ │ │ │ ├── postgres.ts (4b520d9, 167L, 4679ch, 4681B) +│ │ │ │ └── types.ts (52678ae, 116L, 3399ch, 3399B) │ │ │ ├── events.ts (a6a7783, 69L, 1687ch, 1687B) │ │ │ ├── executor.ts (dc3eec9, 1091L, 27190ch, 27198B) │ │ │ ├── index.ts (55c7baa, 246L, 6720ch, 6722B) @@ -661,9 +1245,15 @@ │ │ │ ├── storage.ts (e2932f7, 757L, 20949ch, 21205B) │ │ │ └── types.ts (4060169, 151L, 3679ch, 3683B) │ │ ├── version/ (5) -│ │ │ ├── schema/ (1 file, 1 dir) -│ │ │ ├── settings/ (1 file, 1 dir) -│ │ │ ├── state/ (1 file, 1 dir) +│ │ │ ├── schema/ (2) +│ │ │ │ ├── migrations/ (2 files, 0 dirs) +│ │ │ │ └── index.ts (e7e4f5f, 586L, 15620ch, 15622B) +│ │ │ ├── settings/ (2) +│ │ │ │ ├── migrations/ (1 file, 0 dirs) +│ │ │ │ └── index.ts (a0b9333, 193L, 4744ch, 4744B) +│ │ │ ├── state/ (2) +│ │ │ │ ├── migrations/ (3 files, 0 dirs) +│ │ │ │ └── index.ts (5cd2133, 195L, 4663ch, 4663B) │ │ │ ├── index.ts (8abf6c2, 302L, 7550ch, 8282B) │ │ │ └── types.ts (a54aa04, 253L, 6917ch, 8381B) │ │ ├── worker-bridge/ (7) @@ -675,8 +1265,8 @@ │ │ │ ├── pool.ts (aea709d, 86L, 2314ch, 2314B) │ │ │ └── types.ts (c142541, 46L, 1770ch, 1776B) │ │ ├── environment.ts (08ebb56, 141L, 3015ch, 3017B) -│ │ ├── index.ts (f600b25, 418L, 8876ch, 8876B) -│ │ ├── observer.ts (4f0e741, 264L, 10076ch, 10078B) +│ │ ├── index.ts (aeb0945, 423L, 9045ch, 9045B) +│ │ ├── observer.ts (ecc7e41, 283L, 10724ch, 10726B) │ │ ├── project-init.ts (3a2c70e, 174L, 5041ch, 5043B) │ │ ├── project.ts (18af87e, 255L, 6533ch, 6533B) │ │ └── theme.ts (f8e8f4a, 545L, 12937ch, 14702B) @@ -725,79 +1315,219 @@ │ │ ├── state.ts (2e875de, 54L, 1779ch, 2267B) │ │ ├── tvp.ts (05b7f0a, 168L, 4695ch, 5433B) │ │ └── types.ts (5fec5ed, 183L, 4980ch, 5958B) -│ ├── tui/ (14) +│ ├── tui/ (16) │ │ ├── components/ (10) -│ │ │ ├── dialogs/ (6 files, 0 dirs) -│ │ │ ├── feedback/ (4 files, 0 dirs) -│ │ │ ├── forms/ (2 files, 0 dirs) -│ │ │ ├── layout/ (5 files, 0 dirs) -│ │ │ ├── lists/ (5 files, 0 dirs) -│ │ │ ├── overlays/ (2 files, 0 dirs) -│ │ │ ├── secrets/ (6 files, 0 dirs) -│ │ │ ├── status/ (3 files, 0 dirs) -│ │ │ ├── terminal/ (3 files, 0 dirs) -│ │ │ └── index.ts (c5ec099, 95L, 2533ch, 2533B) -│ │ ├── hooks/ (14) -│ │ │ ├── index.ts (4220ca8, 69L, 1387ch, 1387B) +│ │ │ ├── dialogs/ (6) +│ │ │ │ ├── Confirm.tsx (e46197c, 176L, 3944ch, 3948B) +│ │ │ │ ├── FilePicker.tsx (a6eb320, 427L, 10532ch, 10544B) +│ │ │ │ ├── KeyHandler.tsx (d8494fa, 88L, 1703ch, 1703B) +│ │ │ │ ├── ProtectedConfirm.tsx (435c708, 179L, 4533ch, 4535B) +│ │ │ │ ├── SmartConfirm.tsx (9f52ff6, 102L, 3099ch, 3101B) +│ │ │ │ └── index.ts (68fdcb0, 14L, 577ch, 577B) +│ │ │ ├── feedback/ (4) +│ │ │ │ ├── DismissableAlert.tsx (b537ede, 285L, 7054ch, 7054B) +│ │ │ │ ├── Toast.tsx (00239f9, 182L, 3678ch, 3686B) +│ │ │ │ ├── index.ts (f607506, 20L, 653ch, 653B) +│ │ │ │ └── useDismissableAlert.ts (fbc7d79, 154L, 3928ch, 3928B) +│ │ │ ├── forms/ (3) +│ │ │ │ ├── Form.tsx (4d966ae, 911L, 24467ch, 24533B) +│ │ │ │ ├── TextInput.tsx (dc2871a, 295L, 8782ch, 8788B) +│ │ │ │ └── index.ts (8c36061, 20L, 524ch, 524B) +│ │ │ ├── layout/ (5) +│ │ │ │ ├── Divider.tsx (70b643b, 76L, 1716ch, 1718B) +│ │ │ │ ├── MissingParamPanel.tsx (1d113c3, 54L, 1365ch, 1365B) +│ │ │ │ ├── NotFoundPanel.tsx (8d10d01, 52L, 1204ch, 1204B) +│ │ │ │ ├── Panel.tsx (e5667ab, 89L, 1791ch, 1791B) +│ │ │ │ └── index.ts (cf6871a, 12L, 447ch, 447B) +│ │ │ ├── lists/ (5) +│ │ │ │ ├── ActionList.tsx (8dc107a, 95L, 2098ch, 2098B) +│ │ │ │ ├── SearchableList.tsx (1c0ccf2, 359L, 10312ch, 10314B) +│ │ │ │ ├── SelectList.tsx (77c5acc, 577L, 18715ch, 18733B) +│ │ │ │ ├── StatusList.tsx (ff6d6de, 98L, 2642ch, 2654B) +│ │ │ │ └── index.ts (e80cd6c, 12L, 556ch, 556B) +│ │ │ ├── overlays/ (3) +│ │ │ │ ├── LogViewerOverlay.tsx (709d56c, 446L, 11706ch, 11706B) +│ │ │ │ ├── TextOverlay.tsx (f98bc93, 85L, 3023ch, 3031B) +│ │ │ │ └── index.ts (a71d4b9, 10L, 325ch, 325B) +│ │ │ ├── secrets/ (6) +│ │ │ │ ├── SecretDefinitionForm.tsx (24a4695, 163L, 4569ch, 4569B) +│ │ │ │ ├── SecretDefinitionList.tsx (db1de23, 343L, 9210ch, 9214B) +│ │ │ │ ├── SecretValueForm.tsx (5e8a032, 226L, 6082ch, 6082B) +│ │ │ │ ├── SecretValueList.tsx (a629e7a, 286L, 7334ch, 7338B) +│ │ │ │ ├── index.ts (9335267, 30L, 1123ch, 1123B) +│ │ │ │ └── types.ts (206426c, 104L, 2379ch, 2379B) +│ │ │ ├── status/ (4) +│ │ │ │ ├── ConnectionStatus.tsx (7204c77, 137L, 3599ch, 3599B) +│ │ │ │ ├── LockStatus.tsx (79340c7, 134L, 3086ch, 3086B) +│ │ │ │ ├── StatementProgress.tsx (a57a322, 101L, 3336ch, 3338B) +│ │ │ │ └── index.ts (4707bcb, 10L, 452ch, 452B) +│ │ │ ├── terminal/ (9) +│ │ │ │ ├── ResultBrowser.tsx (71fe003, 157L, 5349ch, 5359B) +│ │ │ │ ├── ResultTable.tsx (dd05769, 1008L, 28800ch, 28843B) +│ │ │ │ ├── RowViewOverlay.tsx (9e5b198, 202L, 6640ch, 6672B) +│ │ │ │ ├── ScrollPane.tsx (6466009, 93L, 3442ch, 3450B) +│ │ │ │ ├── SqlInput.tsx (b60d76c, 289L, 7229ch, 7229B) +│ │ │ │ ├── columnFit.ts (3dd3ca7, 185L, 6307ch, 6323B) +│ │ │ │ ├── index.ts (80bd931, 38L, 1204ch, 1204B) +│ │ │ │ ├── rowDocument.ts (2cca900, 270L, 9404ch, 9422B) +│ │ │ │ └── viewport.ts (bfd2be1, 199L, 6213ch, 6227B) +│ │ │ └── index.ts (bb7cbf7, 97L, 2635ch, 2635B) +│ │ ├── hooks/ (18) +│ │ │ ├── index.ts (0648b0b, 87L, 1811ch, 1811B) +│ │ │ ├── useAbortableTask.ts (f290efa, 112L, 2908ch, 2908B) │ │ │ ├── useAsyncEffect.ts (9e1d1d5, 43L, 903ch, 903B) -│ │ │ ├── useChangeProgress.ts (12b8c8d, 111L, 3230ch, 3230B) +│ │ │ ├── useChangeProgress.ts (d2c0fc3, 125L, 3773ch, 3773B) │ │ │ ├── useConnection.ts (516f590, 229L, 6526ch, 6526B) +│ │ │ ├── useDoublePress.ts (6b96d6a, 40L, 1066ch, 1066B) │ │ │ ├── useLoadGuard.ts (e4d449c, 51L, 1054ch, 1054B) │ │ │ ├── useLockStatus.ts (91d7117, 145L, 3664ch, 3664B) │ │ │ ├── useObserver.ts (e5db37d, 158L, 4195ch, 4197B) -│ │ │ ├── useRunProgress.ts (8174572, 279L, 6494ch, 6494B) +│ │ │ ├── useRunProgress.ts (5a58a85, 301L, 7110ch, 7110B) │ │ │ ├── useSecretSource.ts (2004e16, 74L, 1995ch, 1995B) │ │ │ ├── useSettingsOperation.ts (cdd5226, 104L, 2795ch, 2811B) -│ │ │ ├── useTransferProgress.ts (268d0bf, 583L, 14501ch, 14501B) +│ │ │ ├── useTransferProgress.ts (b20b67e, 583L, 14509ch, 14509B) │ │ │ ├── useUpdateChecker.ts (eac4e68, 167L, 4040ch, 4040B) +│ │ │ ├── useUpdateProgress.ts (522dc03, 162L, 3284ch, 3284B) │ │ │ ├── useVaultConnection.ts (993630d, 141L, 3774ch, 3784B) -│ │ │ └── useVaultSecretKeys.ts (4677fc7, 165L, 4807ch, 4821B) +│ │ │ ├── useVaultSecretKeys.ts (4677fc7, 165L, 4807ch, 4821B) +│ │ │ └── useViewportRows.ts (8b46f8a, 121L, 4428ch, 4448B) │ │ ├── providers/ (1) │ │ │ └── ConnectionProvider.tsx (b4340b1, 231L, 5790ch, 5794B) │ │ ├── screens/ (15) -│ │ │ ├── change/ (12 files, 0 dirs) -│ │ │ ├── config/ (11 files, 0 dirs) -│ │ │ ├── db/ (11 files, 1 dir) -│ │ │ ├── debug/ (4 files, 0 dirs) -│ │ │ ├── identity/ (6 files, 0 dirs) -│ │ │ ├── init/ (4 files, 0 dirs) -│ │ │ ├── lock/ (6 files, 0 dirs) -│ │ │ ├── run/ (7 files, 0 dirs) -│ │ │ ├── secret/ (4 files, 0 dirs) -│ │ │ ├── settings/ (17 files, 0 dirs) -│ │ │ ├── vault/ (5 files, 0 dirs) +│ │ │ ├── change/ (12) +│ │ │ │ ├── ChangeAddScreen.tsx (c19a374, 333L, 8506ch, 8506B) +│ │ │ │ ├── ChangeEditScreen.tsx (b2e1c65, 196L, 5033ch, 5033B) +│ │ │ │ ├── ChangeFFScreen.tsx (5d7fc89, 366L, 10448ch, 10450B) +│ │ │ │ ├── ChangeHistoryDetailScreen.tsx (1cb3565, 344L, 11390ch, 11402B) +│ │ │ │ ├── ChangeHistoryScreen.tsx (f147bb1, 339L, 11843ch, 11857B) +│ │ │ │ ├── ChangeListScreen.tsx (0f03069, 402L, 11851ch, 11859B) +│ │ │ │ ├── ChangeNextScreen.tsx (32df0ba, 421L, 11755ch, 11757B) +│ │ │ │ ├── ChangeRemoveScreen.tsx (f94fbbe, 310L, 8800ch, 8800B) +│ │ │ │ ├── ChangeRevertScreen.tsx (576e522, 356L, 10197ch, 10201B) +│ │ │ │ ├── ChangeRewindScreen.tsx (0f4f2da, 489L, 13838ch, 13842B) +│ │ │ │ ├── ChangeRunScreen.tsx (f126156, 394L, 12018ch, 12024B) +│ │ │ │ └── index.ts (c38eaad, 17L, 746ch, 746B) +│ │ │ ├── config/ (11) +│ │ │ │ ├── ConfigAddScreen.tsx (fe4e52b, 305L, 9169ch, 9171B) +│ │ │ │ ├── ConfigCopyScreen.tsx (6d7006d, 206L, 4990ch, 4990B) +│ │ │ │ ├── ConfigEditScreen.tsx (8636a0b, 320L, 10293ch, 10293B) +│ │ │ │ ├── ConfigExportScreen.tsx (f125704, 369L, 10631ch, 10631B) +│ │ │ │ ├── ConfigImportScreen.tsx (189fbe2, 487L, 14415ch, 14415B) +│ │ │ │ ├── ConfigListScreen.tsx (904e4cd, 234L, 6249ch, 6253B) +│ │ │ │ ├── ConfigMoreScreen.tsx (945c645, 143L, 3605ch, 3605B) +│ │ │ │ ├── ConfigRemoveScreen.tsx (68ab313, 253L, 7450ch, 7450B) +│ │ │ │ ├── ConfigUseScreen.tsx (6a36514, 196L, 4984ch, 4984B) +│ │ │ │ ├── ConfigValidateScreen.tsx (5f38ae1, 174L, 4539ch, 4539B) +│ │ │ │ └── index.ts (ae41ce9, 13L, 630ch, 630B) +│ │ │ ├── db/ (12) +│ │ │ │ ├── explore/ (7 files, 0 dirs) +│ │ │ │ ├── DbCreateScreen.tsx (9a75540, 312L, 8053ch, 8053B) +│ │ │ │ ├── DbDestroyScreen.tsx (983db00, 345L, 9235ch, 9237B) +│ │ │ │ ├── DbListScreen.tsx (d83126e, 364L, 10685ch, 10685B) +│ │ │ │ ├── DbTeardownScreen.tsx (03304c4, 621L, 20691ch, 20699B) +│ │ │ │ ├── DbTransferScreen.tsx (490ea39, 1813L, 58357ch, 58385B) +│ │ │ │ ├── DbTruncateScreen.tsx (1cc7aad, 376L, 11582ch, 11582B) +│ │ │ │ ├── DtModifyScreen.tsx (c979dc4, 1699L, 56436ch, 56480B) +│ │ │ │ ├── SqlClearScreen.tsx (e3f5e2f, 329L, 8946ch, 8950B) +│ │ │ │ ├── SqlHistoryScreen.tsx (92cbe98, 438L, 12919ch, 12931B) +│ │ │ │ ├── SqlTerminalScreen.tsx (ab82390, 597L, 18640ch, 18649B) +│ │ │ │ └── index.ts (437d6a5, 37L, 1396ch, 1396B) +│ │ │ ├── debug/ (4) +│ │ │ │ ├── DebugDetailScreen.tsx (7bc6edd, 365L, 9805ch, 10049B) +│ │ │ │ ├── DebugListScreen.tsx (de2cf08, 537L, 15312ch, 15556B) +│ │ │ │ ├── DebugOverviewScreen.tsx (74feb04, 264L, 8409ch, 8411B) +│ │ │ │ └── index.ts (f291976, 9L, 293ch, 293B) +│ │ │ ├── identity/ (6) +│ │ │ │ ├── IdentityEditScreen.tsx (80eced9, 214L, 6466ch, 6466B) +│ │ │ │ ├── IdentityExportScreen.tsx (bb616b6, 157L, 4457ch, 4457B) +│ │ │ │ ├── IdentityInitScreen.tsx (0bffbde, 224L, 6714ch, 6720B) +│ │ │ │ ├── IdentityScreen.tsx (6048d54, 222L, 6402ch, 6402B) +│ │ │ │ ├── KnownUsersScreen.tsx (6e3d956, 267L, 9787ch, 9797B) +│ │ │ │ └── index.ts (138be43, 8L, 345ch, 345B) +│ │ │ ├── init/ (4) +│ │ │ │ ├── IdentitySetup.tsx (e3a2323, 198L, 6399ch, 6409B) +│ │ │ │ ├── InitScreen.tsx (59c5fd3, 545L, 17329ch, 17345B) +│ │ │ │ ├── ProjectSetup.tsx (759ed42, 157L, 4834ch, 4852B) +│ │ │ │ └── index.ts (14e323d, 12L, 309ch, 309B) +│ │ │ ├── lock/ (6) +│ │ │ │ ├── LockAcquireScreen.tsx (d971c1b, 348L, 10116ch, 10116B) +│ │ │ │ ├── LockForceScreen.tsx (0a73309, 397L, 11602ch, 11604B) +│ │ │ │ ├── LockListScreen.tsx (b324755, 270L, 7598ch, 7598B) +│ │ │ │ ├── LockReleaseScreen.tsx (c366668, 335L, 8603ch, 8603B) +│ │ │ │ ├── LockStatusScreen.tsx (34ab652, 347L, 9918ch, 9918B) +│ │ │ │ └── index.ts (144f483, 27L, 835ch, 835B) +│ │ │ ├── run/ (7) +│ │ │ │ ├── RunBuildScreen.tsx (9b42729, 571L, 20572ch, 20574B) +│ │ │ │ ├── RunDirScreen.tsx (fcb4c49, 798L, 27739ch, 27741B) +│ │ │ │ ├── RunExecScreen.tsx (366caa5, 432L, 13581ch, 13581B) +│ │ │ │ ├── RunFileScreen.tsx (6682193, 618L, 19931ch, 19931B) +│ │ │ │ ├── RunInspectScreen.tsx (9d12203, 995L, 33278ch, 33327B) +│ │ │ │ ├── RunListScreen.tsx (fba2854, 264L, 8125ch, 8125B) +│ │ │ │ └── index.ts (019e1cf, 30L, 955ch, 955B) +│ │ │ ├── secret/ (4) +│ │ │ │ ├── SecretListScreen.tsx (5e39116, 259L, 7803ch, 7803B) +│ │ │ │ ├── SecretRemoveScreen.tsx (a6b8df4, 277L, 8425ch, 8427B) +│ │ │ │ ├── SecretSetScreen.tsx (b707801, 236L, 6856ch, 6856B) +│ │ │ │ └── index.ts (aa62b82, 6L, 217ch, 217B) +│ │ │ ├── settings/ (17) +│ │ │ │ ├── SettingsBuildScreen.tsx (15d1862, 154L, 4304ch, 4304B) +│ │ │ │ ├── SettingsInitScreen.tsx (04f1f67, 282L, 7506ch, 7506B) +│ │ │ │ ├── SettingsListScreen.tsx (94cad7e, 272L, 7310ch, 7312B) +│ │ │ │ ├── SettingsLoggingScreen.tsx (402114b, 166L, 4961ch, 4961B) +│ │ │ │ ├── SettingsPathsScreen.tsx (689ca17, 111L, 3138ch, 3138B) +│ │ │ │ ├── SettingsRuleEditScreen.tsx (7f121e4, 242L, 7312ch, 7312B) +│ │ │ │ ├── SettingsRulesListScreen.tsx (5759a57, 328L, 8621ch, 8623B) +│ │ │ │ ├── SettingsSecretEditScreen.tsx (5d7f46d, 166L, 4733ch, 4733B) +│ │ │ │ ├── SettingsSecretRemoveScreen.tsx (1748d5c, 219L, 6239ch, 6239B) +│ │ │ │ ├── SettingsSecretsListScreen.tsx (5430282, 129L, 3642ch, 3644B) +│ │ │ │ ├── SettingsStageEditScreen.tsx (1483653, 252L, 7886ch, 7886B) +│ │ │ │ ├── SettingsStageSecretEditScreen.tsx (0334925, 7L, 328ch, 328B) +│ │ │ │ ├── SettingsStageSecretRemoveScreen.tsx (3a34ba6, 7L, 340ch, 340B) +│ │ │ │ ├── SettingsStageSecretsListScreen.tsx (ecc796d, 184L, 5066ch, 5068B) +│ │ │ │ ├── SettingsStagesListScreen.tsx (d39e550, 290L, 7463ch, 7468B) +│ │ │ │ ├── SettingsStrictScreen.tsx (271c2a0, 139L, 3815ch, 3815B) +│ │ │ │ └── index.ts (b2d3a78, 23L, 1252ch, 1252B) +│ │ │ ├── vault/ (5) +│ │ │ │ ├── VaultInitScreen.tsx (24a66fe, 198L, 5464ch, 5464B) +│ │ │ │ ├── VaultRemoveScreen.tsx (1215a1d, 264L, 7167ch, 7167B) +│ │ │ │ ├── VaultScreen.tsx (9458309, 528L, 16179ch, 16185B) +│ │ │ │ ├── VaultSetScreen.tsx (a8a0952, 299L, 7922ch, 7922B) +│ │ │ │ └── index.ts (f0d3889, 7L, 258ch, 258B) │ │ │ ├── MoreScreen.tsx (8f8530e, 80L, 2503ch, 2503B) -│ │ │ ├── UpdateScreen.tsx (d71a562, 191L, 4892ch, 4892B) +│ │ │ ├── UpdateScreen.tsx (1cd89c8, 217L, 6105ch, 6107B) │ │ │ ├── home.tsx (7f5fdf6, 635L, 21702ch, 21712B) │ │ │ └── not-found.tsx (135b44e, 72L, 1958ch, 1958B) -│ │ ├── utils/ (13) +│ │ ├── utils/ (14) │ │ │ ├── change-context.ts (962b3e2, 78L, 2735ch, 2735B) │ │ │ ├── change-loader.ts (4ad58de, 252L, 6631ch, 6633B) │ │ │ ├── clipboard.ts (93a7f8c, 95L, 1991ch, 1991B) │ │ │ ├── config-validation.ts (425dfc6, 218L, 5936ch, 5944B) -│ │ │ ├── connection.ts (e20a2e0, 82L, 2065ch, 2065B) +│ │ │ ├── connection.ts (6391d55, 92L, 2475ch, 2477B) │ │ │ ├── date.ts (ff07926, 20L, 391ch, 391B) │ │ │ ├── error.ts (97c198b, 33L, 736ch, 736B) │ │ │ ├── identity.ts (a44606a, 32L, 870ch, 870B) -│ │ │ ├── index.ts (7054044, 30L, 1040ch, 1040B) +│ │ │ ├── index.ts (99b6143, 31L, 1144ch, 1144B) │ │ │ ├── paths.ts (ee0cf63, 53L, 1329ch, 1329B) -│ │ │ ├── run-context.ts (ea7c320, 97L, 3095ch, 3097B) +│ │ │ ├── progress.ts (3025949, 16L, 474ch, 476B) +│ │ │ ├── run-context.ts (18ec692, 116L, 3785ch, 3787B) │ │ │ ├── settings-validation.ts (b155b66, 49L, 1148ch, 1152B) -│ │ │ └── string.ts (8c22fe4, 39L, 1182ch, 1182B) -│ │ ├── app-context.tsx (dc19f0e, 1198L, 29681ch, 30915B) -│ │ ├── app.tsx (10f41d3, 418L, 12294ch, 12352B) -│ │ ├── focus.tsx (342fe30, 250L, 5134ch, 5134B) -│ │ ├── keyboard.tsx (5d1144b, 401L, 9159ch, 9161B) +│ │ │ └── string.ts (6714859, 57L, 1770ch, 1774B) +│ │ ├── app-context.tsx (88bd623, 1188L, 29352ch, 30586B) +│ │ ├── app.tsx (dd9bac9, 480L, 15247ch, 15311B) +│ │ ├── focus.tsx (6fee0c2, 294L, 6236ch, 6236B) +│ │ ├── keyboard.tsx (d50d8aa, 402L, 9116ch, 9118B) +│ │ ├── list-memory.ts (7fd5400, 118L, 3694ch, 3694B) +│ │ ├── mouse.tsx (01ec948, 680L, 19469ch, 19483B) │ │ ├── observer-context.ts (305604b, 24L, 806ch, 806B) -│ │ ├── router.tsx (6e92c8a, 255L, 5954ch, 5954B) +│ │ ├── router.tsx (8e4dda6, 285L, 7015ch, 7015B) │ │ ├── screens.tsx (0e4497b, 664L, 15342ch, 15342B) │ │ ├── shutdown.tsx (8aa78da, 306L, 7951ch, 7955B) -│ │ └── types.ts (1767af6, 470L, 9688ch, 9690B) +│ │ └── types.ts (05bbbc5, 497L, 10586ch, 10588B) │ └── workers/ (2) │ ├── compute.ts (c7b828a, 46L, 1233ch, 1233B) │ └── connection.ts (128339f, 241L, 5955ch, 5957B) ├── tests/ (11) -│ ├── cli/ (41) +│ ├── cli/ (43) │ │ ├── change/ (4) │ │ │ ├── history.test.ts (e6ab24d, 140L, 4580ch, 4586B) │ │ │ ├── list.test.ts (07f0afa, 158L, 5380ch, 5386B) @@ -809,14 +1539,20 @@ │ │ │ ├── identity-new.test.ts (2462d0f, 88L, 2587ch, 2587B) │ │ │ ├── init.test.ts (9e09d8d, 389L, 10906ch, 10910B) │ │ │ └── secrets.test.ts (8f20782, 216L, 6180ch, 6180B) -│ │ ├── components/ (7) +│ │ ├── components/ (13) │ │ │ ├── DismissableAlert.test.tsx (ad0c85f, 340L, 9391ch, 9391B) │ │ │ ├── dialogs.test.tsx (67db743, 343L, 9979ch, 9983B) -│ │ │ ├── form-navigation.test.tsx (7f6d5a4, 175L, 4812ch, 4816B) -│ │ │ ├── forms.test.tsx (d67738d, 171L, 4982ch, 4990B) +│ │ │ ├── form-navigation.test.tsx (e369dcc, 655L, 21980ch, 22048B) +│ │ │ ├── forms.test.tsx (8f82d01, 532L, 16899ch, 16949B) │ │ │ ├── layout.test.tsx (811eaec, 110L, 2675ch, 2683B) +│ │ │ ├── list-position.test.tsx (16ebf02, 512L, 16109ch, 16145B) │ │ │ ├── lists.test.tsx (3807cc8, 220L, 6656ch, 6670B) -│ │ │ └── status.test.tsx (345fc65, 191L, 5245ch, 5245B) +│ │ │ ├── scroll-pane.test.tsx (cbf6307, 261L, 7905ch, 7909B) +│ │ │ ├── select-list-window.test.tsx (1bd5998, 268L, 8766ch, 8772B) +│ │ │ ├── status.test.tsx (345fc65, 191L, 5245ch, 5245B) +│ │ │ ├── terminal.test.tsx (3c0e770, 683L, 22741ch, 22759B) +│ │ │ ├── text-input.test.tsx (1d4bf52, 317L, 10112ch, 10118B) +│ │ │ └── text-overlay.test.tsx (3deee3e, 162L, 4781ch, 4785B) │ │ ├── config/ (6) │ │ │ ├── add.test.ts (3008ad8, 62L, 1805ch, 1815B) │ │ │ ├── edit.test.ts (3d84874, 79L, 2337ch, 2351B) @@ -832,11 +1568,17 @@ │ │ │ ├── reset.test.ts (08260c7, 261L, 8966ch, 8978B) │ │ │ ├── teardown.test.ts (65dcedd, 128L, 3931ch, 3937B) │ │ │ └── transfer.test.ts (e01475c, 209L, 7012ch, 7032B) -│ │ ├── hooks/ (4) +│ │ ├── fixtures/ (2) +│ │ │ ├── mouse-sigint-restore.ts (cba37b2, 32L, 1036ch, 1036B) +│ │ │ └── text-input-render-parity.tsx (ef2c720, 76L, 2737ch, 2737B) +│ │ ├── hooks/ (7) +│ │ │ ├── useAbortableTask.test.tsx (07cc476, 167L, 4047ch, 4047B) │ │ │ ├── useObserver.test.tsx (e827049, 392L, 10127ch, 10129B) │ │ │ ├── useTransferProgress.test.tsx (5267f2e, 322L, 10736ch, 10744B) │ │ │ ├── useUpdateChecker.test.tsx (3073b2c, 283L, 8301ch, 8301B) -│ │ │ └── useVaultSecretKeys.test.tsx (0bb45af, 294L, 8826ch, 8828B) +│ │ │ ├── useUpdateProgress.test.tsx (002ca62, 131L, 4287ch, 4287B) +│ │ │ ├── useVaultSecretKeys.test.tsx (0bb45af, 294L, 8826ch, 8828B) +│ │ │ └── useViewportRows.test.tsx (68abecd, 212L, 6156ch, 6158B) │ │ ├── identity/ (2) │ │ │ ├── edit.test.ts (80a5af4, 171L, 5253ch, 5255B) │ │ │ └── init.test.ts (999f271, 182L, 5058ch, 5064B) @@ -857,19 +1599,44 @@ │ │ │ ├── preview-inspect-policy.test.ts (e14cc87, 123L, 4173ch, 4181B) │ │ │ ├── preview-inspect-vault-probe.test.ts (4be2724, 154L, 5466ch, 5470B) │ │ │ └── sql.test.ts (29d578f, 123L, 3370ch, 3372B) -│ │ ├── screens/ (4) -│ │ │ ├── change/ (1 file, 0 dirs) -│ │ │ ├── config/ (2 files, 0 dirs) -│ │ │ ├── db/ (2 files, 0 dirs) -│ │ │ └── init/ (4 files, 0 dirs) +│ │ ├── screens/ (7) +│ │ │ ├── change/ (2) +│ │ │ │ ├── change-add-mouse.test.tsx (4ab6bd4, 252L, 8298ch, 8308B) +│ │ │ │ └── change-dry-run.test.tsx (c36431c, 293L, 10103ch, 10105B) +│ │ │ ├── config/ (2) +│ │ │ │ ├── ConfigEditScreen.test.tsx (38085f2, 380L, 13339ch, 13347B) +│ │ │ │ └── ConfigRemoveScreen.test.tsx (72c0c1a, 166L, 5412ch, 5412B) +│ │ │ ├── db/ (10) +│ │ │ │ ├── DbTransferScreen.test.tsx (2743f58, 433L, 13516ch, 13520B) +│ │ │ │ ├── db-dry-run.test.tsx (d0b0a2b, 290L, 9157ch, 9157B) +│ │ │ │ ├── explore-layout.test.tsx (c585f20, 384L, 13614ch, 13618B) +│ │ │ │ ├── explore-list.test.tsx (ac9289e, 161L, 5006ch, 5010B) +│ │ │ │ ├── explore-peek.test.tsx (ad75cff, 608L, 18223ch, 18232B) +│ │ │ │ ├── explore-row-view.test.tsx (6e85b43, 1152L, 35158ch, 35180B) +│ │ │ │ ├── explore-scroll.test.tsx (03c0146, 821L, 27276ch, 27362B) +│ │ │ │ ├── explore-value.test.tsx (979e954, 428L, 13765ch, 13771B) +│ │ │ │ ├── sql-cancel.test.tsx (052073e, 290L, 9346ch, 9350B) +│ │ │ │ └── sql-row-view.test.tsx (075c380, 370L, 12256ch, 12274B) +│ │ │ ├── init/ (4) +│ │ │ │ ├── identity-setup.test.tsx (4db4baf, 188L, 4585ch, 4585B) +│ │ │ │ ├── init-flow.test.tsx (e1c7731, 389L, 10596ch, 10596B) +│ │ │ │ ├── init-screen.test.tsx (d0a5faa, 246L, 6828ch, 6828B) +│ │ │ │ └── project-setup.test.tsx (f77732f, 260L, 7367ch, 7373B) +│ │ │ ├── lock/ (1) +│ │ │ │ └── lock-force-cancel.test.tsx (9a3f9ba, 226L, 7542ch, 7544B) +│ │ │ ├── run/ (1) +│ │ │ │ └── inspect-lines.test.tsx (7a214a4, 238L, 7920ch, 7928B) +│ │ │ └── settings/ (1) +│ │ │ └── rules-list.test.tsx (2c946de, 61L, 2366ch, 2366B) │ │ ├── sql/ (1) │ │ │ └── history-config.test.ts (9960369, 213L, 7053ch, 7065B) -│ │ ├── utils/ (1) -│ │ │ └── change-context.test.ts (9740ac3, 234L, 7678ch, 7678B) +│ │ ├── utils/ (2) +│ │ │ ├── change-context.test.ts (9740ac3, 234L, 7678ch, 7678B) +│ │ │ └── progress.test.ts (d066ec5, 21L, 580ch, 580B) │ │ ├── VaultScreen.test.tsx (06317d5, 280L, 8956ch, 8958B) │ │ ├── agent-channel-escalation.test.ts (c445315, 286L, 8469ch, 8477B) -│ │ ├── app-context.test.tsx (e15ad92, 628L, 16709ch, 16711B) -│ │ ├── app.test.tsx (3a6f3d3, 241L, 6170ch, 6181B) +│ │ ├── app-context.test.tsx (9ac4dfd, 639L, 17664ch, 17670B) +│ │ ├── app.test.tsx (de90d01, 443L, 13008ch, 13025B) │ │ ├── change-edit.test.ts (295ec3a, 103L, 2985ch, 2985B) │ │ ├── change-prompts.test.ts (9be33db, 107L, 2822ch, 2822B) │ │ ├── citty-args.ts (f0b91d5, 27L, 816ch, 816B) @@ -879,15 +1646,16 @@ │ │ ├── debug-pid.test.tsx (d07de2e, 11L, 188ch, 188B) │ │ ├── env-bootstrap.test.ts (048bf9c, 75L, 2314ch, 2314B) │ │ ├── exit-codes.test.ts (19c705c, 250L, 8496ch, 8504B) -│ │ ├── focus.test.tsx (77608ce, 662L, 15713ch, 15713B) +│ │ ├── focus.test.tsx (3e48447, 664L, 15855ch, 15855B) │ │ ├── global-flags.test.ts (4310f6f, 221L, 6996ch, 7000B) │ │ ├── init.test.ts (b721435, 124L, 3924ch, 3926B) │ │ ├── insecure-flag.test.ts (bdb8f17, 104L, 2560ch, 2562B) │ │ ├── json-envelope.test.ts (9453611, 216L, 6200ch, 6212B) -│ │ ├── keyboard.test.tsx (83d53c1, 573L, 14942ch, 14942B) +│ │ ├── keyboard.test.tsx (5105611, 727L, 19748ch, 19748B) │ │ ├── lazy-startup.test.ts (f8758c9, 143L, 4488ch, 4488B) +│ │ ├── mouse.test.tsx (6e09b80, 1079L, 34071ch, 34129B) │ │ ├── postinstall-source-checkout.test.ts (70f941e, 195L, 6214ch, 6214B) -│ │ ├── router.test.tsx (952530c, 489L, 13626ch, 13626B) +│ │ ├── router.test.tsx (3eb11c5, 509L, 14347ch, 14347B) │ │ ├── screens.test.tsx (6edec68, 277L, 7530ch, 7530B) │ │ ├── settings-edit.test.ts (671529f, 56L, 1416ch, 1416B) │ │ ├── settings-secret.test.ts (6236925, 53L, 1365ch, 1365B) @@ -897,7 +1665,10 @@ │ │ └── yes-flag.test.ts (7eef63b, 476L, 12490ch, 12492B) │ ├── core/ (30) │ │ ├── change/ (9) -│ │ │ ├── fixtures/ (0 files, 3 dirs) +│ │ │ ├── fixtures/ (3) +│ │ │ │ ├── changes/ (0 files, 9 dirs) +│ │ │ │ ├── manifests/ (5 files, 0 dirs) +│ │ │ │ └── schema/ (0 files, 2 dirs) │ │ │ ├── executor-retry.test.ts (812eeff, 287L, 11181ch, 11193B) │ │ │ ├── executor.test.ts (59f53c9, 388L, 14130ch, 14134B) │ │ │ ├── history.test.ts (9b7b5dd, 202L, 6278ch, 6282B) @@ -911,13 +1682,19 @@ │ │ │ ├── resolver.test.ts (932661e, 830L, 24807ch, 24809B) │ │ │ ├── schema.test.ts (3c705a8, 505L, 14798ch, 14799B) │ │ │ └── validate.test.ts (b38e5f9, 72L, 2255ch, 2255B) -│ │ ├── connection/ (4) -│ │ │ ├── dialects/ (1 file, 0 dirs) +│ │ ├── connection/ (6) +│ │ │ ├── dialects/ (1) +│ │ │ │ └── mssql.test.ts (fee58a6, 187L, 6205ch, 6207B) │ │ │ ├── defaults.test.ts (9203348, 50L, 1640ch, 1642B) +│ │ │ ├── errors.test.ts (28197a2, 113L, 3857ch, 3857B) │ │ │ ├── factory.test.ts (dea2d3f, 305L, 9902ch, 10120B) -│ │ │ └── manager.test.ts (fbf9968, 192L, 5449ch, 5449B) +│ │ │ ├── manager.test.ts (fbf9968, 192L, 5449ch, 5449B) +│ │ │ └── timeout.test.ts (2dcc8d7, 165L, 4817ch, 4817B) │ │ ├── db/ (2) -│ │ │ ├── dialects/ (3 files, 0 dirs) +│ │ │ ├── dialects/ (3) +│ │ │ │ ├── mssql.test.ts (c0493aa, 139L, 4163ch, 4165B) +│ │ │ │ ├── mysql.test.ts (ddfa8a3, 114L, 3329ch, 3329B) +│ │ │ │ └── postgres.test.ts (3c7c9a4, 114L, 3219ch, 3219B) │ │ │ └── operations.test.ts (7491818, 181L, 5705ch, 6133B) │ │ ├── debug/ (1) │ │ │ └── operations.test.ts (b8472ca, 621L, 19396ch, 19890B) @@ -936,10 +1713,15 @@ │ │ │ ├── type-map.test.ts (0d0c9d7, 508L, 15680ch, 15680B) │ │ │ ├── worker-pipeline.test.ts (39f13d6, 323L, 10611ch, 10619B) │ │ │ └── writer.test.ts (c97dc9e, 221L, 6635ch, 6635B) -│ │ ├── explore/ (4) -│ │ │ ├── dialects/ (4 files, 0 dirs) +│ │ ├── explore/ (5) +│ │ │ ├── dialects/ (4) +│ │ │ │ ├── mssql.test.ts (b3804c1, 389L, 13263ch, 13263B) +│ │ │ │ ├── mysql.test.ts (4f2421b, 325L, 11012ch, 11012B) +│ │ │ │ ├── postgres.test.ts (0681aa8, 444L, 15943ch, 15943B) +│ │ │ │ └── sqlite.test.ts (c94b6fe, 393L, 13786ch, 13790B) │ │ │ ├── dispatch.test.ts (a76df89, 255L, 8761ch, 8761B) │ │ │ ├── operations.test.ts (64e9a5f, 589L, 16355ch, 16371B) +│ │ │ ├── peek.test.ts (f416956, 423L, 15039ch, 15041B) │ │ │ └── recording-db.ts (cca9453, 233L, 6106ch, 6108B) │ │ ├── identity/ (10) │ │ │ ├── crypto.test.ts (03e046a, 401L, 11922ch, 11926B) @@ -996,8 +1778,14 @@ │ │ │ ├── session-not-found.test.ts (1e7939f, 76L, 2663ch, 2665B) │ │ │ ├── session-status.test.ts (a9a1f86, 230L, 7452ch, 7454B) │ │ │ └── session.test.ts (196c2e2, 194L, 5760ch, 5760B) -│ │ ├── runner/ (10) -│ │ │ ├── fixtures/ (6 files, 0 dirs) +│ │ ├── runner/ (11) +│ │ │ ├── fixtures/ (6) +│ │ │ │ ├── create-table.sql (5ea918f, 1L, 28ch, 28B) +│ │ │ │ ├── raw.sql (d1ec24b, 1L, 20ch, 20B) +│ │ │ │ ├── secrets.sql.tmpl (db20068, 1L, 59ch, 59B) +│ │ │ │ ├── select-1.sql (17db4fd, 1L, 9ch, 9B) +│ │ │ │ ├── select-2.sql (8e7003d, 1L, 9ch, 9B) +│ │ │ │ └── template.sql.tmpl (8e8f2e5, 1L, 36ch, 36B) │ │ │ ├── checksum.test.ts (16c5853, 151L, 4092ch, 4092B) │ │ │ ├── dry-run-output.test.ts (3f9e652, 136L, 4787ch, 4789B) │ │ │ ├── execute-files.test.ts (0eb62b0, 111L, 3835ch, 3837B) @@ -1005,37 +1793,51 @@ │ │ │ ├── runner.test.ts (9a75b44, 248L, 7884ch, 7886B) │ │ │ ├── sqlite-multi-statement.test.ts (ae75da5, 194L, 6351ch, 6355B) │ │ │ ├── sqlite-statements.test.ts (6e299e0, 144L, 4338ch, 4340B) +│ │ │ ├── statement-watcher.test.ts (86b069c, 243L, 8019ch, 8019B) │ │ │ ├── template-dedup.test.ts (434e39a, 165L, 5880ch, 5888B) │ │ │ └── tracker.test.ts (9098589, 382L, 14862ch, 14870B) │ │ ├── secrets/ (1) │ │ │ └── leakage.test.ts (e06aa00, 174L, 5317ch, 5321B) │ │ ├── settings/ (5) │ │ │ ├── env-override.test.ts (70e079d, 240L, 7571ch, 7573B) -│ │ │ ├── manager.test.ts (0801cec, 1122L, 25261ch, 25261B) +│ │ │ ├── manager.test.ts (f898974, 1173L, 26570ch, 26570B) │ │ │ ├── rules.test.ts (dfc4b91, 416L, 13289ch, 13289B) -│ │ │ ├── schema.test.ts (5e7baac, 517L, 13184ch, 13184B) +│ │ │ ├── schema.test.ts (29ab05e, 592L, 15455ch, 15455B) │ │ │ └── setTeardown.test.ts (c2d7f2b, 84L, 2603ch, 2603B) -│ │ ├── shared/ (4) +│ │ ├── shared/ (6) +│ │ │ ├── abort.test.ts (23b6b1e, 221L, 6241ch, 6241B) │ │ │ ├── errors.test.ts (e78da77, 279L, 7975ch, 9571B) │ │ │ ├── files.test.ts (cad503b, 233L, 6495ch, 6497B) +│ │ │ ├── mask.test.ts (dc9243e, 119L, 3736ch, 3764B) │ │ │ ├── operation-id.test.ts (dd60e8d, 357L, 12035ch, 12037B) │ │ │ └── tables.test.ts (b53c90b, 110L, 3272ch, 3272B) -│ │ ├── sql-terminal/ (2) +│ │ ├── sql-terminal/ (3) +│ │ │ ├── executor-abort.test.ts (9dde51c, 213L, 6484ch, 6486B) │ │ │ ├── executor.test.ts (567ba1f, 480L, 16875ch, 16881B) │ │ │ └── history.test.ts (caa6399, 1071L, 36485ch, 36485B) │ │ ├── state/ (7) -│ │ │ ├── encryption/ (1 file, 0 dirs) -│ │ │ ├── fixtures/ (1 file, 0 dirs) +│ │ │ ├── encryption/ (1) +│ │ │ │ └── crypto.test.ts (5d8515e, 394L, 12799ch, 12821B) +│ │ │ ├── fixtures/ (1) +│ │ │ │ └── concurrent-writer.ts (dc72d3c, 21L, 694ch, 694B) │ │ │ ├── access.test.ts (de9f179, 128L, 4410ch, 4412B) │ │ │ ├── durability.test.ts (396279c, 445L, 14751ch, 15733B) │ │ │ ├── manager.test.ts (20b1f8d, 1141L, 38399ch, 40353B) │ │ │ ├── merge.test.ts (5cb6c34, 172L, 4838ch, 4840B) │ │ │ └── migrations.test.ts (d4d9816, 147L, 4690ch, 4690B) │ │ ├── teardown/ (2) -│ │ │ ├── dialects/ (4 files, 0 dirs) +│ │ │ ├── dialects/ (4) +│ │ │ │ ├── mssql.test.ts (eb31e6d, 394L, 12114ch, 12116B) +│ │ │ │ ├── mysql.test.ts (dfa3f0c, 271L, 7479ch, 7479B) +│ │ │ │ ├── postgres.test.ts (9a0dc04, 331L, 9847ch, 9847B) +│ │ │ │ └── sqlite.test.ts (164ade1, 255L, 7556ch, 7556B) │ │ │ └── operations.test.ts (c28fbea, 1088L, 38138ch, 40630B) │ │ ├── template/ (8) -│ │ │ ├── fixtures/ (0 files, 4 dirs) +│ │ │ ├── fixtures/ (4) +│ │ │ │ ├── engine/ (10 files, 2 dirs) +│ │ │ │ ├── helpers/ (0 files, 9 dirs) +│ │ │ │ ├── loaders/ (13 files, 0 dirs) +│ │ │ │ └── security/ (0 files, 3 dirs) │ │ │ ├── context.test.ts (e77b083, 93L, 3444ch, 3448B) │ │ │ ├── engine.test.ts (5bfd5f9, 403L, 12250ch, 12250B) │ │ │ ├── helpers.test.ts (4d0a204, 130L, 3944ch, 3944B) @@ -1044,7 +1846,11 @@ │ │ │ ├── security.test.ts (b0b47a6, 285L, 9507ch, 9509B) │ │ │ └── utils.test.ts (8967c7c, 174L, 3814ch, 3816B) │ │ ├── transfer/ (9) -│ │ │ ├── dialects/ (4 files, 0 dirs) +│ │ │ ├── dialects/ (4) +│ │ │ │ ├── index.test.ts (c39ef88, 119L, 3189ch, 3189B) +│ │ │ │ ├── mssql.test.ts (eedfe44, 263L, 7644ch, 7644B) +│ │ │ │ ├── mysql.test.ts (9026a4b, 207L, 5576ch, 5576B) +│ │ │ │ └── postgres.test.ts (86c4af1, 209L, 6176ch, 6180B) │ │ │ ├── counts.test.ts (900d9b1, 187L, 6691ch, 6695B) │ │ │ ├── events.test.ts (ab6d02e, 364L, 10614ch, 10614B) │ │ │ ├── executor.test.ts (602b084, 420L, 13400ch, 13400B) @@ -1083,14 +1889,42 @@ │ │ └── project.test.ts (cd6cc41, 243L, 6575ch, 6575B) │ ├── fixtures/ (3) │ │ ├── ci/ (2) -│ │ │ ├── failure/ (0 files, 1 dir) -│ │ │ └── success/ (0 files, 1 dir) +│ │ │ ├── failure/ (1) +│ │ │ │ └── sql/ (2 files, 0 dirs) +│ │ │ └── success/ (1) +│ │ │ └── sql/ (2 files, 0 dirs) │ │ ├── sql/ (5) -│ │ │ ├── edge-cases/ (5 files, 0 dirs) -│ │ │ ├── mssql/ (9 files, 0 dirs) -│ │ │ ├── mysql/ (4 files, 0 dirs) -│ │ │ ├── postgres/ (5 files, 0 dirs) -│ │ │ └── sqlite/ (3 files, 0 dirs) +│ │ │ ├── edge-cases/ (5) +│ │ │ │ ├── circular-fk.sql (3e04841, 28L, 950ch, 950B) +│ │ │ │ ├── composite-keys.sql (23856e8, 36L, 1284ch, 1284B) +│ │ │ │ ├── long-names.sql (f335270, 18L, 738ch, 738B) +│ │ │ │ ├── self-ref-fk.sql (ac7f0bb, 26L, 868ch, 868B) +│ │ │ │ └── special-chars.sql (9a5fe96, 24L, 756ch, 756B) +│ │ │ ├── mssql/ (9) +│ │ │ │ ├── 001_types.sql (97b4d01, 17L, 540ch, 540B) +│ │ │ │ ├── 002_tables.sql (520f7f3, 54L, 2402ch, 2406B) +│ │ │ │ ├── 003_views.sql (ae7b836, 55L, 1360ch, 1362B) +│ │ │ │ ├── 004_functions.sql (0ec23d3, 43L, 1111ch, 1111B) +│ │ │ │ ├── 005_procedures.sql (f1b332a, 224L, 6495ch, 6495B) +│ │ │ │ ├── 006_triggers.sql (0e7130f, 145L, 3785ch, 3785B) +│ │ │ │ ├── 006_tvfs.sql (37d8d6e, 32L, 887ch, 887B) +│ │ │ │ ├── 008_tvps.sql (21472ae, 18L, 513ch, 513B) +│ │ │ │ └── 009_tvp_procedures.sql (0f499ff, 50L, 1346ch, 1346B) +│ │ │ ├── mysql/ (4) +│ │ │ │ ├── 001_tables.sql (7ac6f92, 54L, 2105ch, 2109B) +│ │ │ │ ├── 002_views.sql (4f555d0, 52L, 1264ch, 1266B) +│ │ │ │ ├── 003_procedures.sql (9780142, 187L, 5605ch, 5605B) +│ │ │ │ └── 004_triggers.sql (6371be2, 107L, 3072ch, 3072B) +│ │ │ ├── postgres/ (5) +│ │ │ │ ├── 001_tables.sql (09f6493, 51L, 2095ch, 2095B) +│ │ │ │ ├── 002_views.sql (219ea4b, 51L, 1200ch, 1200B) +│ │ │ │ ├── 003_functions.sql (ea3ddef, 279L, 7745ch, 7745B) +│ │ │ │ ├── 004_triggers.sql (748e373, 80L, 2889ch, 2889B) +│ │ │ │ └── 004_tvfs.sql (e46df5c, 55L, 1483ch, 1483B) +│ │ │ └── sqlite/ (3) +│ │ │ ├── 001_tables.sql (f73d593, 51L, 2706ch, 2710B) +│ │ │ ├── 002_views.sql (63ad3b7, 55L, 1358ch, 1358B) +│ │ │ └── 003_triggers.sql (2b237a4, 120L, 3807ch, 3807B) │ │ └── workers/ (4) │ │ ├── adder.ts (3243e61, 15L, 443ch, 443B) │ │ ├── dying.ts (1d8824e, 11L, 296ch, 296B) @@ -1105,23 +1939,28 @@ │ │ │ ├── db.test.ts (87a6c15, 447L, 12079ch, 13545B) │ │ │ ├── lock.test.ts (8ea85ed, 303L, 8252ch, 9472B) │ │ │ └── setup.ts (9e503fb, 450L, 11325ch, 12301B) -│ │ ├── connection/ (1) -│ │ │ └── mssql-sni.test.ts (d453b28, 99L, 2780ch, 2782B) +│ │ ├── connection/ (4) +│ │ │ ├── connection-errors.test.ts (9100e35, 498L, 17702ch, 17702B) +│ │ │ ├── mssql-login.test.ts (dfa25cc, 165L, 5636ch, 5636B) +│ │ │ ├── mssql-sni.test.ts (d453b28, 99L, 2780ch, 2782B) +│ │ │ └── timeout-abort.test.ts (097ca1e, 153L, 5006ch, 5008B) │ │ ├── error-diagnostics/ (2) │ │ │ ├── mssql.test.ts (38889e2, 178L, 5706ch, 5706B) │ │ │ └── postgres.test.ts (2044f49, 103L, 3005ch, 3005B) -│ │ ├── explore/ (6) +│ │ ├── explore/ (7) │ │ │ ├── edge-cases.test.ts (a99161b, 524L, 15926ch, 15926B) │ │ │ ├── mssql.test.ts (7c4a061, 566L, 18515ch, 18515B) │ │ │ ├── multi-schema.test.ts (daa43b0, 463L, 14724ch, 14731B) │ │ │ ├── mysql.test.ts (ad2a8d7, 471L, 14059ch, 14059B) -│ │ │ ├── postgres.test.ts (b4c311b, 444L, 14795ch, 14795B) +│ │ │ ├── postgres.test.ts (a44b0a7, 498L, 16731ch, 16731B) +│ │ │ ├── row-peek.test.ts (904347c, 516L, 17363ch, 17365B) │ │ │ └── sqlite.test.ts (2b44515, 426L, 13079ch, 13079B) │ │ ├── impersonate/ (2) │ │ │ ├── mssql.test.ts (36695d6, 114L, 3625ch, 3629B) │ │ │ └── postgres.test.ts (0c9bdd6, 313L, 10055ch, 10065B) -│ │ ├── runner/ (2) +│ │ ├── runner/ (3) │ │ │ ├── mssql-batches.test.ts (b60b9e8, 268L, 8063ch, 8065B) +│ │ │ ├── statement-watcher.test.ts (388cdfd, 327L, 10559ch, 10559B) │ │ │ └── tracker-dialects.test.ts (377c0de, 159L, 5889ch, 5899B) │ │ ├── sdk/ (8) │ │ │ ├── db-reset.test.ts (ddfaa6e, 119L, 3933ch, 3939B) @@ -1132,7 +1971,8 @@ │ │ │ ├── tvp.test.ts (61c50ec, 599L, 17433ch, 19149B) │ │ │ ├── vault-namespace.test.ts (e7dffc5, 291L, 9810ch, 10312B) │ │ │ └── with-schema.test.ts (95a2dc0, 648L, 23109ch, 24355B) -│ │ ├── sql-terminal/ (5) +│ │ ├── sql-terminal/ (6) +│ │ │ ├── cancel.test.ts (adaf546, 286L, 8366ch, 8374B) │ │ │ ├── classifier-differential.test.ts (8419403, 226L, 9384ch, 9392B) │ │ │ ├── mssql.test.ts (1c242a3, 776L, 23168ch, 23168B) │ │ │ ├── mysql.test.ts (19a7a01, 658L, 18773ch, 18773B) @@ -1187,43 +2027,44 @@ ├── .npmrc (60376c8, 1L, 36ch, 36B) ├── .prettierignore (e3b0c44, 0L, 0ch, 0B) ├── .signalsignore (b0287a5, 17L, 662ch, 674B) -├── CLAUDE.md (c67f06a, 213L, 11996ch, 12024B) +├── CLAUDE.md (8b0f96e, 211L, 12003ch, 12031B) ├── CNAME (f3bed50, 1L, 9ch, 9B) ├── LICENSE (cfc7749, 202L, 11358ch, 11358B) ├── NOTICE (d698d9d, 2L, 35ch, 35B) ├── README.md (c149ae7, 124L, 4991ch, 5012B) ├── TODO.md (afd3ad2, 284L, 17713ch, 17967B) -├── bun.lockb (ca7f592, 449L, 263701ch, 265728B) +├── bun.lockb (5c93336, 462L, 267003ch, 269072B) ├── bunfig.toml (dab752d, 5L, 89ch, 89B) ├── docker-compose.test.yml (d2d79cb, 59L, 1534ch, 1534B) ├── eslint.config.js (cd11fe6, 56L, 2111ch, 2111B) ├── install.sh (5cb346d, 200L, 5892ch, 5892B) -├── package.json (937e22d, 97L, 2877ch, 2877B) +├── package.json (2463313, 97L, 2877ch, 2877B) ├── tsconfig.json (640d95f, 21L, 616ch, 616B) ├── tsconfig.sdk-types.json (47a2c39, 13L, 303ch, 303B) ├── tsconfig.test.json (a5e75f7, 10L, 221ch, 221B) ├── tsup.cli.config.ts (8de2742, 45L, 1440ch, 1440B) └── tsup.sdk.config.ts (a6518e1, 40L, 1107ch, 1111B) +``` ## Manifests - docs/package.json: name=@noormdev/docs, scripts=[build, dev, preview] -- examples/llm-memory-db-mssql/package.json: name=@noormdev/example-llm-memory-db-mssql, version=0.0.3, scripts=[test, test:watch, typecheck] -- examples/llm-memory-db-pg/package.json: name=@noormdev/example-llm-memory-db-pg, version=0.0.3, scripts=[test, test:watch, typecheck] -- examples/todo-db/package.json: name=@noormdev/example-todo-db, version=0.0.3, scripts=[test, test:watch, typecheck] +- examples/llm-memory-db-mssql/package.json: name=@noormdev/example-llm-memory-db-mssql, version=0.0.7, scripts=[test, test:watch, typecheck] +- examples/llm-memory-db-pg/package.json: name=@noormdev/example-llm-memory-db-pg, version=0.0.7, scripts=[test, test:watch, typecheck] +- examples/todo-db/package.json: name=@noormdev/example-todo-db, version=0.0.7, scripts=[test, test:watch, typecheck] - package.json: name=@noormdev/main, version=0.0.1, scripts=[build, build:binary, build:packages, changeset, clean, dev, lint, lint:docs, lint:fix, prepublishOnly, release, start, test, test:coverage, test:watch, typecheck, typecheck:tests, version] -- packages/cli/package.json: name=@noormdev/cli, version=1.0.2, scripts=[postinstall] -- packages/sdk/package.json: name=@noormdev/sdk, version=1.0.2 +- packages/cli/package.json: name=@noormdev/cli, version=1.4.0, scripts=[postinstall] +- packages/sdk/package.json: name=@noormdev/sdk, version=1.4.0 ## Languages -- TypeScript: 242110 LOC (80%), 1033 files (77%) -- Markdown: 50039 LOC (16%), 184 files (13%) +- TypeScript: 264274 LOC (81%), 1097 files (78%) +- Markdown: 51566 LOC (15%), 192 files (13%) - HTML: 2977 LOC (0%), 30 files (2%) -- JavaScript: 1261 LOC (0%), 22 files (1%) +- JavaScript: 1265 LOC (0%), 22 files (1%) - YAML: 1186 LOC (0%), 19 files (1%) - CSS: 1103 LOC (0%), 3 files (0%) -- Shell: 932 LOC (0%), 7 files (0%) +- Shell: 952 LOC (0%), 7 files (0%) - JSON: 473 LOC (0%), 22 files (1%) - Vue: 205 LOC (0%), 3 files (0%) - TOML: 11 LOC (0%), 3 files (0%) diff --git a/docs/wiki/tui.md b/docs/wiki/tui.md index a5d5c835..d41a3d45 100644 --- a/docs/wiki/tui.md +++ b/docs/wiki/tui.md @@ -1,59 +1,156 @@ --- type: Domain description: Ink/React TUI launched by `noorm ui` — a custom focus-stack keyboard router, per-domain screen registry, and shared providers for state, connection, and shutdown lifecycle. +tags: [tui, ink, react] --- # tui ## What it does -- Renders the interactive terminal UI launched via `noorm ui` ([`src/cli/ui.ts`](../../src/cli/ui.ts)), the only CLI subcommand that mounts Ink. -- Routes navigation through a 109-member string-union `Route` type ([`src/tui/types.ts`](../../src/tui/types.ts)); the flat `SCREENS` registry ([`src/tui/screens.tsx`](../../src/tui/screens.tsx)), keyed by route, registers 94 of those routes to screens — unregistered routes fall through to `NotFoundScreen`. +- Renders the interactive terminal UI mounted by `noorm ui` ([`src/cli/ui.ts`](../../src/cli/ui.ts)) and the SQL REPL ([`src/cli/sql/repl.ts`](../../src/cli/sql/repl.ts)). +- Routes navigation through a string-union `Route` type ([`src/tui/types.ts`](../../src/tui/types.ts)); the flat `SCREENS` registry ([`src/tui/screens.tsx`](../../src/tui/screens.tsx)), keyed by route, maps routes to screen components — an unregistered route falls through to `NotFoundScreen`. - Owns a custom focus stack ([`src/tui/focus.tsx`](../../src/tui/focus.tsx)) that gates which component receives keyboard input, used instead of `@inkjs/ui`'s incompatible internal focus system. - Bridges core managers (`StateManager`, `SettingsManager`) and the `@logosdx/observer` event bus into React state via `AppContextProvider` ([`src/tui/app-context.tsx`](../../src/tui/app-context.tsx)). +- Surfaces what a long-running SQL statement is doing on the server (`StatementProgress`, [`src/tui/components/status/StatementProgress.tsx`](../../src/tui/components/status/StatementProgress.tsx)) on every run and change-apply screen: `RunBuildScreen`, `RunExecScreen`, `RunDirScreen`, `RunFileScreen`, and the `change/` screens. `RunBuildScreen`/`RunExecScreen` gate their cancel behind a double-`Escape` (`useDoublePress`, [`src/tui/hooks/useDoublePress.ts`](../../src/tui/hooks/useDoublePress.ts)); `RunDirScreen`/`RunFileScreen` cancel on a single `Escape`. -## Artifacts - -- [`.claude/rules/tui-development.md`](../../.claude/rules/tui-development.md) — path-scoped rules for this domain: focus system, `@inkjs/ui` boundary, keyboard handling, screen focus ownership, UI patterns (toast + `back()`), Ink layout, observer hooks, and testing conventions. Frontmatter scopes it to `src/tui/**/*.{ts,tsx}, tests/tui/**/*.{ts,tsx}`. -- [`tests/cli/components/`](../../tests/cli/components) — component tests (`dialogs`, `DismissableAlert`, `form-navigation`, `forms`, `layout`, `lists`, `status`) using `ink-testing-library`. -- [`tests/cli/hooks/`](../../tests/cli/hooks) — hook tests (`useObserver`, `useTransferProgress`, `useUpdateChecker`, `useVaultSecretKeys`). -- [`tests/cli/screens/`](../../tests/cli/screens) — screen tests, mirroring [`src/tui/screens/`](../../src/tui/screens) subdirectories (`change/`, `config/`, `db/`, `init/`). - -## CLI code - -- [`src/tui/app.tsx`](../../src/tui/app.tsx) — root `App` component; wires the provider hierarchy (`ShutdownProvider` → `NoormObserver` → `AppContextProvider` → `ConnectionProvider` → `ToastProvider` → `FocusProvider` → `RouterProvider` → `AppShell`) and renders the header/breadcrumb, `ScreenRenderer`, and status bar. -- [`src/tui/app-context.tsx`](../../src/tui/app-context.tsx) — `AppContextProvider`/`useAppContext` and derived hooks (`useActiveConfig`, `useLockStatus`, `useGlobalModes`, `useDryRunMode`, `useForceMode`, `useExploreFilters`); also exports `LoadingGuard`, `ConfigGuard`, `IdentityGuard`. -- [`src/tui/focus.tsx`](../../src/tui/focus.tsx) — `FocusProvider`, `useFocusScope`, `useIsFocused`, `useActiveFocus`; stack-based, last-pushed-wins. -- [`src/tui/keyboard.tsx`](../../src/tui/keyboard.tsx) — `GlobalKeyboard` (Ctrl+C, Shift+L log viewer, Shift+Q SQL terminal, `?` help / 4-press-within-750ms debug-mode easter egg, `D`/`F` dry-run/force toggles); `useFocusedInput`, `useListKeys`, `useQuitHandler`. -- [`src/tui/router.tsx`](../../src/tui/router.tsx) — `RouterProvider`/`useRouter` with `navigate`/`back`/`replace`/`reset`, a history stack, and `router:navigated`/`router:popped` observer events. -- [`src/tui/screens.tsx`](../../src/tui/screens.tsx) — the `SCREENS` route registry and `ScreenRenderer`; `getRouteLabel`, `getRegisteredRoutes`, `isRouteRegistered`, `registerScreen`. -- [`src/tui/types.ts`](../../src/tui/types.ts) — `Route` union, `RouteParams`, `RouterContextValue`, `FocusContextValue`, `ScreenProps`/`ScreenEntry`, `getSection`/`getParentRoute`/`isNumericString`. -- [`src/tui/shutdown.tsx`](../../src/tui/shutdown.tsx) — `ShutdownProvider`/`useShutdown`; `gracefulExit()` drives `LifecycleManager.shutdown()` and shows a phased `ShutdownScreen` before emitting the `app:exit` observer event. -- [`src/tui/observer-context.ts`](../../src/tui/observer-context.ts) — `NoormObserver`/`useNoormObserver`, built via `@logosdx/react`'s `createObserverContext` over the shared `observer` singleton. -- [`src/tui/components/`](../../src/tui/components) — shared UI: `layout/` (`Panel`, `Divider`, `MissingParamPanel`, `NotFoundPanel`), `lists/` (`SelectList`, `SearchableList`, `ActionList`, `StatusList`), `forms/` (`Form`, `TextInput`), `feedback/` (`Toast`/`ToastProvider`, `DismissableAlert`, plus re-exports of `@inkjs/ui`'s `Spinner`/`ProgressBar`/`Alert`/`StatusMessage`/`Badge`), `dialogs/` (`Confirm`, `ProtectedConfirm`, `SmartConfirm`, `FilePicker`, `KeyHandler`), `status/` (`ConnectionStatus`, `LockStatus`), `secrets/` (definition/value forms and lists shared by settings and stage-secret screens), `overlays/` (`LogViewerOverlay`), `terminal/` (`SqlInput`, `ResultTable` for the SQL REPL). `components/index.ts` is the primary re-export surface consumed by screens, though some screens import subdirectory barrels (or files) directly instead — e.g. [`src/tui/screens/db/SqlTerminalScreen.tsx`](../../src/tui/screens/db/SqlTerminalScreen.tsx) and `SqlHistoryScreen.tsx` from `components/terminal/`, [`src/tui/screens/debug/DebugListScreen.tsx`](../../src/tui/screens/debug/DebugListScreen.tsx) from `components/lists/`, [`src/tui/screens/init/ProjectSetup.tsx`](../../src/tui/screens/init/ProjectSetup.tsx), `IdentitySetup.tsx`, `InitScreen.tsx` from multiple subdirectory barrels, and [`src/tui/screens/identity/IdentityEditScreen.tsx`](../../src/tui/screens/identity/IdentityEditScreen.tsx) from `components/forms/`. -- [`src/tui/hooks/`](../../src/tui/hooks) — `useObserver.ts` (`useOnEvent`, `useOnceEvent`, `useEmit`, `useOnScreenPopped`), plus `useConnection.ts`, `useVaultConnection.ts`, `useVaultSecretKeys.ts`, `useLockStatus.ts`, `useLoadGuard.ts`, `useRunProgress.ts`, `useTransferProgress.ts`, `useChangeProgress.ts`, `useUpdateChecker.ts`, `useSettingsOperation.ts`, `useSecretSource.ts`, `useAsyncEffect.ts`; re-exported from `hooks/index.ts`. -- [`src/tui/providers/ConnectionProvider.tsx`](../../src/tui/providers/ConnectionProvider.tsx) — `ConnectionProvider`/`useConnectionContext`; holds one lazily-created Kysely connection keyed by `activeConfigName`, destroyed on config change or unmount. -- [`src/tui/utils/`](../../src/tui/utils) — screen-facing helpers: `change-context.ts`, `change-loader.ts`, `clipboard.ts`, `config-validation.ts`, `connection.ts`, `date.ts`, `error.ts`, `identity.ts`, `paths.ts`, `run-context.ts`, `settings-validation.ts`, `string.ts`. All but `date.ts` are re-exported from `utils/index.ts`; `date.ts` is not in the barrel — consumers (`home.tsx`, `ChangeHistoryScreen.tsx`, `ChangeListScreen.tsx`, `identity/KnownUsersScreen.tsx`) import `relativeTimeAgo` directly from `../utils/date.js`. -- [`src/tui/screens/`](../../src/tui/screens) — per-domain screen components: `change/`, `config/`, `db/` (including `db/explore/`), `debug/`, `identity/`, `init/`, `lock/`, `run/`, `secret/`, `settings/`, `vault/`, plus top-level `home.tsx`, `MoreScreen.tsx`, `not-found.tsx`, `UpdateScreen.tsx`. Each multi-screen subdirectory exports its screens through a local `index.ts` (e.g. [`src/tui/screens/config/index.ts`](../../src/tui/screens/config/index.ts)) that `screens.tsx` imports from. +## How it works + +Focus is a stack, not a DOM: `useFocusScope(label)` pushes an ID on mount and pops on unmount, and only the ID on top is `isFocused`. Every `useInput` handler in the tree receives every keystroke regardless of focus, so the handler itself is what has to check `isFocused` and no-op otherwise. + +A keystroke reaches every registered handler; only `GlobalKeyboard`'s own checks and the focus stack decide which one acts on it. + +```mermaid +flowchart TD + Key["keypress"] --> Ink["Ink useInput"] + Ink --> Global["GlobalKeyboard"] + Ink --> Local["screen useInput"] + Global --> CtrlC{"Ctrl+C?"} + CtrlC -->|yes| Exit["gracefulExit()"] + CtrlC -->|no| Typing{"isTyping()?"} + Typing -->|"yes"| Drop["types into
the field"] + Typing -->|no| GlobalAct["hotkey callback
(help, dry-run, etc.)"] + Local --> Focused{"isFocused?"} + Focused -->|no| Ignore["no-op"] + Focused -->|yes| ScreenAct["screen
handles key"] +``` + +`useFocusScope` pushes onto the stack inside a `useEffect`, so `isFocused` is `false` on the first render; `useFocusedInput` ([`src/tui/keyboard.tsx`](../../src/tui/keyboard.tsx)) wraps the guard correctly and is the preferred entry point over checking `isFocused` by hand. + +### How a long-running statement's progress reaches the screen + +`StatementWatcher` starts a timer when a file begins, and once that file has run past a watch delay it polls the dialect's status probe, emitting `file:progress` on the shared observer and repeating on a watch interval until the file finishes. + +`StatementWatcher` ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) is core-runner's and emits the event; [`src/core/observer.ts`](../../src/core/observer.ts) declares the event's shape; `useRunProgress`/`useChangeProgress` in this domain are the only consumers that turn it into the `statement` field a screen reads. + +```mermaid +sequenceDiagram + participant Watcher as StatementWatcher + participant Observer as observer + participant Hook as useRunProgress + participant UI as StatementProgress + + Watcher->>Observer: emit file:progress + Observer-->>Hook: useOnEvent('file:progress') + Hook->>Hook: store as statement + Hook-->>UI: report prop + UI->>UI: render elapsed time, server progress, blockers +``` + +Each run boundary clears `statement` back to `null` rather than letting a stale report survive past the file it described: `useRunProgress` clears it on `file:before`, `file:after`, and `build:complete`; `useChangeProgress` clears it on `change:file` and `change:complete`; both also clear it on their own `reset()`, called when a screen starts a new run. + +### Cancelling a running build or exec + +`RunBuildScreen` and `RunExecScreen` are the only screens wired to `useDoublePress`; a single `Escape` during their run only warns, and the abort needs a second press inside the window. + +```mermaid +stateDiagram-v2 + [*] --> Running + Running --> Warned: Escape (confirmCancel() false) + Warned --> Running: 2s window elapses, no second Escape + Warned --> Cancelling: Escape again within 2s (confirmCancel() true) + Cancelling --> Complete: runBuild resolves, error 'Run cancelled' + Running --> Complete: run settles + Running --> Error: connection or policy failure + Complete --> [*]: Escape (back()) + Error --> [*]: Escape (back()) +``` + +`back()` only fires once `phase !== 'running'` (`RunBuildScreen.tsx:196-217`); while `phase === 'running'` every `Escape` is consumed by the arm/confirm logic above, never by navigation. + +`useDoublePress` ([`src/tui/hooks/useDoublePress.ts`](../../src/tui/hooks/useDoublePress.ts)) arms on the first call and returns `false`; a second call inside the 2000ms window returns `true`, and a call after the window re-arms rather than confirming. On confirmation the screen aborts its `AbortController` and shows `runCancelMessage(dialect)` ([`src/tui/utils/run-context.ts`](../../src/tui/utils/run-context.ts)). On Postgres and MySQL that message reads as an immediate stop, because the server is asked to cancel the running statement (`hasServerSideCancel`, [`src/core/connection/session.ts`](../../src/core/connection/session.ts)). Everywhere else it reads as "stopping after the current file finishes", since there is no server-side cancel to issue. + +`RunDirScreen` and `RunFileScreen` take a different path: their `running`-phase view binds `` directly, so a single `Escape` (or `c`) calls `cancelExecution`, which destroys the active connection (`RunDirScreen.tsx:323-343`) with no arm-then-confirm step. + +## Where it lives + +### Core + +| Path | What | +|------|------| +| [`src/tui/app.tsx`](../../src/tui/app.tsx) | Root `App`; wires the provider hierarchy (`ShutdownProvider` → `NoormObserver` → `AppContextProvider` → `ConnectionProvider` → `ToastProvider` → `FocusProvider` → `RouterProvider` → `AppShell`) | +| [`src/tui/app-context.tsx`](../../src/tui/app-context.tsx) | `AppContextProvider`/`useAppContext` and derived hooks (`useActiveConfig`, `useLockStatus`, `useGlobalModes`, `useDryRunMode`, `useForceMode`, `useExploreFilters`); also `LoadingGuard`, `ConfigGuard`, `IdentityGuard` | +| [`src/tui/focus.tsx`](../../src/tui/focus.tsx) | `FocusProvider`, `useFocusScope`, `useIsFocused`, `useActiveFocus`; stack-based, last-pushed-wins | +| [`src/tui/keyboard.tsx`](../../src/tui/keyboard.tsx) | `GlobalKeyboard` (Ctrl+C, Shift+L log viewer, Shift+Q SQL terminal, `?` help, `D`/`F` dry-run/force toggles); `useFocusedInput`, `useListKeys`, `useQuitHandler` | +| [`src/tui/router.tsx`](../../src/tui/router.tsx) | `RouterProvider`/`useRouter`: `navigate`/`back`/`replace`/`reset`, a history stack, `router:navigated`/`router:popped` observer events | +| [`src/tui/screens.tsx`](../../src/tui/screens.tsx) | `SCREENS` route registry and `ScreenRenderer`; `getRouteLabel`, `getRegisteredRoutes`, `isRouteRegistered`, `registerScreen` | +| [`src/tui/types.ts`](../../src/tui/types.ts) | `Route` union, `RouteParams`, `RouterContextValue`, `FocusContextValue`, `ScreenProps`/`ScreenEntry` | +| [`src/tui/shutdown.tsx`](../../src/tui/shutdown.tsx) | `ShutdownProvider`/`useShutdown`; `gracefulExit()` drives `LifecycleManager.shutdown()` and shows a phased `ShutdownScreen` before `app:exit` | +| [`src/tui/observer-context.ts`](../../src/tui/observer-context.ts) | `NoormObserver`/`useNoormObserver`, built via `@logosdx/react`'s `createObserverContext` over the shared `observer` singleton | +| [`src/tui/mouse.tsx`](../../src/tui/mouse.tsx) | The mouse transport: writes tracking escape sequences, parses SGR reports off `useInput`, restores the terminal on exit, hit-tests rows with `measureElement` | + +### Components, hooks, utils + +| Path | What | +|------|------| +| [`src/tui/components/`](../../src/tui/components) | Shared UI: `layout/`, `lists/`, `forms/`, `feedback/`, `dialogs/`, `status/` (`ConnectionStatus`, `LockStatus`, `StatementProgress`), `secrets/`, `overlays/`, `terminal/`. `components/index.ts` is the primary re-export surface | +| [`src/tui/components/status/StatementProgress.tsx`](../../src/tui/components/status/StatementProgress.tsx) | Renders one `file:progress` report: elapsed time, `describeProgress()` for the server's own progress line, and a line per session it is blocked on | +| [`src/tui/hooks/`](../../src/tui/hooks) | `useObserver.ts` (`useOnEvent`, `useOnceEvent`, `useEmit`, `useOnScreenPopped`), `useConnection.ts`, `useVaultConnection.ts`, `useVaultSecretKeys.ts`, `useLockStatus.ts`, `useLoadGuard.ts`, `useRunProgress.ts`, `useChangeProgress.ts`, `useTransferProgress.ts`, `useUpdateChecker.ts`, `useUpdateProgress.ts`, `useSettingsOperation.ts`, `useSecretSource.ts`, `useAsyncEffect.ts`, `useAbortableTask.ts`, `useViewportRows.ts`, `useDoublePress.ts`; re-exported from `hooks/index.ts` | +| [`src/tui/hooks/useRunProgress.ts`](../../src/tui/hooks/useRunProgress.ts) | Tracks `build:start`/`file:before`/`file:progress`/`file:after`/`file:skip`/`file:dry-run`/`build:complete`; exposes `state.statement` (the current file's latest `file:progress`, cleared on `file:before`/`file:after`/`build:complete`/`reset()`) | +| [`src/tui/hooks/useChangeProgress.ts`](../../src/tui/hooks/useChangeProgress.ts) | Tracks `change:start`/`change:file`/`change:complete`/`file:progress`; exposes `statement`, cleared on `change:file`/`change:complete`/`reset()` | +| [`src/tui/hooks/useDoublePress.ts`](../../src/tui/hooks/useDoublePress.ts) | Arm-then-confirm for a hard-to-undo key: first call arms and returns `false`, a second call within `windowMs` (default 2000) returns `true`, a call after the window re-arms | +| [`src/tui/hooks/useAbortableTask.ts`](../../src/tui/hooks/useAbortableTask.ts) | One cancellable database operation per screen; tracks whether a result that resolves after `Escape` still belongs on screen | +| [`src/tui/hooks/useUpdateProgress.ts`](../../src/tui/hooks/useUpdateProgress.ts) | Tracks `update:*` events for the update-download progress shown in `UpdateScreen` | +| [`src/tui/hooks/useViewportRows.ts`](../../src/tui/hooks/useViewportRows.ts) | Computes how many rows a windowed list may draw from the terminal size, so a list never renders past the visible frame | +| [`src/tui/utils/`](../../src/tui/utils) | `change-context.ts`, `change-loader.ts`, `clipboard.ts`, `config-validation.ts`, `connection.ts`, `date.ts`, `error.ts`, `identity.ts`, `paths.ts`, `progress.ts`, `run-context.ts`, `settings-validation.ts`, `string.ts`. All but `date.ts` are re-exported from `utils/index.ts` | +| [`src/tui/utils/run-context.ts`](../../src/tui/utils/run-context.ts) | `buildRunContext()` assembles a `RunContext` from screen state; `runCancelMessage(dialect)` picks the cancel-in-progress copy from `hasServerSideCancel(dialect)` | +| [`src/tui/utils/progress.ts`](../../src/tui/utils/progress.ts) | `progressPercentage(completed, total)` converts completed work into the 0-100 value Ink's `ProgressBar` expects, clamped for over-counted event streams | +| [`src/tui/providers/ConnectionProvider.tsx`](../../src/tui/providers/ConnectionProvider.tsx) | `ConnectionProvider`/`useConnectionContext`; one lazily-created Kysely connection keyed by `activeConfigName`, destroyed on config change or unmount | + +### Screens and tests + +| Path | What | +|------|------| +| [`src/tui/screens/`](../../src/tui/screens) | Per-domain screen components: `change/`, `config/`, `db/` (incl. `db/explore/`), `debug/`, `identity/`, `init/`, `lock/`, `run/`, `secret/`, `settings/`, `vault/`, plus top-level `home.tsx`, `MoreScreen.tsx`, `not-found.tsx`, `UpdateScreen.tsx` | +| [`src/tui/screens/run/RunBuildScreen.tsx`](../../src/tui/screens/run/RunBuildScreen.tsx), `RunExecScreen.tsx` | Render `StatementProgress` from the run progress hook's `statement`; on `Escape` while `phase === 'running'`, gate the abort behind `useDoublePress()` and show `[Esc Esc] Cancel` until confirmed | +| [`src/tui/screens/run/RunDirScreen.tsx`](../../src/tui/screens/run/RunDirScreen.tsx), `RunFileScreen.tsx` | Render `StatementProgress` the same way; the `running`-phase view binds ``, so `Escape` or `c` destroys the active connection immediately, no arm-then-confirm step | +| [`src/tui/screens/change/ChangeRunScreen.tsx`](../../src/tui/screens/change/ChangeRunScreen.tsx), `ChangeFFScreen.tsx`, `ChangeNextScreen.tsx`, `ChangeRevertScreen.tsx`, `ChangeRewindScreen.tsx` | Render `StatementProgress` from `useChangeProgress().statement`; none binds `useDoublePress` or an `Escape` cancel path during a run | +| [`.claude/rules/tui-development.md`](../../.claude/rules/tui-development.md) | Path-scoped rules for this domain: focus system, `@inkjs/ui` boundary, keyboard handling, screen focus ownership, UI patterns, Ink layout, observer hooks, testing conventions. Scoped to `src/tui/**/*.{ts,tsx}, tests/cli/**/*.{ts,tsx}` | +| [`tests/cli/components/`](../../tests/cli/components) | Component tests, one file per component family (`forms.test.tsx`, `dialogs.test.tsx`, `status.test.tsx`, and more), flat files, using `ink-testing-library` | +| [`tests/cli/hooks/`](../../tests/cli/hooks) | Hook tests, one file per tested hook (`useDoublePress`, `useRunProgress`, and `useChangeProgress` have no dedicated test file) | +| [`tests/cli/screens/`](../../tests/cli/screens) | Screen tests, in subdirectories named after their [`src/tui/screens/`](../../src/tui/screens) counterparts | +| [`docs/tui.md`](../tui.md) | User-facing keyboard/screen reference; "Long-Running Files" documents the same status block and the two-`Escape` cancel described above | + +## Constraints + +- Check `isFocused` inside the `useInput` handler body, never via `useInput`'s `isActive` option: the option skips handler registration when `false` on the first render (`isFocused` starts `false`, set in a `useEffect`), and the handler never recovers. +- `@inkjs/ui`'s `Select`, `MultiSelect`, and `ConfirmInput` are unused because they drive their own internal focus, invisible to this app's stack; `SelectList`, `Form`, `Confirm`/`SmartConfirm` are the replacements. `Spinner`, `Badge`, `ProgressBar`, `Alert`, `StatusMessage` from `@inkjs/ui` are used directly since they are display-only or externally controlled. `TextInput` is not one of them: [`src/tui/components/forms/TextInput.tsx`](../../src/tui/components/forms/TextInput.tsx) is a local copy of upstream's component plus a mouse-report guard, kept because `@inkjs/ui`'s exports map publishes only the package root. +- A screen whose primary content is a `Form` (or other self-focusing component) does not call `useFocusScope` at the screen level; two scopes would compete for one stack slot. +- Global hotkeys (`Shift+L`, `Shift+Q`, `?`, `D`, `F`) stand down while a text field is taking input (`useTextEntry`), and `?`/`D`/`F` additionally require `stack.length <= 1` to stay out of nested dialogs. `GlobalKeyboard` deliberately does not handle `Escape`; each screen owns its own, because a global handler firing alongside a screen handler would pop history twice. +- `useDoublePress`'s window is per-hook-instance state (a `ref`), so it resets whenever the owning screen unmounts; leaving `RunBuildScreen` mid-warning and returning re-arms rather than remembering the first press. +- The cancel warning toast (`RunBuildScreen.tsx:204`, `RunExecScreen.tsx:189`) hardcodes `duration: 2000` rather than reading `useDoublePress`'s `windowMs`; changing one without the other makes the toast outlive or undershoot the confirm window. +- Leaving a run screen does not cancel the run (`docs/tui.md:265`). `RunBuildScreen`/`RunExecScreen` cancel through double-`Escape` → `AbortController.abort()`; `RunDirScreen`/`RunFileScreen` through a single `Escape` → `destroy()` on the active connection. +- `useWindowSize()`, not `useStdout().stdout.rows`, is the only correct source for terminal size: Ink's resize handler re-paints existing output without a state update, so a component that reads `stdout.rows` at render time is frozen at mount. +- Tests wrap components in ``/`` (plus `RouterProvider`/`AppContextProvider` for screens) and poll with a `waitFor(predicate)` loop before writing to `stdin` rather than a fixed sleep, because the focus stack initializes in a `useEffect` and a fixed sleep flakes under load (`.claude/rules/tui-development.md:305-330`); tests call `unmount()` to release stdin handlers. ## Coupling -- **core-change**: screens under [`src/tui/screens/change/`](../../src/tui/screens/change) import `ChangeHistory`, `discoverChanges`, and change types directly from [`src/core/change/`](../../src/core/change); `home.tsx` also reads `ChangeHistory` for the recent-activity panel. Changes to `ChangeHistory`'s or `UnifiedHistoryRecord`'s shape force edits here. -- **core-config / core-state**: `app-context.tsx` instantiates `StateManager`/`SettingsManager` from [`src/core/state/`](../../src/core/state) and [`src/core/settings/`](../../src/core/settings) and mirrors their `Config`/`Settings` types; `SettingsProvider` from [`src/core/config/resolver.ts`](../../src/core/config/resolver.ts) is used by config screens (e.g. `ConfigEditScreen.tsx`). -- **core-db** (connection/explore/transfer/teardown): `ConnectionProvider` calls `createConnection` from [`src/core/connection/`](../../src/core/connection); `testConnection` is called separately, by `ConfigAddScreen.tsx`, `ConfigEditScreen.tsx`, `SqlTerminalScreen.tsx`, and `LockForceScreen.tsx`. `db/` screens import `fetchOverview` from [`src/core/explore/`](../../src/core/explore), plus transfer/teardown/dt types from [`src/core/transfer/`](../../src/core/transfer), [`src/core/teardown/`](../../src/core/teardown), [`src/core/dt/`](../../src/core/dt). -- **core-identity**: `identity/` screens and `app-context.tsx` use `loadExistingIdentity`/`CryptoIdentity` from [`src/core/identity/`](../../src/core/identity); `vault/` screens use [`src/core/vault/`](../../src/core/vault). -- **core-policy**: `SmartConfirm`, `ConfigRemoveScreen`, `DbCreateScreen`, `DbDestroyScreen`, `DbTruncateScreen`, `DbTeardownScreen`, `DbTransferScreen`, `LockForceScreen`, `RunBuildScreen`, `VaultScreen`, and all `change/*` mutation screens (`ChangeFFScreen`, `ChangeNextScreen`, `ChangeRemoveScreen`, `ChangeRevertScreen`, `ChangeRewindScreen`, `ChangeRunScreen`) call `checkConfigPolicy`/`confirmationPhraseFor` from [`src/core/policy/`](../../src/core/policy) to decide plain `Confirm` vs. typed-phrase `ProtectedConfirm`; a change to `PolicyCheck`'s shape forces edits in `SmartConfirm.tsx` and every caller listed above. `app-context.tsx` also derives placeholder stage configs' `access` from `GUARDED_ACCESS`/`DEFAULT_ACCESS`. -- **core-identity** (lock): `home.tsx` and `lock/` screens call `getLockManager()` from [`src/core/lock/`](../../src/core/lock). -- **core-state** (observer): `app-context.tsx` subscribes to the shared `observer` singleton ([`src/core/observer.ts`](../../src/core/observer.ts)) directly via `observer.on(...)` (`state:loaded`, `config:activated`, `connection:*`, `lock:*`, etc.) and mirrors it into context state. No screen calls `useOnEvent`/`useOnceEvent`/`observer.on(...)` directly; screens instead consume domain-specific wrapper hooks — `useLockStatus`, `useConnection`, `useRunProgress`, `useTransferProgress`, `useChangeProgress`, `useVaultConnection`, `useVaultSecretKeys`, `useLoadGuard`, `useUpdateChecker` — some of which (`useRunProgress`, `useTransferProgress`, `useChangeProgress`) wrap `useOnEvent` internally, others of which derive from `app-context.tsx`'s state or don't touch the observer at all. Event-contract changes still ripple across this whole domain. -- **cli**: [`src/cli/ui.ts`](../../src/cli/ui.ts) and [`src/cli/sql/repl.ts`](../../src/cli/sql/repl.ts) are the only entry points that mount `App`; both independently (not centrally) own suppressing logger output (an identical null `Writable` stream) so Ink's rendering isn't corrupted, and own the `app:exit` → `clear()`/`unmount()` teardown sequence. - -## Conventions worth knowing - -- Focus routing is stack-based, not DOM-based: `useFocusScope(label)` pushes an ID on mount and pops on unmount; only the top-of-stack ID is `isFocused`. Check `isFocused` inside the `useInput` handler body — never via `useInput`'s `isActive` option — because the option skips handler registration and `isFocused` is still `false` during the first render. -- `@inkjs/ui`'s `Select`, `MultiSelect`, and `ConfirmInput` are not used because they drive their own internal focus, invisible to this app's stack; `SelectList`, `Form`, `Confirm`/`SmartConfirm` are the custom replacements. `TextInput`, `Spinner`, `Badge`, `ProgressBar` from `@inkjs/ui` are used directly since they're display-only or externally controlled. -- A screen whose primary content is a `Form` (or other self-focusing component) does not call `useFocusScope` at the screen level — the two scopes would compete for the same stack slot. `ConfigEditScreen.tsx` follows this: no screen-level `useFocusScope`; `Form` owns focus via its own `focusLabel` prop. -- Global hotkeys route through `GlobalKeyboard` in [`src/tui/keyboard.tsx`](../../src/tui/keyboard.tsx); screen/component-local hotkeys are declared inside each screen's own `useInput` (e.g. `HomeScreen`'s single-letter navigation block). `?`/`D`/`F` are suppressed once `stack.length > 1` (once something below the root has claimed focus, treated as "likely a text input"). -- Route params (`RouteParams` in [`src/tui/types.ts`](../../src/tui/types.ts)) are one shared shape reused across all screens (`name`, `count`, `path`, `stage`, `schema`, `operationId`, `query`, `topic`, `table`, `rowId`, `force`, `fromInit`) rather than per-route param types. -- Screens with multiple named exports live in a subdirectory with a local `index.ts` re-exporting each screen ([`src/tui/screens/config/index.ts`](../../src/tui/screens/config/index.ts), [`src/tui/screens/db/index.ts`](../../src/tui/screens/db/index.ts), etc.); `screens.tsx` imports only from these subdirectory index files, never individual screen files directly. -- Success/error feedback uses `useToast().showToast(...)` followed by `back()` to pop router history, rather than a dedicated confirmation screen — exercised in `ConfigEditScreen.tsx`'s submit handler. -- Tests wrap components in ``/`` (and `RouterProvider`/`AppContextProvider` for screen-level tests) using `ink-testing-library`'s `render`, wait ~50ms after render for the focus stack's `useEffect` to run before sending `stdin.write(...)` keystrokes, and call `unmount()` afterward to release stdin handlers. +- **core-runner**: `StatementWatcher` ([`src/core/runner/statement-watcher.ts`](../../src/core/runner/statement-watcher.ts)) emits `file:progress`, the event `StatementProgress` renders; `run-context.ts` and the run/change screens call `discoverFiles`, `runBuild`, and the `RunContext`/`Dialect` types from [`src/core/runner/`](../../src/core/runner). +- **core-state**: [`src/core/observer.ts`](../../src/core/observer.ts) declares `NoormEvents['file:progress']` (and every other event this domain subscribes to); `app-context.tsx` instantiates `StateManager`/`SettingsManager` from [`src/core/state/`](../../src/core/state) and [`src/core/settings/`](../../src/core/settings) and subscribes to the shared `observer` singleton directly for state/config/connection/lock events, while screens consume domain wrapper hooks (`useRunProgress`, `useChangeProgress`, `useLockStatus`, `useConnection`, `useTransferProgress`, `useVaultConnection`, `useVaultSecretKeys`, `useLoadGuard`, `useUpdateChecker`) instead of calling `useOnEvent` directly. +- **core-state** also covers config resolution: [`src/core/config/`](../../src/core/config) (`SettingsProvider` from [`src/core/config/resolver.ts`](../../src/core/config/resolver.ts), used by config screens) is core-state's, not a separate domain. +- **core-change**: screens under [`src/tui/screens/change/`](../../src/tui/screens/change) import `ChangeHistory`, `discoverChanges`, and change types directly from [`src/core/change/`](../../src/core/change); `home.tsx` also reads `ChangeHistory` for the recent-activity panel. +- **core-db**: `ConnectionProvider` calls `createConnection` from [`src/core/connection/`](../../src/core/connection); `hasServerSideCancel` ([`src/core/connection/session.ts`](../../src/core/connection/session.ts)) decides the cancel message `run-context.ts` shows. `db/` screens import `fetchOverview` from [`src/core/explore/`](../../src/core/explore), plus transfer/teardown types. +- **sdk**: `db/` transfer screens import the DT serialization types from [`src/core/dt/`](../../src/core/dt), which belongs to the sdk domain rather than core-db. +- **core-identity / core-policy**: `identity/` and `vault/` screens use [`src/core/identity/`](../../src/core/identity) and [`src/core/vault/`](../../src/core/vault); mutation screens call `checkConfigPolicy`/`confirmationPhraseFor` from [`src/core/policy/`](../../src/core/policy) to choose plain `Confirm` vs. typed-phrase `ProtectedConfirm`. [`src/cli/sql/`](../../src/cli/sql) (the SQL REPL this domain mounts `App` from) is core-identity's, not cli's. +- **cli**: [`src/cli/ui.ts`](../../src/cli/ui.ts) and [`src/cli/sql/repl.ts`](../../src/cli/sql/repl.ts) are the only entry points that mount `App`; both independently suppress logger output and own the `app:exit` → `clear()`/`unmount()` teardown sequence. + diff --git a/src/core/change/executor.ts b/src/core/change/executor.ts index c2decbc6..edf03b10 100644 --- a/src/core/change/executor.ts +++ b/src/core/change/executor.ts @@ -34,6 +34,7 @@ import { assertPolicy } from '../policy/index.js'; import type { Permission } from '../policy/index.js'; import type { Dialect } from '../connection/types.js'; import { computeChecksum, computeCombinedChecksum } from '../runner/checksum.js'; +import { StatementWatcher } from '../runner/statement-watcher.js'; import { getSqlErrorMessage } from '../shared/index.js'; import type { NoormDatabase } from '../shared/index.js'; import { getLockManager } from '../lock/index.js'; @@ -473,6 +474,7 @@ async function executeFiles( const dialect = context.dialect ?? 'postgres'; const expandedFiles = await expandFiles(files, context.sqlDir); + const watcher = new StatementWatcher(context.db, { dialect }); if (!TRANSACTIONAL_DIALECTS.has(dialect)) { @@ -486,7 +488,8 @@ async function executeFiles( history, context.db, startTime, - ); + watcher, + ).finally(() => watcher.close()); } @@ -514,6 +517,7 @@ async function executeFiles( trxHistory, trx, startTime, + watcher, ); if (batchResult.status !== 'success') { @@ -527,6 +531,8 @@ async function executeFiles( }), ); + await watcher.close(); + if (err) { if (err instanceof ChangeRollback) { @@ -563,6 +569,7 @@ async function runFileBatch( history: ChangeHistory, executor: Kysely, startTime: number, + watcher: StatementWatcher, ): Promise { // Create operation record @@ -746,7 +753,9 @@ async function runFileBatch( } // Execute SQL - const [, execErr] = await attempt(() => sql.raw(sqlContent).execute(executor)); + const [, execErr] = await attempt(() => + watcher.run(file.path, executor, (conn) => sql.raw(sqlContent).execute(conn)), + ); const durationMs = performance.now() - fileStart; diff --git a/src/core/connection/session.ts b/src/core/connection/session.ts new file mode 100644 index 00000000..27f315e0 --- /dev/null +++ b/src/core/connection/session.ts @@ -0,0 +1,87 @@ +/** + * Server-side session identity and cancellation. + * + * Shared by every caller that needs to point at one running statement from + * outside it: the SQL terminal cancelling a query, and the runner's statement + * watcher polling and cancelling a file. Both pin a connection, read its + * session id here, then act on that id from a second connection. + */ +import { sql } from 'kysely'; +import type { QueryExecutorProvider } from 'kysely'; + +import type { Dialect } from './types.js'; + +/** + * Query that reads back the server's own identifier for the connection it + * runs on, as a single `id` column. + * + * sqlite is absent: in-process and single-connection, it has no session a + * second connection could observe. + */ +export const SESSION_ID_SQL: Partial> = { + postgres: 'select pg_backend_pid() as id', + mysql: 'select connection_id() as id', + mssql: 'select @@SPID as id', +}; + +/** + * How a dialect is told to stop a statement that is already running, issued + * from a *different* connection because the first one is busy. + */ +export type ServerCancel = (db: QueryExecutorProvider, sessionId: number) => Promise; + +/** + * Dialects where aborting sends the server a cancel rather than only stopping + * the client from listening. + * + * mssql is absent: tedious exposes `request.cancel()`, but Kysely's `MssqlDialect` + * owns the `Request` object and never hands it out, and `KILL` would end the + * whole session, not the request. sqlite: see `SESSION_ID_SQL`. + */ +export const SERVER_CANCEL: Partial> = { + postgres: (db, sessionId) => sql`select pg_cancel_backend(${sessionId})`.execute(db), + // KILL cannot be prepared, so the id is interpolated. It comes from + // connection_id() and is checked by readSessionId before it gets here, + // never from user input. + mysql: (db, sessionId) => sql.raw(`kill query ${sessionId}`).execute(db), +}; + +/** + * Whether aborting a statement on this dialect actually stops work on the server. + * + * Callers use it to word the outcome: "cancelled" is only true where this + * returns true, and "stopped waiting" is the honest phrasing everywhere else. + * + * @example + * const message = hasServerSideCancel(dialect) + * ? 'Cancelled. The server was asked to stop the query.' + * : 'Stopped waiting. The query may still be running on the server.'; + */ +export function hasServerSideCancel(dialect: Dialect): boolean { + + return SERVER_CANCEL[dialect] !== undefined; + +} + +/** + * Read the server's session identifier out of a `SESSION_ID_SQL` result. + * + * Returns undefined for anything that is not a positive integer, which is what + * keeps the mysql `KILL` interpolation safe: the id is interpolated into a + * statement that cannot be prepared, so this is the only thing standing between + * a driver returning something unexpected and that string. + * + * @example + * readSessionId([{ id: '4711' }]) // => 4711 + * readSessionId([{ id: '4711; drop table users' }]) // => undefined + */ +export function readSessionId(rows: readonly { id?: unknown }[]): number | undefined { + + const raw = rows[0]?.id; + const id = Number(raw); + + if (!Number.isInteger(id) || id <= 0) return undefined; + + return id; + +} diff --git a/src/core/logger/formatter.ts b/src/core/logger/formatter.ts index 0899f688..70f22c59 100644 --- a/src/core/logger/formatter.ts +++ b/src/core/logger/formatter.ts @@ -21,6 +21,7 @@ const MESSAGE_TEMPLATES: Record) => strin ? `Executed ${d['filepath']} (${d['durationMs']}ms)` : `Failed ${d['filepath']}: ${d['error']}`, 'file:skip': (d) => `Skipped ${d['filepath']} (${d['reason']})`, + 'file:progress': (d) => `Still running ${d['filepath']} (${Math.round(Number(d['elapsedMs']) / 1000)}s)`, // Change 'change:start': (d) => diff --git a/src/core/observer.ts b/src/core/observer.ts index 0e56f0cb..350fccac 100644 --- a/src/core/observer.ts +++ b/src/core/observer.ts @@ -27,6 +27,7 @@ import type { TransferEvents } from './transfer/events.js'; import type { DtEvents } from './dt/events.js'; import type { LogLevel } from './logger/types.js'; import type { TruncateResult, TeardownResult } from './teardown/types.js'; +import type { StatementStatus } from './runner/statement-probes.js'; import { isDebug } from './environment.js'; /** @@ -66,6 +67,17 @@ export interface NoormEvents extends SettingsEvents, UpdateEvents, VaultEvents, error?: string; }; 'file:skip': { filepath: string; reason: 'unchanged' | 'already-run' }; + /** + * A file has run past the watch delay. Repeats every watch interval until + * it finishes. `status` is null when the dialect has no probe (sqlite) or + * the server could not be asked. + */ + 'file:progress': { + filepath: string; + elapsedMs: number; + sessionId: number | null; + status: StatementStatus | null; + }; 'file:dry-run': { filepath: string; status: 'success' | 'failed'; diff --git a/src/core/runner/index.ts b/src/core/runner/index.ts index 3cf69f40..f42d85ff 100644 --- a/src/core/runner/index.ts +++ b/src/core/runner/index.ts @@ -53,3 +53,6 @@ export type { } from './types.js'; export { DEFAULT_RUN_OPTIONS } from './types.js'; + +// Statement watcher +export type { OperationProgress } from './statement-probes.js'; diff --git a/src/core/runner/runner.ts b/src/core/runner/runner.ts index 33b85610..d28c2b9a 100644 --- a/src/core/runner/runner.ts +++ b/src/core/runner/runner.ts @@ -37,7 +37,11 @@ import { assertPolicy } from '../policy/index.js'; import type { Permission } from '../policy/index.js'; import { computeChecksum, computeChecksumFromContent, computeCombinedChecksum } from './checksum.js'; import { executeSqlBody } from './mssql-batches.js'; +import { StatementWatcher } from './statement-watcher.js'; import { Tracker } from './tracker.js'; +import { getSqlErrorMessage } from '../shared/index.js'; +import { OperationAbortedError } from '../shared/abort.js'; +import type { NoormDatabase } from '../shared/index.js'; import type { RunOptions, RunContext, @@ -248,12 +252,14 @@ export async function runFile( } - const result = await executeSingleFile(context, filepath, opts, tracker, operationId!); + const watcher = createWatcher(context); + const result = await executeSingleFile(context, filepath, opts, tracker, operationId!, watcher) + .finally(() => watcher.close()); // Finalize operation await tracker.finalizeOperation( operationId!, - result.status === 'failed' ? 'failed' : 'success', + result.status === 'failed' || result.error ? 'failed' : 'success', Math.round(result.durationMs ?? 0), result.checksum, result.error, @@ -589,6 +595,61 @@ const DEFAULT_RUN_OPTIONS_INTERNAL = { output: null as string | null, }; +/** Error and skip reason for a run stopped by `context.signal`. */ +const RUN_CANCELLED = 'Run cancelled'; + +function createWatcher(context: RunContext): StatementWatcher { + + return new StatementWatcher(context.db, { + dialect: context.dialect ?? 'postgres', + signal: context.signal, + }); + +} + +/** Returned by `runWatched` when the run was cancelled before the file's SQL was sent. */ +const NOT_STARTED = Symbol('not-started'); + +/** + * Execute a file's SQL through the watcher. + * + * The connection checkout sits outside `executeSqlBody`'s own error handling, + * so its failure is turned into the file's error here rather than escaping the + * batch unfinalized. A file the watcher refused to start is not a failure. + */ +async function runWatched( + context: RunContext, + watcher: StatementWatcher, + filepath: string, + sqlContent: string, +): Promise { + + const [execErrMsg, err] = await attempt(() => + watcher.run(filepath, context.db, (conn) => executeSqlBody({ ...context, db: conn }, sqlContent)), + ); + + if (err instanceof OperationAbortedError) return NOT_STARTED; + + return err ? getSqlErrorMessage(err) : execErrMsg; + +} + +async function skipRemaining(tracker: Tracker, operationId: number, reason: string): Promise { + + const skipErr = await tracker.skipRemainingFiles(operationId, reason.slice(0, 100)); + + if (skipErr) { + + observer.emit('error', { + source: 'runner:skip-remaining', + error: new Error(skipErr), + context: { operationId }, + }); + + } + +} + /** * Execute multiple files with tracking. * @@ -745,49 +806,76 @@ export async function executeFiles( // Execute files sequentially (concurrency is typically 1 for DDL safety) const results: FileResult[] = []; let failed = false; + let cancelled = false; + const watcher = createWatcher(context); - for (let i = 0; i < files.length; i++) { + try { - const file = files[i]!; - const fileRecord = fileRecords[i]!; + for (let i = 0; i < files.length; i++) { - const result = await executeSingleFileWithUpdate( - context, - file.path, - fileRecord.checksum, - opts, - tracker, - operationId!, - execOptions.changeType, - ); + const file = files[i]!; + const fileRecord = fileRecords[i]!; - results.push(result); + if (context.signal?.aborted) { - // Abort on error if configured - if (result.status === 'failed' && opts.abortOnError) { + failed = true; + cancelled = true; + await skipRemaining(tracker, operationId!, RUN_CANCELLED); - failed = true; + break; + + } - // Mark remaining files as skipped - const skipErr = await tracker.skipRemainingFiles( + const result = await executeSingleFileWithUpdate( + context, + file.path, + fileRecord.checksum, + opts, + tracker, operationId!, - `Skipped: failure in ${path.basename(file.path)}`.slice(0, 100), + execOptions.changeType, + watcher, ); - if (skipErr) { + // Refused before its SQL was sent: its pending row is skipped with the rest. + if (!result) { - observer.emit('error', { - source: 'runner:skip-remaining', - error: new Error(skipErr), - context: { operationId: operationId! }, - }); + failed = true; + cancelled = true; + await skipRemaining(tracker, operationId!, RUN_CANCELLED); + + break; } - break; + results.push(result); + + // A last file that finished despite the abort (mssql, sqlite, or a + // cancel that missed) completed the run; nothing was cut short. + cancelled = (context.signal?.aborted ?? false) + && (i < files.length - 1 || result.status === 'failed'); + + if (cancelled || (result.status === 'failed' && opts.abortOnError)) { + + failed = true; + + await skipRemaining( + tracker, + operationId!, + cancelled ? RUN_CANCELLED : `Skipped: failure in ${path.basename(file.path)}`, + ); + + break; + + } } + } + finally { + + await watcher.close(); + } // When abortOnError stops the loop early, fewer results than files is @@ -831,7 +919,7 @@ export async function executeFiles( finalStatus, Math.round(durationMs), combinedChecksum, - failed ? results.find((r) => r.status === 'failed')?.error : undefined, + cancelled ? RUN_CANCELLED : failed ? results.find((r) => r.status === 'failed')?.error : undefined, ); if (finalizeErr) { @@ -852,6 +940,7 @@ export async function executeFiles( filesFailed, durationMs, changeId: operationId, + error: cancelled ? RUN_CANCELLED : undefined, }; } @@ -909,7 +998,8 @@ async function executeSingleFileWithUpdate( tracker: Tracker, operationId: number, changeType: ChangeType, -): Promise { + watcher: StatementWatcher, +): Promise { const start = performance.now(); @@ -1004,7 +1094,9 @@ async function executeSingleFileWithUpdate( } // Execute SQL (MSSQL splits on `GO` batches; other dialects run as one) - const execErrMsg = await executeSqlBody(context, sqlContent); + const execErrMsg = await runWatched(context, watcher, filepath, sqlContent); + + if (execErrMsg === NOT_STARTED) return null; const durationMs = performance.now() - start; @@ -1080,6 +1172,7 @@ async function executeSingleFile( options: Required> & { output: string | null }, tracker: Tracker, operationId: number, + watcher: StatementWatcher, ): Promise { const start = performance.now(); @@ -1191,10 +1284,24 @@ async function executeSingleFile( } // Execute SQL (MSSQL splits on `GO` batches; other dialects run as one) - const execErrMsg = await executeSqlBody(context, sqlContent); + const execErrMsg = await runWatched(context, watcher, filepath, sqlContent); const durationMs = performance.now() - start; + if (execErrMsg === NOT_STARTED) { + + await tracker.recordExecution({ + changeId: operationId, + filepath: relFilepath, + checksum, + status: 'skipped', + skipReason: RUN_CANCELLED, + }); + + return { filepath, checksum, status: 'skipped', error: RUN_CANCELLED }; + + } + if (execErrMsg) { const error = execErrMsg + (await describePriorSuccesses(tracker, relFilepath, operationId)); diff --git a/src/core/runner/statement-probes.ts b/src/core/runner/statement-probes.ts new file mode 100644 index 00000000..b66d8d51 --- /dev/null +++ b/src/core/runner/statement-probes.ts @@ -0,0 +1,335 @@ +/** + * Per-dialect queries that describe one running statement from outside it. + * + * Every probe runs on the watcher's side connection against the session id of + * the connection executing the file. Each part of a probe (activity, blockers, + * progress) is attempted on its own: a missing view or a missing privilege + * drops that part of the report, never the whole report. + */ +import { sql } from 'kysely'; +import { attempt } from '@logosdx/utils'; +import type { QueryExecutorProvider } from 'kysely'; + +import type { Dialect } from '../connection/types.js'; + +/** + * A session holding a lock the watched statement is waiting on. + */ +export interface BlockingSession { + pid: number; + + /** Start of the blocker's statement text, or its command/status when no text is visible. */ + query: string | null; + + /** How long the blocker has been in its current transaction or request. */ + ageMs: number | null; +} + +/** + * Progress an operation reports about itself, from the dialect's progress views. + */ +export interface OperationProgress { + /** What is running, e.g. `CREATE INDEX`, `VACUUM`, `COPY FROM`. */ + operation: string; + + /** Phase within the operation, e.g. `building index: loading tuples`. */ + phase: string | null; + + relation: string | null; + + done: number | null; + total: number | null; + + /** 0-100, when the dialect reports enough to compute it. */ + percent: number | null; +} + +/** + * What the server says about a running statement at one point in time. + */ +export interface StatementStatus { + /** Session state, e.g. `active`, `suspended`, a mysql thread state. */ + state: string | null; + + /** What the session is waiting on, e.g. `Lock: transactionid`. Null when it is working. */ + waitEvent: string | null; + + blockedBy: BlockingSession[]; + + workers: number; + + progress: OperationProgress | null; +} + +type StatementProbe = (side: QueryExecutorProvider, sessionId: number) => Promise; + +/** + * Percent from a done/total pair, or null when the total is unknown. + * + * @example + * percentOf(4210, 9800) // => 42.96 + * percentOf(12, 0) // => null + */ +function percentOf(done: number | null, total: number | null): number | null { + + if (done === null || total === null || total <= 0) return null; + + return Math.min(100, (done / total) * 100); + +} + +function toNumber(value: unknown): number | null { + + if (value === null || value === undefined) return null; + + const n = Number(value); + + return Number.isFinite(n) ? n : null; + +} + +function toText(value: unknown): string | null { + + if (value === null || value === undefined || value === '') return null; + + return String(value); + +} + +async function rowsOrEmpty(query: () => Promise<{ rows: T[] }>): Promise { + + const [result, err] = await attempt(query); + + return err ? [] : result.rows; + +} + +interface PgActivityRow { + state: string | null; + wait_event_type: string | null; + wait_event: string | null; + blockers: number[] | null; + workers: number; +} + +interface BlockerRow { + pid: number; + query: string | null; + age_ms: number | null; +} + +interface PgProgressRow { + operation: string; + phase: string | null; + relation: string | null; + done: number | null; + total: number | null; +} + +function toBlockers(rows: BlockerRow[]): BlockingSession[] { + + return rows.map((row) => ({ + pid: Number(row.pid), + query: toText(row.query), + ageMs: toNumber(row.age_ms), + })); + +} + +/** + * Postgres: `pg_stat_activity`, `pg_blocking_pids()`, and the + * `pg_stat_progress_*` views. `leader_pid` needs Postgres 13+. + * + * For CREATE INDEX, tuples are preferred when reported; HNSW reports + * `tuples_total = 0`, so it falls back to blocks. + */ +const postgresProbe: StatementProbe = async (side, pid) => { + + const [activity] = await rowsOrEmpty(() => sql` + select a.state, a.wait_event_type, a.wait_event, + pg_blocking_pids(a.pid) as blockers, + (select count(*)::int from pg_stat_activity w + where w.leader_pid = a.pid and w.pid <> a.pid) as workers + from pg_stat_activity a + where a.pid = ${pid} + `.execute(side)); + + const blockerPids = (activity?.blockers ?? []).map(Number); + + const blockers = blockerPids.length === 0 ? [] : await rowsOrEmpty(() => sql` + select pid, left(query, 120) as query, + (extract(epoch from now() - coalesce(xact_start, query_start)) * 1000)::float8 as age_ms + from pg_stat_activity + where pid = any(${blockerPids}::int[]) + `.execute(side)); + + const [progress] = await rowsOrEmpty(() => sql` + select command as operation, phase, + nullif(relid, 0)::regclass::text as relation, + (case when tuples_total > 0 then tuples_done else blocks_done end)::float8 as done, + (case when tuples_total > 0 then tuples_total else blocks_total end)::float8 as total + from pg_stat_progress_create_index where pid = ${pid} + union all + select 'VACUUM', phase, nullif(relid, 0)::regclass::text, + heap_blks_scanned::float8, heap_blks_total::float8 + from pg_stat_progress_vacuum where pid = ${pid} + union all + select command, phase, nullif(relid, 0)::regclass::text, + heap_blks_scanned::float8, heap_blks_total::float8 + from pg_stat_progress_cluster where pid = ${pid} + union all + select command, null, nullif(relid, 0)::regclass::text, + bytes_processed::float8, nullif(bytes_total, 0)::float8 + from pg_stat_progress_copy where pid = ${pid} + union all + select 'ANALYZE', phase, nullif(relid, 0)::regclass::text, + sample_blks_scanned::float8, sample_blks_total::float8 + from pg_stat_progress_analyze where pid = ${pid} + limit 1 + `.execute(side)); + + const waitEvent = activity?.wait_event + ? `${activity.wait_event_type ?? 'Wait'}: ${activity.wait_event}` + : null; + + return { + state: toText(activity?.state), + waitEvent, + blockedBy: toBlockers(blockers), + workers: toNumber(activity?.workers) ?? 0, + progress: progress + ? { + operation: progress.operation, + phase: toText(progress.phase), + relation: toText(progress.relation), + done: toNumber(progress.done), + total: toNumber(progress.total), + percent: percentOf(toNumber(progress.done), toNumber(progress.total)), + } + : null, + }; + +}; + +interface MssqlRequestRow { + status: string | null; + wait_type: string | null; + blocking_session_id: number | null; + percent_complete: number | null; + command: string | null; +} + +/** + * MSSQL: `sys.dm_exec_requests`. Seeing another session's request needs + * `VIEW SERVER STATE` (`VIEW SERVER PERFORMANCE STATE` on 2022); without it + * the report is empty and only elapsed time is shown. + */ +const mssqlProbe: StatementProbe = async (side, spid) => { + + const [request] = await rowsOrEmpty(() => sql` + select status, wait_type, blocking_session_id, percent_complete, command + from sys.dm_exec_requests + where session_id = ${spid} + `.execute(side)); + + const blockerId = toNumber(request?.blocking_session_id); + + const blockers = !blockerId ? [] : await rowsOrEmpty(() => sql` + select s.session_id as pid, + coalesce(left(t.text, 120), r.command, s.status) as query, + datediff_big(millisecond, coalesce(r.start_time, s.last_request_start_time), sysdatetime()) as age_ms + from sys.dm_exec_sessions s + left join sys.dm_exec_requests r on r.session_id = s.session_id + outer apply sys.dm_exec_sql_text(r.sql_handle) t + where s.session_id = ${blockerId} + `.execute(side)); + + const percent = toNumber(request?.percent_complete); + + return { + state: toText(request?.status), + waitEvent: toText(request?.wait_type), + blockedBy: toBlockers(blockers), + workers: 0, + progress: request?.command && percent + ? { operation: request.command, phase: null, relation: null, done: null, total: null, percent } + : null, + }; + +}; + +interface MysqlProcessRow { + state: string | null; +} + +interface MysqlBlockerIdRow { + pid: number; +} + +interface MysqlStageRow { + event_name: string; + work_completed: number | null; + work_estimated: number | null; +} + +/** + * MySQL: `information_schema.processlist`, the `sys` schema's lock-wait views, + * and `performance_schema.events_stages_current`. The last two depend on + * privileges and instrumentation and drop out quietly without them. + */ +const mysqlProbe: StatementProbe = async (side, id) => { + + const [process] = await rowsOrEmpty(() => sql` + select state from information_schema.processlist where id = ${id} + `.execute(side)); + + const blockerIds = await rowsOrEmpty(() => sql` + select blocking_pid as pid from sys.innodb_lock_waits where waiting_pid = ${id} + union + select blocking_pid from sys.schema_table_lock_waits where waiting_pid = ${id} + `.execute(side)); + + const ids = blockerIds.map((row) => Number(row.pid)); + + const blockers = ids.length === 0 ? [] : await rowsOrEmpty(() => sql` + select id as pid, left(coalesce(info, command), 120) as query, time * 1000 as age_ms + from information_schema.processlist + where id in (${sql.join(ids)}) + `.execute(side)); + + const [stage] = await rowsOrEmpty(() => sql` + select s.event_name, s.work_completed, s.work_estimated + from performance_schema.events_stages_current s + join performance_schema.threads t on t.thread_id = s.thread_id + where t.processlist_id = ${id} + `.execute(side)); + + const state = toText(process?.state); + const done = toNumber(stage?.work_completed); + const total = toNumber(stage?.work_estimated); + + return { + state, + waitEvent: state?.startsWith('Waiting') ? state : null, + blockedBy: toBlockers(blockers), + workers: 0, + progress: stage + ? { + operation: stage.event_name.replace(/^stage\/\w+\//, ''), + phase: null, + relation: null, + done, + total, + percent: percentOf(done, total), + } + : null, + }; + +}; + +/** Probes by dialect; sqlite has none (see `SESSION_ID_SQL`). */ +export const STATEMENT_PROBES: Partial> = { + postgres: postgresProbe, + mssql: mssqlProbe, + mysql: mysqlProbe, +}; diff --git a/src/core/runner/statement-watcher.ts b/src/core/runner/statement-watcher.ts new file mode 100644 index 00000000..5a275520 --- /dev/null +++ b/src/core/runner/statement-watcher.ts @@ -0,0 +1,341 @@ +/** + * Statement watcher: `file:progress` reports and server-side cancel for the + * files of one run. Lifecycle and per-dialect behavior are in + * docs/dev/runner.md, "Long-Running Statements". + */ +import { sql } from 'kysely'; +import { attempt, runWithTimeout } from '@logosdx/utils'; +import type { Kysely } from 'kysely'; + +import { observer } from '../observer.js'; +import { SERVER_CANCEL, SESSION_ID_SQL, readSessionId } from '../connection/session.js'; +import { OperationAbortedError } from '../shared/abort.js'; +import { STATEMENT_PROBES } from './statement-probes.js'; +import type { Dialect } from '../connection/types.js'; +import type { StatementStatus } from './statement-probes.js'; + +const DEFAULT_WATCH_DELAY_MS = 10_000; +const DEFAULT_WATCH_INTERVAL_MS = 10_000; + +/** How long a report or cancel waits for the side connection before the watcher gives it up for the run. */ +const SIDE_CONNECTION_WAIT_MS = 5_000; + +interface StatementWatcherOptions { + /** Without a probe for the dialect, reports carry elapsed time only. */ + dialect?: Dialect; + + /** Aborting asks the server to cancel whatever statement is running. */ + signal?: AbortSignal; + + delayMs?: number; + intervalMs?: number; +} + +interface WatchedStatement { + filepath: string; + sessionId: number | null; + startedAt: number; + timer: ReturnType | undefined; + finished: boolean; + cancelling: Promise | null; +} + +/** + * Watches the statements of one run and cancels them on abort. + * + * One watcher per run: it owns the side connection, and `close()` must be + * called when the run ends to return that connection to the pool. + * + * @example + * const watcher = new StatementWatcher(context.db, { dialect, signal }); + * + * try { + * + * const err = await watcher.run(filepath, context.db, (conn) => + * executeSqlBody({ ...context, db: conn }, sqlContent)); + * + * } + * finally { + * + * await watcher.close(); + * + * } + */ +export class StatementWatcher { + + #pool: Kysely; + #dialect: Dialect | undefined; + #signal: AbortSignal | undefined; + #delayMs: number; + #intervalMs: number; + + #active: WatchedStatement | null = null; + #side: Promise | null> | null = null; + #sideAbandoned = false; + #releaseSide: (() => void) | null = null; + #sideHeld: Promise = Promise.resolve(); + #closed = false; + + #onAbort = (): void => { + + const statement = this.#active; + + if (statement) void this.#cancel(statement); + + }; + + /** + * @param pool - Root connection pool the side connection is checked out + * from. Never a transaction: the side connection must be a separate session. + */ + constructor(pool: Kysely, options: StatementWatcherOptions = {}) { + + this.#pool = pool; + this.#dialect = options.dialect; + this.#signal = options.signal; + this.#delayMs = options.delayMs ?? DEFAULT_WATCH_DELAY_MS; + this.#intervalMs = options.intervalMs ?? DEFAULT_WATCH_INTERVAL_MS; + + this.#signal?.addEventListener('abort', this.#onAbort, { once: true }); + + } + + /** + * Run `fn` on one pinned connection while watching it. + * + * A transaction is already one connection, so it is used as is; anything + * else is pinned with `connection()` so the session id read up front is + * the session the SQL runs on. + * + * @throws OperationAbortedError when the signal aborted before `fn` started + */ + async run(filepath: string, executor: Kysely, fn: (conn: Kysely) => Promise): Promise { + + if (executor.isTransaction) { + + return this.#watch(filepath, executor, fn); + + } + + return executor.connection().execute((conn) => this.#watch(filepath, conn, fn)); + + } + + /** + * Stop watching and return the side connection to the pool. + */ + async close(): Promise { + + this.#closed = true; + this.#signal?.removeEventListener('abort', this.#onAbort); + this.#releaseSide?.(); + + await this.#sideHeld; + + } + + async #watch(filepath: string, conn: Kysely, fn: (conn: Kysely) => Promise): Promise { + + const sessionId = await this.#readSessionId(conn); + const statement: WatchedStatement = { + filepath, sessionId, startedAt: performance.now(), timer: undefined, finished: false, cancelling: null, + }; + + // The abort listener fires once and finds nothing active before this + // point, so an abort during render, checkout, or the id read lands here. + if (this.#signal?.aborted) { + + throw new OperationAbortedError('Run cancelled before the file started'); + + } + + this.#active = statement; + statement.timer = setTimeout(() => void this.#report(statement), this.#delayMs); + + try { + + return await fn(conn); + + } + finally { + + clearTimeout(statement.timer); + this.#active = null; + statement.finished = true; + + // A cancel still in flight would otherwise land on this session + // after the pool hands it to the runner's next tracking query. + await statement.cancelling; + + } + + } + + async #readSessionId(conn: Kysely): Promise { + + const query = this.#dialect ? SESSION_ID_SQL[this.#dialect] : undefined; + + if (!query) return null; + + const [result, err] = await attempt(() => sql.raw<{ id?: unknown }>(query).execute(conn)); + + if (err) return null; + + return readSessionId(result.rows) ?? null; + + } + + async #report(statement: WatchedStatement): Promise { + + if (this.#active !== statement) return; + + const status = await this.#poll(statement); + + if (this.#active !== statement) return; + + observer.emit('file:progress', { + filepath: statement.filepath, + elapsedMs: performance.now() - statement.startedAt, + sessionId: statement.sessionId, + status, + }); + + statement.timer = setTimeout(() => void this.#report(statement), this.#intervalMs); + + } + + async #poll(statement: WatchedStatement): Promise { + + const probe = this.#dialect ? STATEMENT_PROBES[this.#dialect] : undefined; + const sessionId = statement.sessionId; + + if (!probe || sessionId === null) return null; + + const side = await this.#sideConnection(); + + if (!side) return null; + + const [status, err] = await attempt(() => probe(side, sessionId)); + + return err ? null : status; + + } + + async #cancel(statement: WatchedStatement): Promise { + + const cancel = this.#dialect ? SERVER_CANCEL[this.#dialect] : undefined; + const sessionId = statement.sessionId; + + if (sessionId === null || !cancel) return; + + const side = await this.#sideConnection(); + + if (statement.finished || this.#closed) return; + + if (!side) { + + observer.emit('error', { + source: 'runner:cancel', + error: new Error('No free connection to send the cancel from'), + context: { filepath: statement.filepath, sessionId }, + }); + + return; + + } + + // Only the send is awaited by #watch: waiting on the checkout too would + // hold the pinned connection a pool with no spare one needs to serve it. + const sending = attempt(() => cancel(side, sessionId)); + statement.cancelling = sending.then(() => undefined); + + const [, err] = await sending; + + if (err) { + + observer.emit('error', { + source: 'runner:cancel', + error: err, + context: { filepath: statement.filepath, sessionId }, + }); + + } + + } + + async #sideConnection(): Promise | null> { + + if (this.#closed || this.#sideAbandoned) return null; + + this.#side ??= this.#checkoutSide(); + + const pending = this.#side; + const [side] = await attempt(() => runWithTimeout(() => pending, { timeout: SIDE_CONNECTION_WAIT_MS })); + + // A checkout handed back for having no statement to watch resets + // `#side`; only a timeout or a failed checkout leaves it in place. + if (!side && this.#side === pending) this.#sideAbandoned = true; + + return side ?? null; + + } + + /** + * Check a connection out of the pool and hold it until `close()`. + * + * Kysely only lends a dedicated connection for the duration of a callback, + * so the callback parks on a promise that `close()` resolves. A checkout + * that arrives after the watcher gave up on it returns at once. + */ + #checkoutSide(): Promise | null> { + + return new Promise((resolveSide) => { + + const released = new Promise((release) => { + + this.#releaseSide = release; + + }); + + this.#sideHeld = attempt(() => this.#pool.connection().execute(async (side) => { + + if (this.#sideAbandoned || this.#closed) return; + + // With no spare connection in the pool, this checkout is only + // served by the file releasing its own, so it arrives with no + // statement running. Holding it then would starve the runner's + // tracking queries; hand it back and let the next slow file retry. + if (this.#active === null) { + + this.#side = null; + resolveSide(null); + + return; + + } + + resolveSide(side); + + await released; + + })).then(([, err]) => { + + if (err) { + + resolveSide(null); + + observer.emit('error', { + source: 'runner:watch', + error: err, + context: { operation: 'side-connection' }, + }); + + } + + }); + + }); + + } + +} diff --git a/src/core/runner/types.ts b/src/core/runner/types.ts index a5938d4d..a748d616 100644 --- a/src/core/runner/types.ts +++ b/src/core/runner/types.ts @@ -117,6 +117,13 @@ export interface RunContext { /** Global secrets for template context */ globalSecrets?: Record; + + /** + * Aborting stops the run: the server is asked to cancel the running + * statement (postgres, mysql), and no further file starts. On dialects + * without a server-side cancel the current file runs to completion first. + */ + signal?: AbortSignal; } // ───────────────────────────────────────────────────────────── diff --git a/src/core/sql-terminal/executor.ts b/src/core/sql-terminal/executor.ts index 5c5830ba..ff310b3f 100644 --- a/src/core/sql-terminal/executor.ts +++ b/src/core/sql-terminal/executor.ts @@ -11,9 +11,13 @@ import { observer } from '../observer.js'; import { assertPolicy, classifyStatements } from '../policy/index.js'; import { OperationAbortedError, raceAbort } from '../shared/abort.js'; import type { Channel, ConfigAccess, Permission, SqlClass } from '../policy/index.js'; +import { SERVER_CANCEL, SESSION_ID_SQL, hasServerSideCancel, readSessionId } from '../connection/session.js'; +import type { ServerCancel } from '../connection/session.js'; import type { Dialect } from '../connection/types.js'; import type { SqlExecutionResult } from './types.js'; +export { hasServerSideCancel }; + /** Maps a classified statement to the permission it's gated by. */ const CLASS_PERMISSION: Record = { read: 'sql:read', @@ -21,60 +25,6 @@ const CLASS_PERMISSION: Record = { ddl: 'sql:ddl', }; -/** - * How a dialect is told to stop a query that is already running. - * - * `sessionId` reads back the server's own identifier for the connection the - * query will run on; `cancel` is the statement that stops it, issued from a - * *different* connection because the first one is busy. - */ -interface ServerCancel { - sessionId: string; - cancel: (db: Kysely, sessionId: number) => Promise; -} - -/** - * Dialects where aborting sends the server a cancel rather than only stopping - * the client from listening. - * - * Absent by design: - * - **mssql**: tedious exposes `request.cancel()`, but Kysely's `MssqlDialect` - * owns the `Request` object and never hands it out, so there is nothing to - * call it on from here. - * - **sqlite**: in-process and single-connection; there is no second - * connection from which to interrupt the first. - */ -const SERVER_CANCEL: Partial> = { - postgres: { - sessionId: 'select pg_backend_pid() as id', - cancel: (db, sessionId) => sql`select pg_cancel_backend(${sessionId})`.execute(db), - }, - mysql: { - // KILL cannot be prepared, so the id is interpolated. It comes from - // connection_id() and is checked to be a positive integer before it - // gets here, never from user input. - sessionId: 'select connection_id() as id', - cancel: (db, sessionId) => sql.raw(`kill query ${sessionId}`).execute(db), - }, -}; - -/** - * Whether aborting a query on this dialect actually stops work on the server. - * - * Callers use it to word the outcome: "cancelled" is only true where this - * returns true, and "stopped waiting" is the honest phrasing everywhere else. - * - * @example - * const message = hasServerSideCancel(dialect) - * ? 'Cancelled. The server was asked to stop the query.' - * : 'Stopped waiting. The query may still be running on the server.'; - */ -export function hasServerSideCancel(dialect: Dialect): boolean { - - return SERVER_CANCEL[dialect] !== undefined; - -} - /** Message for an abort that reached the server. */ const SERVER_CANCEL_MESSAGE = 'Cancelled. The server was asked to stop the query.'; @@ -84,7 +34,7 @@ const STOPPED_WAITING_MESSAGE = 'Stopped waiting. The query may still be running /** * The honest one-line outcome of aborting a query on `dialect`. * - * Lives next to the strategy table so the UI cannot drift into claiming a + * Reads the shared cancel table so the UI cannot drift into claiming a * cancellation the dialect never had. A screen that reports an abort before * the executor returns uses this rather than wording its own. * @@ -122,32 +72,6 @@ interface CancelArming { armed: boolean; } -/** - * Read the server's session identifier out of a `sessionId` probe result. - * - * Returns undefined for anything that is not a positive integer, which is what - * keeps the mysql `KILL` interpolation safe: the id is interpolated into a - * statement that cannot be prepared, so this is the only thing standing between - * a driver returning something unexpected and that string. - * - * Exported for its own tests. It is a validator, and the value it rejects never - * occurs on a healthy connection, so nothing else can exercise it. - * - * @example - * readSessionId([{ id: '4711' }]) // => 4711 - * readSessionId([{ id: '4711; drop table users' }]) // => undefined - */ -export function readSessionId(rows: readonly { id?: unknown }[]): number | undefined { - - const raw = rows[0]?.id; - const id = Number(raw); - - if (!Number.isInteger(id) || id <= 0) return undefined; - - return id; - -} - /** * Run `query` on one pinned connection, asking the server to kill it on abort. * @@ -160,14 +84,15 @@ export function readSessionId(rows: readonly { id?: unknown }[]): number | undef async function runWithServerCancel( db: Kysely, query: string, - strategy: ServerCancel, + sessionIdSql: string, + cancel: ServerCancel, signal: AbortSignal, arming: CancelArming, ): Promise> { return db.connection().execute(async (pinned) => { - const probe = await sql.raw<{ id?: unknown }>(strategy.sessionId).execute(pinned); + const probe = await sql.raw<{ id?: unknown }>(sessionIdSql).execute(pinned); const sessionId = readSessionId(probe.rows); if (sessionId === undefined) { @@ -181,7 +106,7 @@ async function runWithServerCancel( // Fire and forget: the caller has already been handed back control // by raceAbort, and a cancel that cannot be delivered leaves the // outcome exactly where it would have been without one. - void attempt(() => strategy.cancel(db, sessionId)); + void attempt(() => cancel(db, sessionId)); }; @@ -226,15 +151,16 @@ function runQuery( if (!signal) return sql.raw(query).execute(db); - const strategy = dialect ? SERVER_CANCEL[dialect] : undefined; + const sessionIdSql = dialect ? SESSION_ID_SQL[dialect] : undefined; + const cancel = dialect ? SERVER_CANCEL[dialect] : undefined; - if (!strategy) { + if (!sessionIdSql || !cancel) { return raceAbort(sql.raw(query).execute(db), signal); } - return raceAbort(runWithServerCancel(db, query, strategy, signal, arming), signal); + return raceAbort(runWithServerCancel(db, query, sessionIdSql, cancel, signal, arming), signal); } diff --git a/src/tui/components/index.ts b/src/tui/components/index.ts index b53a8b86..6e8c1bce 100644 --- a/src/tui/components/index.ts +++ b/src/tui/components/index.ts @@ -56,12 +56,13 @@ export type { } from './dialogs/index.js'; // Status -export { LockStatus, ConnectionStatus, isDatabaseNotFoundError } from './status/index.js'; +export { LockStatus, ConnectionStatus, isDatabaseNotFoundError, StatementProgress } from './status/index.js'; export type { LockStatusProps, LockStatusType, ConnectionStatusProps, ConnectionStatusType, + StatementProgressProps, } from './status/index.js'; // Secrets diff --git a/src/tui/components/status/StatementProgress.tsx b/src/tui/components/status/StatementProgress.tsx new file mode 100644 index 00000000..077c4780 --- /dev/null +++ b/src/tui/components/status/StatementProgress.tsx @@ -0,0 +1,101 @@ +/** + * StatementProgress - what a long-running file is doing on the server. + * + * Renders the latest `file:progress` report: elapsed time, the operation's + * own progress when the server reports it, and the sessions it is waiting on. + * A blocked file and a slow file look the same without the second line. + * + * @example + * ```tsx + * {progress.statement && } + * ``` + */ +import { Box, Text } from 'ink'; + +import type { ReactElement } from 'react'; +import type { NoormEvents } from '../../../core/observer.js'; +import type { OperationProgress } from '../../../core/runner/index.js'; + +type StatementReport = NoormEvents['file:progress']; + +export interface StatementProgressProps { + report: StatementReport; +} + +/** + * `17m05s`, `1h02m`, `42s`. + */ +function formatElapsed(ms: number): string { + + const totalSeconds = Math.floor(ms / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) return `${hours}h${String(minutes).padStart(2, '0')}m`; + if (minutes > 0) return `${minutes}m${String(seconds).padStart(2, '0')}s`; + + return `${seconds}s`; + +} + +function describeProgress(progress: OperationProgress): string { + + const parts = [progress.relation ? `${progress.operation} on ${progress.relation}` : progress.operation]; + + if (progress.phase) parts.push(progress.phase); + + if (progress.done !== null && progress.total !== null) { + + const percent = progress.percent === null ? '' : ` (${Math.floor(progress.percent)}%)`; + parts.push(`${progress.done}/${progress.total}${percent}`); + + } + else if (progress.percent !== null) { + + parts.push(`${Math.floor(progress.percent)}%`); + + } + + return parts.join(' · '); + +} + +/** + * Status lines for the file that is currently running. + */ +export function StatementProgress({ report }: StatementProgressProps): ReactElement { + + const { status } = report; + const blocked = status?.blockedBy ?? []; + const idleDetail = status && !status.progress && blocked.length === 0 ? status.waitEvent ?? status.state : null; + + return ( + + + Running for + {formatElapsed(report.elapsedMs)} + {report.sessionId !== null && pid {report.sessionId}} + {idleDetail && {idleDetail}} + + + {status?.progress && ( + + {describeProgress(status.progress)} + {status.workers > 0 && {status.workers} parallel workers} + + )} + + {blocked.map((blocker) => ( + + Waiting on pid {blocker.pid} + {blocker.query && ` (${blocker.query.replace(/\s+/g, ' ')}`} + {blocker.query && blocker.ageMs !== null && `, ${formatElapsed(blocker.ageMs)}`} + {blocker.query && ')'} + {status?.waitEvent && ` · ${status.waitEvent}`} + + ))} + + ); + +} diff --git a/src/tui/components/status/index.ts b/src/tui/components/status/index.ts index 53d7e8b5..883bdaaf 100644 --- a/src/tui/components/status/index.ts +++ b/src/tui/components/status/index.ts @@ -3,6 +3,8 @@ */ export { LockStatus } from './LockStatus.js'; export { ConnectionStatus, isDatabaseNotFoundError } from './ConnectionStatus.js'; +export { StatementProgress } from './StatementProgress.js'; export type { LockStatusProps, LockStatusType } from './LockStatus.js'; export type { ConnectionStatusProps, ConnectionStatusType } from './ConnectionStatus.js'; +export type { StatementProgressProps } from './StatementProgress.js'; diff --git a/src/tui/hooks/index.ts b/src/tui/hooks/index.ts index e83d4984..0ae14f22 100644 --- a/src/tui/hooks/index.ts +++ b/src/tui/hooks/index.ts @@ -66,6 +66,7 @@ export { export { useAsyncEffect } from './useAsyncEffect.js'; export { useAbortableTask, type AbortableTask } from './useAbortableTask.js'; +export { useDoublePress, DOUBLE_PRESS_WINDOW_MS } from './useDoublePress.js'; export { useViewportRows, diff --git a/src/tui/hooks/useChangeProgress.ts b/src/tui/hooks/useChangeProgress.ts index 47e12cd8..2a049541 100644 --- a/src/tui/hooks/useChangeProgress.ts +++ b/src/tui/hooks/useChangeProgress.ts @@ -1,9 +1,9 @@ /** * Hook for tracking change execution progress via observer events. * - * Subscribes to change:start, change:complete, and change:file events, - * maintaining state for results list, current change name, progress - * counter, and per-file progress. + * Subscribes to change:start, change:complete, change:file, and file:progress + * events, maintaining state for results list, current change name, progress + * counter, per-file progress, and the running file's server report. * * @example * ```tsx @@ -15,6 +15,7 @@ import { useState, useCallback } from 'react'; import type { StatusListItem } from '../components/lists/index.js'; +import type { NoormEvents } from '../../core/observer.js'; import { useOnEvent } from './useObserver.js'; /** @@ -36,6 +37,9 @@ export interface ChangeProgressState { /** File-level progress within current change */ fileProgress: { current: number; total: number }; + /** Latest server report on the current file, once it has run past the watch delay */ + statement: NoormEvents['file:progress'] | null; + /** Reset state for a new batch with given total */ reset: (total: number) => void; } @@ -43,7 +47,7 @@ export interface ChangeProgressState { /** * Track change execution progress via observer events. * - * Subscribes to change:start, change:complete, and change:file, + * Subscribes to change:start, change:complete, change:file, and file:progress, * returning reactive state that updates as changes execute. * * @example @@ -66,6 +70,7 @@ export function useChangeProgress(): ChangeProgressState { const [progress, setProgress] = useState({ current: 0, total: 0 }); const [currentFile, setCurrentFile] = useState(''); const [fileProgress, setFileProgress] = useState({ current: 0, total: 0 }); + const [statement, setStatement] = useState(null); useOnEvent('change:start', (data) => { @@ -86,6 +91,7 @@ export function useChangeProgress(): ChangeProgressState { ]); setProgress((prev) => ({ ...prev, current: prev.current + 1 })); + setStatement(null); }, []); @@ -93,6 +99,13 @@ export function useChangeProgress(): ChangeProgressState { setCurrentFile(data.filepath); setFileProgress({ current: data.index + 1, total: data.total }); + setStatement(null); + + }, []); + + useOnEvent('file:progress', (data) => { + + setStatement(data); }, []); @@ -103,9 +116,10 @@ export function useChangeProgress(): ChangeProgressState { setProgress({ current: 0, total }); setCurrentFile(''); setFileProgress({ current: 0, total: 0 }); + setStatement(null); }, []); - return { results, currentChange, progress, currentFile, fileProgress, reset }; + return { results, currentChange, progress, currentFile, fileProgress, statement, reset }; } diff --git a/src/tui/hooks/useDoublePress.ts b/src/tui/hooks/useDoublePress.ts new file mode 100644 index 00000000..7169be50 --- /dev/null +++ b/src/tui/hooks/useDoublePress.ts @@ -0,0 +1,43 @@ +/** + * Arm-then-confirm for a key that does something hard to undo. + * + * The first press arms and returns false, so the screen can warn; a second + * press within `windowMs` returns true. A press after the window re-arms + * instead of confirming, so a stray key minutes later never acts alone. + * + * @example + * ```tsx + * const confirmCancel = useDoublePress(); + * + * if (key.escape && phase === 'running') { + * + * if (confirmCancel()) task.cancel(); + * else showToast({ message: 'Press Esc again to cancel', variant: 'warning' }); + * + * } + * ``` + */ +import { useCallback, useRef } from 'react'; + +/** Time a second press has to arrive in to confirm the first. */ +export const DOUBLE_PRESS_WINDOW_MS = 2000; + +/** + * Returns a press handler that reports whether this press confirms the last one. + */ +export function useDoublePress(windowMs = DOUBLE_PRESS_WINDOW_MS): () => boolean { + + const armedAt = useRef(null); + + return useCallback(() => { + + const now = Date.now(); + const confirmed = armedAt.current !== null && now - armedAt.current <= windowMs; + + armedAt.current = confirmed ? null : now; + + return confirmed; + + }, [windowMs]); + +} diff --git a/src/tui/hooks/useRunProgress.ts b/src/tui/hooks/useRunProgress.ts index e1ea5083..42622289 100644 --- a/src/tui/hooks/useRunProgress.ts +++ b/src/tui/hooks/useRunProgress.ts @@ -30,6 +30,7 @@ import { useState, useCallback } from 'react'; import { useOnEvent } from './useObserver.js'; +import type { NoormEvents } from '../../core/observer.js'; /** * Phase of the run operation. @@ -71,6 +72,9 @@ export interface RunProgressState { /** Currently executing file (or null if between files) */ currentFile: string | null; + /** Latest server report on the current file, once it has run past the watch delay */ + statement: NoormEvents['file:progress'] | null; + /** Total number of files to process */ filesTotal: number; @@ -105,6 +109,7 @@ export interface RunProgressState { const INITIAL_STATE: RunProgressState = { phase: 'idle', currentFile: null, + statement: null, filesTotal: 0, filesRun: 0, filesSkipped: 0, @@ -168,6 +173,20 @@ export function useRunProgress(): UseRunProgressReturn { setState((prev) => ({ ...prev, currentFile: data.filepath, + statement: null, + })); + + }, + [], + ); + + useOnEvent( + 'file:progress', + (data) => { + + setState((prev) => ({ + ...prev, + statement: data, })); }, @@ -195,6 +214,7 @@ export function useRunProgress(): UseRunProgressReturn { filesFailed: prev.filesFailed + (isSuccess ? 0 : 1), results: [...prev.results, result], currentFile: null, + statement: null, }; }); @@ -268,6 +288,7 @@ export function useRunProgress(): UseRunProgressReturn { status: data.status, durationMs: data.durationMs, currentFile: null, + statement: null, error: data.error ?? null, })); diff --git a/src/tui/screens/change/ChangeFFScreen.tsx b/src/tui/screens/change/ChangeFFScreen.tsx index f0dbb241..78f625a0 100644 --- a/src/tui/screens/change/ChangeFFScreen.tsx +++ b/src/tui/screens/change/ChangeFFScreen.tsx @@ -29,6 +29,7 @@ import { StatusMessage, SmartConfirm, StatusList, + StatementProgress, } from '../../components/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; import { useChangeProgress, useAsyncEffect } from '../../hooks/index.js'; @@ -63,7 +64,7 @@ export function ChangeFFScreen({ params: _params }: ScreenProps): ReactElement { const { activeConfig, activeConfigName, projectRoot, settings, stateManager, identity: cryptoIdentity, globalModes } = useAppContext(); const check = activeConfig ? checkConfigPolicy('user', activeConfig, 'change:ff') : null; - const { results, currentChange, progress, reset: resetProgress } = useChangeProgress(); + const { results, currentChange, progress, statement, reset: resetProgress } = useChangeProgress(); const [step, setStep] = useState('loading'); const [pendingChanges, setPendingChanges] = useState([]); @@ -311,6 +312,8 @@ export function ChangeFFScreen({ params: _params }: ScreenProps): ReactElement { {currentChange && ` - ${currentChange}`} + {statement && } + {results.length > 0 && ( diff --git a/src/tui/screens/change/ChangeNextScreen.tsx b/src/tui/screens/change/ChangeNextScreen.tsx index 4ad70ebb..ec556b6c 100644 --- a/src/tui/screens/change/ChangeNextScreen.tsx +++ b/src/tui/screens/change/ChangeNextScreen.tsx @@ -31,6 +31,7 @@ import { SmartConfirm, StatusList, TextInput, + StatementProgress, } from '../../components/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; import { useChangeProgress, useAsyncEffect } from '../../hooks/index.js'; @@ -68,7 +69,7 @@ export function ChangeNextScreen({ params }: ScreenProps): ReactElement { // Pre-fill count from params const initialCount = params.count ? parseInt(String(params.count), 10) : 1; - const { results, currentChange, progress, reset: resetProgress } = useChangeProgress(); + const { results, currentChange, progress, statement, reset: resetProgress } = useChangeProgress(); const [step, setStep] = useState('loading'); const [pendingChanges, setPendingChanges] = useState([]); @@ -366,6 +367,8 @@ export function ChangeNextScreen({ params }: ScreenProps): ReactElement { {currentChange && ` - ${currentChange}`} + {statement && } + {results.length > 0 && ( diff --git a/src/tui/screens/change/ChangeRevertScreen.tsx b/src/tui/screens/change/ChangeRevertScreen.tsx index 57fa262e..055f61bd 100644 --- a/src/tui/screens/change/ChangeRevertScreen.tsx +++ b/src/tui/screens/change/ChangeRevertScreen.tsx @@ -30,6 +30,7 @@ import { StatusMessage, SmartConfirm, MissingParamPanel, + StatementProgress, } from '../../components/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; import { useChangeProgress, useAsyncEffect } from '../../hooks/index.js'; @@ -63,7 +64,7 @@ export function ChangeRevertScreen({ params }: ScreenProps): ReactElement { const changeName = params.name; - const { currentFile, fileProgress } = useChangeProgress(); + const { currentFile, fileProgress, statement } = useChangeProgress(); const [step, setStep] = useState('loading'); const [change, setChange] = useState(null); @@ -297,6 +298,8 @@ export function ChangeRevertScreen({ params }: ScreenProps): ReactElement { {fileProgress.current}/{fileProgress.total} files {currentFile && ` - ${currentFile.split('/').pop()}`} + + {statement && } ); diff --git a/src/tui/screens/change/ChangeRewindScreen.tsx b/src/tui/screens/change/ChangeRewindScreen.tsx index 7aa4cad6..bb338f92 100644 --- a/src/tui/screens/change/ChangeRewindScreen.tsx +++ b/src/tui/screens/change/ChangeRewindScreen.tsx @@ -32,6 +32,7 @@ import { SmartConfirm, StatusList, TextInput, + StatementProgress, } from '../../components/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; import { useChangeProgress, useAsyncEffect } from '../../hooks/index.js'; @@ -69,7 +70,7 @@ export function ChangeRewindScreen({ params }: ScreenProps): ReactElement { // Pre-fill from params - can be count or change name const target = params.count ? String(params.count) : (params.name ?? ''); - const { results, currentChange, progress, reset: resetProgress } = useChangeProgress(); + const { results, currentChange, progress, statement, reset: resetProgress } = useChangeProgress(); const [step, setStep] = useState('loading'); const [appliedChanges, setAppliedChanges] = useState([]); @@ -434,6 +435,8 @@ export function ChangeRewindScreen({ params }: ScreenProps): ReactElement { {currentChange && ` - ${currentChange}`} + {statement && } + {results.length > 0 && ( diff --git a/src/tui/screens/change/ChangeRunScreen.tsx b/src/tui/screens/change/ChangeRunScreen.tsx index 90aaf5bf..3d45a23e 100644 --- a/src/tui/screens/change/ChangeRunScreen.tsx +++ b/src/tui/screens/change/ChangeRunScreen.tsx @@ -30,6 +30,7 @@ import { StatusMessage, SmartConfirm, MissingParamPanel, + StatementProgress, } from '../../components/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; import { useChangeProgress, useAsyncEffect } from '../../hooks/index.js'; @@ -64,7 +65,7 @@ export function ChangeRunScreen({ params }: ScreenProps): ReactElement { const changeName = params.name; - const { currentFile, fileProgress } = useChangeProgress(); + const { currentFile, fileProgress, statement } = useChangeProgress(); const [step, setStep] = useState('loading'); const [change, setChange] = useState(null); @@ -293,6 +294,8 @@ export function ChangeRunScreen({ params }: ScreenProps): ReactElement { {fileProgress.current}/{fileProgress.total} files {currentFile && ` - ${currentFile.split('/').pop()}`} + + {statement && } ); diff --git a/src/tui/screens/run/RunBuildScreen.tsx b/src/tui/screens/run/RunBuildScreen.tsx index 07c2b2f2..f921e37f 100644 --- a/src/tui/screens/run/RunBuildScreen.tsx +++ b/src/tui/screens/run/RunBuildScreen.tsx @@ -14,7 +14,7 @@ * noorm run build # Opens this screen * ``` */ -import { useState, useCallback } from 'react'; +import { useState, useCallback, useRef } from 'react'; import { Box, Text, useInput } from 'ink'; import { ProgressBar } from '@inkjs/ui'; import { join, relative } from 'path'; @@ -25,13 +25,20 @@ import type { ScreenProps } from '../../types.js'; import { useRouter } from '../../router.js'; import { useFocusScope } from '../../focus.js'; import { useSettings, useGlobalModes, useAppContext } from '../../app-context.js'; -import { Panel, Spinner, SmartConfirm, useToast } from '../../components/index.js'; -import { useRunProgress, useAsyncEffect } from '../../hooks/index.js'; +import { Panel, Spinner, SmartConfirm, StatementProgress, useToast } from '../../components/index.js'; +import { useRunProgress, useAsyncEffect, useDoublePress, DOUBLE_PRESS_WINDOW_MS } from '../../hooks/index.js'; import { getEffectiveBuildPaths } from '../../../core/settings/rules.js'; import { discoverFiles, runBuild } from '../../../core/runner/index.js'; import { filterFilesByPaths, findUnmatchedIncludePatterns } from '../../../core/shared/index.js'; import { checkConfigPolicy } from '../../../core/policy/index.js'; -import { getErrorMessage, resolveScreenIdentity, buildRunContext, withScreenConnection, progressPercentage } from '../../utils/index.js'; +import { + getErrorMessage, + resolveScreenIdentity, + buildRunContext, + withScreenConnection, + progressPercentage, + runCancelMessage, +} from '../../utils/index.js'; import { attempt } from '@logosdx/utils'; type Phase = 'loading' | 'confirm' | 'running' | 'complete' | 'error'; @@ -49,8 +56,11 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { const check = activeConfig ? checkConfigPolicy('user', activeConfig, 'run:build') : null; const { showToast } = useToast(); const { state: progress, reset: resetProgress } = useRunProgress(); + const runController = useRef(null); + const confirmCancel = useDoublePress(); const [phase, setPhase] = useState('loading'); + const [cancelling, setCancelling] = useState(false); const [files, setFiles] = useState([]); const [sqlPath, setSqlPath] = useState(''); const [error, setError] = useState(null); @@ -123,8 +133,11 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { if (!activeConfig || !activeConfigName || !stateManager) return; setPhase('running'); + setCancelling(false); resetProgress(files.length); + const controller = new AbortController(); + runController.current = controller; const projectRoot = process.cwd(); // Resolve identity @@ -148,7 +161,7 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { }; // Run build with filtered files - await runBuild(context, sqlPath, options, files); + await runBuild({ ...context, signal: controller.signal }, sqlPath, options, files); }, ); @@ -184,7 +197,18 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { if (phase === 'running') { - showToast({ message: 'Cannot cancel running build', variant: 'warning' }); + if (cancelling) return; + + if (!confirmCancel()) { + + showToast({ message: 'Press Esc again to cancel the build', variant: 'warning', duration: DOUBLE_PRESS_WINDOW_MS }); + + return; + + } + + runController.current?.abort(); + setCancelling(true); return; @@ -389,6 +413,8 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { )} + {progress.statement && } + @@ -411,6 +437,12 @@ export function RunBuildScreen({ params: _params }: ScreenProps): ReactElement { + + + {cancelling + ? {runCancelMessage(activeConfig.connection.dialect)} + : [Esc Esc] Cancel} + ); diff --git a/src/tui/screens/run/RunDirScreen.tsx b/src/tui/screens/run/RunDirScreen.tsx index 5a271f13..7364b245 100644 --- a/src/tui/screens/run/RunDirScreen.tsx +++ b/src/tui/screens/run/RunDirScreen.tsx @@ -19,7 +19,7 @@ import type { ScreenProps } from '../../types.js'; import { useRouter } from '../../router.js'; import { useSettings, useGlobalModes, useAppContext } from '../../app-context.js'; -import { Panel, Spinner, Confirm, SelectList, FilePicker, KeyHandler, useToast } from '../../components/index.js'; +import { Panel, Spinner, Confirm, SelectList, FilePicker, KeyHandler, StatementProgress, useToast } from '../../components/index.js'; import { useRunProgress, useAsyncEffect, modeBannerRows } from '../../hooks/index.js'; import { discoverFiles, runFiles, checkFilesStatus } from '../../../core/runner/index.js'; import type { FilesStatusResult } from '../../../core/runner/index.js'; @@ -702,6 +702,8 @@ export function RunDirScreen({ params }: ScreenProps): ReactElement { {progress.currentFile.split('/').pop()} )} + {progress.statement && } + diff --git a/src/tui/screens/run/RunExecScreen.tsx b/src/tui/screens/run/RunExecScreen.tsx index 835f8b2a..c74bd1cf 100644 --- a/src/tui/screens/run/RunExecScreen.tsx +++ b/src/tui/screens/run/RunExecScreen.tsx @@ -15,7 +15,7 @@ * noorm run exec # Opens this screen * ``` */ -import { useState, useCallback } from 'react'; +import { useState, useCallback, useRef } from 'react'; import { Box, Text } from 'ink'; import { ProgressBar } from '@inkjs/ui'; import { join, relative } from 'path'; @@ -25,10 +25,26 @@ import type { ScreenProps } from '../../types.js'; import { useRouter } from '../../router.js'; import { useSettings, useGlobalModes, useAppContext } from '../../app-context.js'; -import { Panel, Spinner, SelectList, type SelectListItem, Confirm, KeyHandler, useToast } from '../../components/index.js'; -import { useRunProgress, useAsyncEffect, modeBannerRows } from '../../hooks/index.js'; +import { + Panel, + Spinner, + SelectList, + type SelectListItem, + Confirm, + KeyHandler, + StatementProgress, + useToast, +} from '../../components/index.js'; +import { useRunProgress, useAsyncEffect, useDoublePress, DOUBLE_PRESS_WINDOW_MS, modeBannerRows } from '../../hooks/index.js'; import { discoverFiles, runFiles } from '../../../core/runner/index.js'; -import { getErrorMessage, resolveScreenIdentity, buildRunContext, withScreenConnection, progressPercentage } from '../../utils/index.js'; +import { + getErrorMessage, + resolveScreenIdentity, + buildRunContext, + withScreenConnection, + progressPercentage, + runCancelMessage, +} from '../../utils/index.js'; import { attempt } from '@logosdx/utils'; type Phase = 'loading' | 'picker' | 'confirm' | 'running' | 'complete' | 'error'; @@ -45,8 +61,11 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { const globalModes = useGlobalModes(); const { showToast } = useToast(); const { state: progress, reset: resetProgress } = useRunProgress(); + const runController = useRef(null); + const confirmCancel = useDoublePress(); const [phase, setPhase] = useState('loading'); + const [cancelling, setCancelling] = useState(false); const [allFiles, setAllFiles] = useState([]); const [selectedFiles, setSelectedFiles] = useState>(new Set()); const [error, setError] = useState(null); @@ -112,8 +131,12 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { const filesToRun = Array.from(selectedFiles); setPhase('running'); + setCancelling(false); resetProgress(filesToRun.length); + const controller = new AbortController(); + runController.current = controller; + // Resolve identity const identity = resolveScreenIdentity(cryptoIdentity); @@ -133,7 +156,7 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { }; // Run all files as a single batch operation - await runFiles(context, filesToRun, options); + await runFiles({ ...context, signal: controller.signal }, filesToRun, options); }, ); @@ -147,9 +170,32 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { } + if (controller.signal.aborted) { + + showToast({ message: 'Run cancelled', variant: 'warning' }); + + } + setPhase('complete'); - }, [activeConfig, activeConfigName, stateManager, cryptoIdentity, selectedFiles, globalModes, resetProgress, projectRoot]); + }, [activeConfig, activeConfigName, stateManager, cryptoIdentity, selectedFiles, globalModes, resetProgress, projectRoot, showToast]); + + const requestCancel = useCallback(() => { + + if (cancelling) return; + + if (!confirmCancel()) { + + showToast({ message: 'Press Esc again to cancel the run', variant: 'warning', duration: DOUBLE_PRESS_WINDOW_MS }); + + return; + + } + + runController.current?.abort(); + setCancelling(true); + + }, [cancelling, confirmCancel, showToast]); // Submit selection (Enter in picker) const handleSubmit = useCallback(() => { @@ -320,16 +366,15 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { return ( - showToast({ message: 'Cannot cancel running files', variant: 'warning' })} - /> + {progress.currentFile && ( {progress.currentFile.split('/').pop()} )} + {progress.statement && } + @@ -341,6 +386,12 @@ export function RunExecScreen({ params: _params }: ScreenProps): ReactElement { + + + {cancelling + ? {runCancelMessage(activeConfig.connection.dialect)} + : [Esc Esc] Cancel} + ); diff --git a/src/tui/screens/run/RunFileScreen.tsx b/src/tui/screens/run/RunFileScreen.tsx index 4ac9b24f..dd665c6a 100644 --- a/src/tui/screens/run/RunFileScreen.tsx +++ b/src/tui/screens/run/RunFileScreen.tsx @@ -18,7 +18,7 @@ import type { ScreenProps } from '../../types.js'; import { useRouter } from '../../router.js'; import { useSettings, useGlobalModes, useAppContext } from '../../app-context.js'; -import { Panel, Spinner, Confirm, SearchableList, KeyHandler, useToast } from '../../components/index.js'; +import { Panel, Spinner, Confirm, SearchableList, KeyHandler, StatementProgress, useToast } from '../../components/index.js'; import { useRunProgress, useAsyncEffect, modeBannerRows } from '../../hooks/index.js'; import { discoverFiles, runFile, checkFilesStatus } from '../../../core/runner/index.js'; import type { FilesStatusResult } from '../../../core/runner/index.js'; @@ -536,7 +536,10 @@ export function RunFileScreen({ params }: ScreenProps): ReactElement { onCancel={cancelExecution} /> - + + + {progress.statement && } + [Esc] Cancel diff --git a/src/tui/utils/index.ts b/src/tui/utils/index.ts index 128cb7ff..3472ffe3 100644 --- a/src/tui/utils/index.ts +++ b/src/tui/utils/index.ts @@ -6,7 +6,7 @@ export { toKebabCase, oneLine } from './string.js'; export { resolveChangesDir, resolveSqlDir } from './paths.js'; export { resolveScreenIdentity } from './identity.js'; export { createChangeManager, type CreateChangeManagerOptions } from './change-context.js'; -export { buildRunContext, type BuildRunContextOptions } from './run-context.js'; +export { buildRunContext, runCancelMessage, type BuildRunContextOptions } from './run-context.js'; export { withScreenConnection, STOPPED_WAITING_MESSAGE } from './connection.js'; export { progressPercentage } from './progress.js'; export { diff --git a/src/tui/utils/run-context.ts b/src/tui/utils/run-context.ts index 8dac011f..64e7c7ae 100644 --- a/src/tui/utils/run-context.ts +++ b/src/tui/utils/run-context.ts @@ -22,6 +22,7 @@ import { loadIdentityMetadata, loadPrivateKey } from '../../core/identity/storag import type { Config } from '../../core/config/types.js'; import type { StateManager } from '../../core/state/index.js'; import { getVaultKey, buildSecretsContext } from '../../core/vault/index.js'; +import { hasServerSideCancel } from '../../core/connection/session.js'; /** * Options for building a RunContext. @@ -95,3 +96,21 @@ export async function buildRunContext(options: BuildRunContextOptions): Promise< }; } + +/** + * What a run screen says after the user confirms a cancel. + * + * Postgres and mysql get a server-side cancel, so the running statement stops + * now. Elsewhere the runner can only decline to start the next file, and + * saying "cancelling" would claim a stop that has not happened. + * + * @example + * {runCancelMessage(config.connection.dialect)} + */ +export function runCancelMessage(dialect: Dialect): string { + + return hasServerSideCancel(dialect) + ? 'Cancelling. The server was asked to stop the running statement.' + : 'Stopping after the current file finishes.'; + +} diff --git a/tests/core/runner/statement-watcher.test.ts b/tests/core/runner/statement-watcher.test.ts new file mode 100644 index 00000000..c81556d4 --- /dev/null +++ b/tests/core/runner/statement-watcher.test.ts @@ -0,0 +1,243 @@ +/** + * Statement watcher and run cancellation, on sqlite. + * + * sqlite has no server to ask, so these pin the dialect-independent contract: + * a file that runs past the delay reports elapsed time on every interval, a + * fast file reports nothing, reports stop the moment the file ends, and an + * aborted run starts no further files. The server-side half (probes, blockers, + * pg_cancel_backend) is covered in tests/integration/runner/statement-watcher.test.ts. + */ +import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Kysely, SqliteDialect, sql } from 'kysely'; +import { BunSqliteDatabase } from '../../../src/core/connection/dialects/sqlite-bun.js'; + +import { observer } from '../../../src/core/observer.js'; +import { OperationAbortedError } from '../../../src/core/shared/abort.js'; +import { runFiles } from '../../../src/core/runner/runner.js'; +import { StatementWatcher } from '../../../src/core/runner/statement-watcher.js'; +import { v1 } from '../../../src/core/version/schema/migrations/v1.js'; +import type { NoormDatabase } from '../../../src/core/shared/index.js'; +import type { NoormEvents } from '../../../src/core/observer.js'; +import type { RunContext } from '../../../src/core/runner/types.js'; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +describe('runner: statement watcher', () => { + + let db: Kysely; + let reports: NoormEvents['file:progress'][]; + let stopListening: () => void; + + beforeEach(() => { + + db = new Kysely({ + dialect: new SqliteDialect({ database: new BunSqliteDatabase(':memory:') as never }), + }); + + reports = []; + stopListening = observer.on('file:progress', (data) => { + + reports.push(data); + + }); + + }); + + afterEach(async () => { + + stopListening(); + await db.destroy(); + + }); + + it('should report a statement that outlives the delay, once per interval, with elapsed time', async () => { + + const watcher = new StatementWatcher(db, { dialect: 'sqlite', delayMs: 30, intervalMs: 40 }); + + await watcher.run('slow.sql', db, async () => sleep(160)); + await watcher.close(); + + expect(reports.length).toBeGreaterThanOrEqual(2); + expect(reports.every((r) => r.filepath === 'slow.sql')).toBe(true); + expect(reports[0]!.elapsedMs).toBeGreaterThanOrEqual(30); + expect(reports[1]!.elapsedMs).toBeGreaterThan(reports[0]!.elapsedMs); + + expect(reports[0]!.sessionId).toBeNull(); + expect(reports[0]!.status).toBeNull(); + + }); + + it('should stay quiet for a statement that finishes before the delay', async () => { + + const watcher = new StatementWatcher(db, { dialect: 'sqlite', delayMs: 50, intervalMs: 50 }); + + await watcher.run('fast.sql', db, async () => sleep(5)); + await sleep(120); + await watcher.close(); + + expect(reports).toHaveLength(0); + + }); + + it('should stop reporting as soon as the statement ends', async () => { + + const watcher = new StatementWatcher(db, { dialect: 'sqlite', delayMs: 20, intervalMs: 20 }); + + await watcher.run('slow.sql', db, async () => sleep(70)); + const countAtEnd = reports.length; + + await sleep(100); + await watcher.close(); + + expect(reports.length).toBe(countAtEnd); + + }); + + it('should refuse to start a file once the signal has aborted', async () => { + + const controller = new AbortController(); + const watcher = new StatementWatcher(db, { dialect: 'sqlite', signal: controller.signal }); + let started = false; + + controller.abort(); + + const outcome = await watcher.run('late.sql', db, async () => { + + started = true; + + }).then(() => null, (err: Error) => err); + await watcher.close(); + + // The abort listener has nothing to cancel before a file is active, so + // without this check the file would run to completion after a cancel. + expect(outcome).toBeInstanceOf(OperationAbortedError); + expect(started).toBe(false); + + }); + + it('should run the SQL on the connection it hands out and return its result', async () => { + + const watcher = new StatementWatcher(db, { dialect: 'sqlite' }); + + const result = await watcher.run('one.sql', db, (conn) => sql<{ n: number }>`select 1 as n`.execute(conn)); + await watcher.close(); + + expect(result.rows[0]?.n).toBe(1); + + }); + +}); + +describe('runner: run cancellation', () => { + + let db: Kysely; + let tempDir: string; + let files: string[]; + + function buildContext(signal: AbortSignal): RunContext { + + return { + db, + configName: 'test', + identity: { name: 'Test User', email: 'test@example.com', source: 'config' }, + projectRoot: tempDir, + access: { user: 'admin', agent: 'admin' }, + channel: 'user', + dialect: 'sqlite', + signal, + }; + + } + + async function tableExists(name: string): Promise { + + const rows = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${name} + `.execute(db); + + return rows.rows.length > 0; + + } + + beforeEach(async () => { + + tempDir = await mkdtemp(join(tmpdir(), 'noorm-runner-cancel-test-')); + await mkdir(join(tempDir, 'sql'), { recursive: true }); + + db = new Kysely({ + dialect: new SqliteDialect({ database: new BunSqliteDatabase(':memory:') as never }), + }); + + await v1.up(db as Kysely, 'sqlite'); + + files = [join(tempDir, 'sql', '001_a.sql'), join(tempDir, 'sql', '002_b.sql')]; + await writeFile(files[0]!, 'CREATE TABLE cancel_a (id INTEGER PRIMARY KEY);\n', 'utf-8'); + await writeFile(files[1]!, 'CREATE TABLE cancel_b (id INTEGER PRIMARY KEY);\n', 'utf-8'); + + }); + + afterEach(async () => { + + await db.destroy(); + await rm(tempDir, { recursive: true, force: true }); + + }); + + it('should start no file when the run is cancelled before it begins', async () => { + + const controller = new AbortController(); + controller.abort(); + + const result = await runFiles(buildContext(controller.signal), files); + + expect(result.status).toBe('failed'); + expect(result.error).toBe('Run cancelled'); + expect(result.filesRun).toBe(0); + expect(await tableExists('cancel_a')).toBe(false); + + }); + + it('should not start the next file once cancelled between files', async () => { + + const controller = new AbortController(); + const stop = observer.on('file:after', ({ filepath }) => { + + if (filepath === files[0]) controller.abort(); + + }); + + const result = await runFiles(buildContext(controller.signal), files); + stop(); + + expect(await tableExists('cancel_a')).toBe(true); + expect(await tableExists('cancel_b')).toBe(false); + expect(result.error).toBe('Run cancelled'); + expect(result.status).toBe('partial'); + + }); + + it('should not run a file whose cancel landed after it was picked up but before it started', async () => { + + const controller = new AbortController(); + const stop = observer.on('file:before', ({ filepath }) => { + + if (filepath === files[1]) controller.abort(); + + }); + + const result = await runFiles(buildContext(controller.signal), files); + stop(); + + // file:before fires before the SQL is sent, when no statement is active + // for the abort listener to cancel. + expect(await tableExists('cancel_b')).toBe(false); + expect(result.error).toBe('Run cancelled'); + expect(result.files).toHaveLength(1); + expect(result.filesFailed).toBe(0); + + }); + +}); diff --git a/tests/core/sql-terminal/executor-abort.test.ts b/tests/core/sql-terminal/executor-abort.test.ts index c105b2fb..94935fcc 100644 --- a/tests/core/sql-terminal/executor-abort.test.ts +++ b/tests/core/sql-terminal/executor-abort.test.ts @@ -17,8 +17,8 @@ import { executeRawSql, executeRawSqlUnchecked, hasServerSideCancel, - readSessionId, } from '../../../src/core/sql-terminal/executor.js'; +import { readSessionId } from '../../../src/core/connection/session.js'; import { DEFAULT_ACCESS } from '../../../src/core/policy/index.js'; import type { ConnectionResult } from '../../../src/core/connection/types.js'; diff --git a/tests/integration/runner/statement-watcher.test.ts b/tests/integration/runner/statement-watcher.test.ts new file mode 100644 index 00000000..571a50c1 --- /dev/null +++ b/tests/integration/runner/statement-watcher.test.ts @@ -0,0 +1,327 @@ +/** + * Statement watcher against live servers. + * + * The claims only a real server can check: the report names the session a + * blocked file is waiting on, it carries the progress the server publishes for + * the running command, and aborting the run's signal stops the statement on + * the server rather than only on the client. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test'; +import { sql } from 'kysely'; +import type { Kysely } from 'kysely'; + +import { observer } from '../../../src/core/observer.js'; +import { StatementWatcher } from '../../../src/core/runner/statement-watcher.js'; +import { createConnection } from '../../../src/core/connection/factory.js'; +import { TEST_CONNECTIONS, assertTestDatabase, createTestConnection, skipIfNoContainer } from '../../utils/db.js'; +import type { NoormEvents } from '../../../src/core/observer.js'; + +type Report = NoormEvents['file:progress']; + +const WATCH = { delayMs: 100, intervalMs: 150 }; + +/** Resolve with the first report matching `predicate`, or null after `timeoutMs`. */ +async function waitForReport(reports: Report[], predicate: (r: Report) => boolean, timeoutMs = 8_000): Promise { + + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + + const match = reports.find(predicate); + + if (match) return match; + + await new Promise((r) => setTimeout(r, 50)); + + } + + return null; + +} + +/** + * Hold `setup` open on its own connection until the returned release is called. + */ +function holdSession(db: Kysely, setup: (conn: Kysely) => Promise) { + + let release!: () => void; + const released = new Promise((r) => { + + release = r; + + }); + + let ready!: (pid: number) => void; + const pid = new Promise((r) => { + + ready = r; + + }); + + const done = db.connection().execute(async (conn) => { + + const result = await sql<{ id: number }>`select pg_backend_pid() as id`.execute(conn); + await setup(conn); + ready(Number(result.rows[0]!.id)); + await released; + await sql`rollback`.execute(conn); + + }); + + return { pid, release, done }; + +} + +describe('integration: postgres statement watcher', () => { + + let db: Kysely; + let destroy: () => Promise; + let reports: Report[]; + let stopListening: () => void; + + beforeAll(async () => { + + await skipIfNoContainer('postgres'); + + const conn = await createTestConnection('postgres'); + db = conn.db; + destroy = conn.destroy; + + await sql`drop table if exists watcher_items`.execute(db); + await sql`create table watcher_items (id int)`.execute(db); + + }); + + afterAll(async () => { + + if (!db) return; + + await sql`drop table if exists watcher_items`.execute(db); + await destroy(); + + }); + + beforeEach(() => { + + reports = []; + stopListening = observer.on('file:progress', (data) => { + + reports.push(data); + + }); + + }); + + afterEach(() => stopListening()); + + it('should name the session a blocked file is waiting on', async () => { + + const holder = holdSession(db, async (conn) => { + + await sql`begin`.execute(conn); + await sql`lock table watcher_items in access exclusive mode`.execute(conn); + + }); + const holderPid = await holder.pid; + + const watcher = new StatementWatcher(db, { dialect: 'postgres', ...WATCH }); + const pending = watcher.run('blocked.sql', db, (conn) => sql`select count(*) from watcher_items`.execute(conn)); + + const report = await waitForReport(reports, (r) => (r.status?.blockedBy.length ?? 0) > 0); + + holder.release(); + await holder.done; + await pending; + await watcher.close(); + + expect(report).not.toBeNull(); + expect(report!.sessionId).not.toBeNull(); + expect(report!.status!.blockedBy.map((b) => b.pid)).toContain(holderPid); + expect(report!.status!.waitEvent).toContain('Lock'); + + }, 20_000); + + it('should carry the progress postgres reports for CREATE INDEX', async () => { + + // CONCURRENTLY waits for open writers inside its own progress phase, + // which holds the build still long enough to observe. + const holder = holdSession(db, async (conn) => { + + await sql`begin`.execute(conn); + await sql`insert into watcher_items values (1)`.execute(conn); + + }); + await holder.pid; + + const watcher = new StatementWatcher(db, { dialect: 'postgres', ...WATCH }); + const pending = watcher.run('index.sql', db, (conn) => + sql.raw('create index concurrently watcher_items_id on watcher_items (id)').execute(conn)); + + const report = await waitForReport(reports, (r) => r.status?.progress !== null && r.status?.progress !== undefined); + + holder.release(); + await holder.done; + await pending; + await watcher.close(); + await sql`drop index if exists watcher_items_id`.execute(db); + + expect(report).not.toBeNull(); + expect(report!.status!.progress!.operation).toBe('CREATE INDEX CONCURRENTLY'); + expect(report!.status!.progress!.relation).toBe('watcher_items'); + expect(report!.status!.progress!.phase).toContain('waiting'); + + }, 20_000); + + it('should leave a single-connection pool usable after a slow file', async () => { + + const config = { ...TEST_CONNECTIONS.postgres, pool: { max: 1 } }; + assertTestDatabase(config); + const single = await createConnection(config, '__test_postgres_max1__'); + + const watcher = new StatementWatcher(single.db, { dialect: 'postgres', ...WATCH }); + + // Longer than the side-connection wait, so the watcher gives up on a + // checkout that can only be served once the file releases the pool. + await watcher.run('slow.sql', single.db, (conn) => sql`select pg_sleep(6)`.execute(conn)); + + const started = Date.now(); + const followUp = await sql<{ n: number }>`select 1 as n`.execute(single.db); + const waitedMs = Date.now() - started; + + await watcher.close(); + await single.destroy(); + + expect(followUp.rows[0]?.n).toBe(1); + expect(waitedMs).toBeLessThan(2_000); + + }, 30_000); + + it('should hand a single-connection pool back when the file ends inside the checkout wait', async () => { + + const config = { ...TEST_CONNECTIONS.postgres, pool: { max: 1 } }; + assertTestDatabase(config); + const single = await createConnection(config, '__test_postgres_max1_short__'); + + const watcher = new StatementWatcher(single.db, { dialect: 'postgres', ...WATCH }); + + // The side checkout queues behind this file and is served the moment + // the file releases the pool, well inside the 5s wait. + await watcher.run('short.sql', single.db, (conn) => sql`select pg_sleep(2)`.execute(conn)); + + const started = Date.now(); + const followUp = await sql<{ n: number }>`select 1 as n`.execute(single.db); + const waitedMs = Date.now() - started; + + await watcher.close(); + await single.destroy(); + + expect(followUp.rows[0]?.n).toBe(1); + expect(waitedMs).toBeLessThan(2_000); + + }, 30_000); + + it('should not hold a single-connection pool waiting for a cancel that cannot be sent', async () => { + + const config = { ...TEST_CONNECTIONS.postgres, pool: { max: 1 } }; + assertTestDatabase(config); + const single = await createConnection(config, '__test_postgres_max1_cancel__'); + + const controller = new AbortController(); + const watcher = new StatementWatcher(single.db, { dialect: 'postgres', signal: controller.signal, ...WATCH }); + const started = Date.now(); + + const pending = watcher.run('short.sql', single.db, (conn) => sql`select pg_sleep(1)`.execute(conn)); + + await new Promise((r) => setTimeout(r, 500)); + controller.abort(); + + await pending; + const elapsed = Date.now() - started; + + await watcher.close(); + await single.destroy(); + + // The side checkout can only be served by this file's own connection, + // so the cancel is never sent and the file must not wait out the 5s. + expect(elapsed).toBeLessThan(2_500); + + }, 30_000); + + it('should stop the running statement on the server when the signal aborts', async () => { + + const controller = new AbortController(); + const watcher = new StatementWatcher(db, { dialect: 'postgres', signal: controller.signal, ...WATCH }); + const started = Date.now(); + + const pending = watcher.run('sleep.sql', db, (conn) => sql`select pg_sleep(20)`.execute(conn)); + + await new Promise((r) => setTimeout(r, 500)); + controller.abort(); + + const outcome = await pending.then(() => null, (err: Error) => err); + await watcher.close(); + + expect(outcome?.message).toContain('canceling statement due to user request'); + expect(Date.now() - started).toBeLessThan(5_000); + + }, 30_000); + +}); + +describe('integration: mysql statement watcher', () => { + + let db: Kysely; + let destroy: () => Promise; + let reports: Report[]; + let stopListening: () => void; + + beforeAll(async () => { + + await skipIfNoContainer('mysql'); + + const conn = await createTestConnection('mysql'); + db = conn.db; + destroy = conn.destroy; + + }); + + afterAll(async () => { + + if (destroy) await destroy(); + + }); + + beforeEach(() => { + + reports = []; + stopListening = observer.on('file:progress', (data) => { + + reports.push(data); + + }); + + }); + + afterEach(() => stopListening()); + + it('should report on and then kill a running query', async () => { + + const controller = new AbortController(); + const watcher = new StatementWatcher(db, { dialect: 'mysql', signal: controller.signal, ...WATCH }); + const started = Date.now(); + + const pending = watcher.run('sleep.sql', db, (conn) => sql`select sleep(20)`.execute(conn)); + + const report = await waitForReport(reports, (r) => r.status !== null); + controller.abort(); + + await pending.then(() => null, () => null); + await watcher.close(); + + expect(report!.sessionId).not.toBeNull(); + expect(report!.status!.state).toBe('User sleep'); + expect(Date.now() - started).toBeLessThan(5_000); + + }, 30_000); + +});