diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..2a62ec3 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,9 @@ +# cargo-audit configuration (read by `cargo audit` and the CI audit job). + +[advisories] +ignore = [ + # rmcp < 1.4: DNS rebinding in the Streamable HTTP *server* transport. + # task-journal-mcp only serves over stdio (transport-io), which the + # advisory states is not affected. Drop this once rmcp is upgraded to 1.x. + "RUSTSEC-2026-0189", +] diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8548135..da354d1 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,14 +6,14 @@ }, "metadata": { "description": "Task Journal — append-only reasoning chain memory for AI-coding tasks", - "version": "0.29.0" + "version": "0.30.0" }, "plugins": [ { "name": "task-journal", "source": "./plugin", "description": "Append-only journal of AI-coding task reasoning chains. Captures hypotheses, decisions, rejections, evidence — renders compact resume packs so an agent can pick up a 2-week-old task with full context.", - "version": "0.29.0", + "version": "0.30.0", "author": { "name": "Digital-Threads" }, diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d313280..5c88038 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,8 +7,34 @@ on: workflow_dispatch: jobs: + # A tag doesn't wait for CI, so a tag on a red commit would publish. Gate on + # the same checks as ci.yml's test job. + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: cargo fmt + run: cargo fmt --all --check + + - name: cargo clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: cargo test + run: cargo test --workspace --all-targets + publish: name: cargo publish + needs: test runs-on: ubuntu-latest env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c6fd1d..f1a55ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,8 +17,34 @@ env: CARGO_TERM_COLOR: always jobs: + # A tag doesn't wait for CI, so a tag on a red commit would ship binaries. + # Gate on the same checks as ci.yml's test job. + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: cargo fmt + run: cargo fmt --all --check + + - name: cargo clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: cargo test + run: cargo test --workspace --all-targets + build: name: build (${{ matrix.target }}) + needs: test strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index c19b5bd..5d60f9b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ # Local design specs & plans (not versioned) .docs/ + +# Types Claude Code lays beside a mod it loads from this folder +plugin/.claude-plugin/types/ +plugin/tsconfig.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff6b3d..735b8f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,92 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.30.0] - 2026-10-04 + +Claude Code 2.1.287 added mods: plugins that run inside Claude Code and can +shape the system prompt, the compaction and the status line. This release ships +a Task Journal mod, makes every session (Claude Code and Codex alike) keep its +own active task, and fixes what a full audit of the journal turned up. + +### Added +- **Task Journal mod for Claude Code 2.1.287+**, bundled with the plugin + (`plugin/hooks/`). It keeps the session's active task in the system prompt, so + a compaction can no longer lose it, and asks the compaction summary to keep + the task id. Right before a compaction it asks the model what was decided but + never logged and records those as `suggested` events. It shows the task in + the status line, toasts each entry, and reminds the agent to log only after + several prompts without an entry. Options: `nudge_after_turns`, + `distill_on_compact`. While it runs, the classic hooks skip what it replaces + (the reminder, per-message classification, the transcript catch-ups) and keep + the rest (resume packs, push-recall, `/rewind`). On Claude Code older than + 2.1.287 the mod doesn't load and everything else works as before. +- **Per-session active task.** Events carry their session id and the index + keeps it, so two sessions in one project each resume their own task instead + of whichever task was touched last. Codex sessions bind the same way: the MCP + server reads the session id Codex sends with each tool call. Only deliberate + writes bind a session — a hook's model-switch note or a classifier's guess + landing on another task no longer pulls a fresh session onto it. +- `task-journal state [--session] [--prefer]`: the session's active task, + counts and latest entries as JSON (schema `tj-state/1`). +- `task-journal event --suggested --session --origin` for integrations, and an + optional `session_id` on the MCP write tools. +- `task_search` honours `status` and `project`; with an empty query it lists the + project's tasks, and its result gains a `tasks` field (title, status, goal). +- `doctor` reports whether `codex` is on PATH. + +### Fixed +- **Re-indexing is linear and resumable.** Rebuilding the index cost time + quadratic in the journal's size (a 7,000-event journal: ~30 s); it is now + linear (~1.5 s for the same journal), commits in chunks, and a run that is + cut off picks up where it stopped instead of starting over. +- **An older binary next to this one can't corrupt the index.** A 0.29 server + still running in an open session would rebuild the index without the new + columns; the next 0.30 call now notices and repairs it. +- **Goals and external links survive a rebuild.** They lived only in SQLite; a + rebuild from the log dropped them, Loom links included. They are now in the + journal (on the open event, or an `amend` event for later changes). +- **Corrections take effect.** A `correction` now hides the event it corrects + from every pack section and from `export-pr`. +- **Packs say what is unconfirmed.** Suggested decisions and rejections are + marked; compaction markers and model-switch records no longer appear as + decisions, in `export-pr`, in recall or in global memory; a compact pack keeps + its gap summary; a cached pack no longer shows stale linked-task status. +- **Auto-capture stays in its project.** The shared `pending/` queue was + processed by whichever project's worker ran first, so one project's chunks + could land in another's journal. Entries are now scoped to their project, + processed oldest first, and the classify worker holds an OS file lock, so two + workers can no longer pick up the same chunk. +- A crash mid-write no longer costs the next event too; schema migrations are + atomic and safe when two processes open a fresh database. +- `PostToolUse` chunks are capped and no longer turn tool output into bogus + `constraint` events; push-recall shows each hit once per session. +- `pending retry` really retries, and `ingest-hook` no longer deletes failed + chunks before they can be retried. +- The `codex` backend no longer triggers your own Codex hooks from inside a + `dream` or `complete` run. +- `dream` no longer skips sessions forever after a scoped or partly failed run, + drops events for task ids the model made up, and survives an unreadable + transcript. `backfill` detects imported sessions reliably and indexes them. +- FTS search no longer fails on punctuation or words like `AND`. +- An empty `.git` directory (as Codex's sandbox briefly mounts in `/tmp`) is no + longer taken for a repository root; like git, a `.git` directory needs `HEAD`. +- `task_close` harvests artifacts from the project dir; `event_add` and + `artifact_add` reject unknown task ids instead of writing orphan events. +- `migrate-project` carries the WAL, the pending queue and the push-recall log + along, and keeps a backup of a destination journal it replaces. +- `export`, `events list`, `rejected`, the TUI and the extractor got smaller + fixes (malformed lines, char-safe ids, key repeats, char counts). + +### Changed +- The removed `ClaudeBinaryRunner`, the unused dream backends and the + `FileChanged` / `watchPaths` / `TJ_ASYNC_REWAKE` paths are gone; they were never + reachable. +- `task-journal-core` is pinned once in the workspace; publishing waits for fmt, + clippy and tests to pass. +- An `amend` line is skipped (with a warning) by binaries older than 0.30.0. +- After upgrading, restart open Claude Code and Codex sessions; the first + command re-indexes the project once. + ## [0.29.0] - 2026-09-25 Three months of Claude Code releases (2.1.196 → 2.1.281) changed how hooks are diff --git a/Cargo.lock b/Cargo.lock index 529d838..321513b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,7 +88,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -99,14 +99,14 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "assert-json-diff" @@ -360,7 +360,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -730,7 +730,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -781,7 +781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1026,9 +1026,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1402,7 +1402,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1699,7 +1699,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2165,14 +2165,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -2194,9 +2194,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -2420,7 +2420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2516,7 +2516,7 @@ dependencies = [ [[package]] name = "task-journal-cli" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "assert_cmd", @@ -2524,7 +2524,7 @@ dependencies = [ "chrono", "clap", "crossterm", - "libc", + "fd-lock", "mockito", "predicates", "ratatui", @@ -2540,7 +2540,7 @@ dependencies = [ [[package]] name = "task-journal-core" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "chrono", @@ -2565,7 +2565,7 @@ dependencies = [ [[package]] name = "task-journal-mcp" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "chrono", @@ -2593,7 +2593,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3091,7 +3091,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0c150d5..42b3c2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.29.0" +version = "0.30.0" edition = "2021" rust-version = "1.88" license = "MIT" @@ -18,6 +18,9 @@ keywords = ["task-journal", "ai-coding", "mcp", "memory", "agent"] categories = ["command-line-utilities", "development-tools"] [workspace.dependencies] +# `version` must equal workspace.package.version: `cargo publish` uploads it as +# the requirement on core. Cargo can't inherit it; plugin_metadata.rs checks it. +tj-core = { package = "task-journal-core", version = "0.30.0", path = "crates/tj-core", default-features = false } anyhow = "1" thiserror = "1" serde = { version = "1", features = ["derive"] } diff --git a/INSTALL.md b/INSTALL.md index 9f8d366..cd63c58 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,35 +1,86 @@ # Installation +Task Journal is one Rust core (the `task-journal` CLI and the `task-journal-mcp` +server) plus thin integrations for each agent: + +| Client | What you install | What it gets | +|--------|------------------|--------------| +| Claude Code 2.1.287+ | the plugin (MCP server + skill + mod) | journal tools, the session's task kept in the system prompt, catch-up before compaction, status line | +| Claude Code, older | the plugin, optionally `install-hooks` | journal tools, resume packs on session start | +| Codex | `codex mcp add` + `install-hooks --client codex` | journal tools, resume packs, optional auto-capture | +| Any other MCP client | the MCP server | the seven journal tools | + ## Prerequisites -- **Rust toolchain 1.83+** — `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- (Optional, for auto-capture) **`ANTHROPIC_API_KEY`** exported in your shell +- **Rust 1.88+** — `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` +- No API key is needed. Self-tagging through the MCP tools is free; the optional + LLM-backed features reuse your logged-in `claude` or `codex` CLI, or + `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` if you prefer. + +## Install the binaries + +```bash +cargo install task-journal-cli task-journal-mcp +``` -## Build & install +or from a clone: ```bash -git clone -cd claude-memory -cargo build --release --workspace +git clone https://github.com/Digital-Threads/Task-Journal +cd Task-Journal cargo install --path crates/tj-cli cargo install --path crates/tj-mcp ``` -This installs two binaries to `~/.cargo/bin/`: +This installs `task-journal` (CLI) and `task-journal-mcp` (MCP server) into +`~/.cargo/bin/`. Check with `task-journal --version`. -- `task-journal` — CLI -- `task-journal-mcp` — MCP server - -Verify: +## Claude Code ```bash -task-journal --version -task-journal-mcp --help 2>/dev/null || true # MCP server speaks JSON-RPC over stdin +claude plugin install github:Digital-Threads/Task-Journal ``` -## MCP server (Claude Code) +The plugin brings the MCP server, the `task-journal` skill and, on Claude Code +2.1.287 or newer, the **Task Journal mod**. Restart Claude Code after installing. + +### The mod (Claude Code 2.1.287+) -Add to `~/.claude/settings.json`: +The mod runs inside Claude Code and needs no setup. It: + +- keeps the session's active task (id, title, goal) in the system prompt, so a + compaction can't lose it, and asks the compaction summary to keep its id; +- gives every session its own active task, even with several sessions open in + one project; +- shows the task in the status line and a short toast when an entry is recorded; +- reminds the agent to log only after several prompts without an entry, instead + of on every prompt; +- right before a compaction, asks the model which decisions, rejections and + findings were never logged and records them as `suggested` events (one extra + model call per compaction, billed to your plan, mostly served from the prompt + cache). + +Two options live in `/config` (or `claude plugin configure task-journal`): +`nudge_after_turns` (default 6, 0 turns the reminder off) and +`distill_on_compact` (default on). + +The mod calls the `task-journal` CLI, so the binaries must be installed and +current (0.30+). If they aren't, it says so once and stays off. On Claude Code +older than 2.1.287 the mod doesn't load; the MCP tools and the skill work as +before. + +**Classic hooks next to the mod.** If you ran `task-journal install-hooks` +before, you can keep it. The mod tells those hooks it is active +(`TJ_MOD_ACTIVE=1`), and they skip what the mod replaces — the reminder, +per-message classification (`--auto-capture`) and the transcript catch-ups at +stop, compaction and `/clear` (the mod catches up once, right before each +compaction). The rest keeps running: the session-start resume +pack, the model-switch record, push-recall and `/rewind` corrections. To remove +them anyway: `task-journal install-hooks --scope user --uninstall`. + +### Older Claude Code, or without the plugin + +Wire the MCP server yourself in `~/.claude/settings.json`: ```json { @@ -39,59 +90,83 @@ Add to `~/.claude/settings.json`: } ``` -Restart Claude Code. The 5 tools (`task_pack`, `task_search`, `task_create`, `event_add`, `task_close`) become available to the agent. - -## Auto-capture hooks - -Install once for your user: +and, for resume packs at session start, install the hooks: ```bash -export ANTHROPIC_API_KEY=sk-ant-... -task-journal install-hooks --scope user -# → /home//.claude/settings.json +task-journal install-hooks --scope user # → ~/.claude/settings.json +task-journal install-hooks --scope project # → .claude/settings.json +task-journal install-hooks --scope user --auto-capture # also classify prompts and tool calls in the background ``` -Or per-project: +The default install adds only a read-only `SessionStart` hook (resume packs) and +a `UserPromptSubmit` reminder. `--auto-capture` adds background classification +of prompts, tool calls and session ends; it is a safety net under self-tagging, +not a replacement, and with `--backend hybrid` (the default) it may spawn +`claude -p` for chunks the local heuristic can't classify. + +The installer keeps every unrelated key in the settings file, wraps each hook in +`|| true` so a failure never breaks the session, runs the capture hooks `async` +so the chat never waits, and gives `SessionEnd` an explicit timeout. + +> The hook payload arrives as JSON on stdin. There are no `$CLAUDE_HOOK_*` +> environment variables. + +## Codex ```bash -cd /path/to/project -task-journal install-hooks --scope project -# → .claude/settings.json +codex mcp add task-journal -- task-journal-mcp +task-journal install-hooks --scope user --client codex # → ~/.codex/hooks.json ``` -The installer: +Codex asks before each MCP tool call. To let the agent journal without a prompt +every time, approve the server's tools in `~/.codex/config.toml`: + +```toml +[mcp_servers.task-journal] +command = "task-journal-mcp" +default_tools_approval_mode = "approve" +``` -- Preserves all unrelated keys in `settings.json` (e.g. `theme`, other MCP servers). -- Wraps the hook command with `|| true` so a classifier failure (network down, rate limit, missing key) **never** breaks Claude Code. -- Failures land in `/pending/.json` and are replayed on next successful ingest. +In `codex exec` (non-interactive) an unapproved MCP call is refused outright, so +this setting is required there. -- Gives each hook the timing it needs: the capture hooks run `async` so the chat never waits on them, and `SessionEnd` carries an explicit `timeout` — without one Claude Code cancels it after 1.5 seconds and the tail of the session is lost. +Codex sends its session id with every tool call, so events recorded from Codex +are bound to their session like Claude Code's: each Codex session resumes its own +task. `--client codex` trims the hook wiring to what Codex supports (its +`SessionEnd` budget is 3 seconds and it has no `PostModelSwitch` event). `dream`, +`backfill` and `complete --enrich` read Claude Code transcripts only. -> **Note**: the hook payload arrives as JSON on stdin (`hook_event_name`, `source`, `cwd`, `transcript_path`, …). There are no `$CLAUDE_HOOK_*` environment variables — an older template used them and always fed the classifier empty text. +## Other MCP clients -## Codex +Point the client at the `task-journal-mcp` command (stdio). The server's +built-in instructions describe the workflow, and the seven tools work the same: +`task_create`, `event_add`, `artifact_add`, `task_close`, `task_pack`, +`task_search`, `task_check`. The project is the server's working directory; start +it with `--project-dir ` if the client launches it elsewhere. A client that +knows its session id can pass `session_id` to the write tools so each session +keeps its own active task. -Codex speaks the same hook protocol, so the same journal works there: +## Upgrading ```bash -codex mcp add task-journal -- task-journal-mcp -task-journal install-hooks --scope user --client codex -# → /home//.codex/hooks.json +cargo install task-journal-cli task-journal-mcp --force ``` -`--client codex` writes to Codex's own hooks file and trims the wiring to what Codex supports: its `SessionEnd` budget caps at 3 seconds, and it has no `PostModelSwitch` event. Everything else — the resume pack on `SessionStart`, the nudge, the background capture — behaves the same. +Then restart every open Claude Code and Codex session, so no old MCP server +keeps writing next to the new one. The first command after the upgrade +re-indexes the project once (a few seconds for a large journal); later calls +are incremental. -## Verify install +## Verify ```bash -task-journal create "Test task" -# → tj-xxxxxx -task-journal event tj-xxxxxx --type decision --text "Adopt my plan" -task-journal pack tj-xxxxxx --mode full +task-journal create "Test task" --goal "Check the install" +# → tj-xxxxxxxxxx +task-journal event tj-xxxxxxxxxx --type decision --text "Adopt my plan" +task-journal pack tj-xxxxxxxxxx --mode full +task-journal doctor ``` -You should see Markdown output with the title and the `[decision]` event. - ## Uninstall hooks ```bash @@ -99,7 +174,11 @@ task-journal install-hooks --scope user --uninstall # add --client codex to remove them from ~/.codex/hooks.json ``` -This removes only the `hooks` block from `settings.json` (keeps theme, other servers, everything else). +This removes only task-journal's own hook entries (and its old +`TJ_CLASSIFIER_CLI` env key) from `~/.claude/settings.json`, or from +`~/.codex/hooks.json` with `--client codex`. Everything else stays. With +`--scope project` the file is `.claude/settings.json` or `.codex/hooks.json` in +the current directory. ## Where data lives @@ -109,35 +188,28 @@ This removes only the `hooks` block from `settings.json` (keeps theme, other ser | macOS | `~/Library/Application Support/task-journal` | | Windows | `%LOCALAPPDATA%\task-journal` | -Inside: +`TASK_JOURNAL_DATA_DIR` overrides it. Inside: ``` task-journal/ -├── events/.jsonl # append-only event log (source of truth) -├── state/.sqlite # derived state (rebuildable from JSONL) -├── metrics/.jsonl # classifier telemetry -└── pending/.json # failed classifications awaiting retry +├── events/.jsonl # append-only event log (source of truth) +├── state/.sqlite # derived state (rebuildable from the log) +├── metrics/.jsonl # classifier telemetry +└── pending/..json # auto-capture chunks waiting for the classifier ``` -To reset one project's state: `rm /state/.sqlite` (regenerated on next read). - -To wipe entire journal: `rm -rf `. - -## Diagnostics - -| Issue | Check | -|-------|-------| -| Hook didn't write anything | `ls /events/` — JSONL exists? | -| `[?]` markers everywhere | Classifier confidence consistently below 0.85; consider `task-journal stats` for accuracy ratio | -| Pack is huge | Check `metadata.truncated` flag; pack is capped at 10 KB (full) / 2 KB (compact) | -| `ANTHROPIC_API_KEY env var not set` | Hook runs with same env as Claude Code; export the key in your shell rc file | -| `pending/` growing | Real classifier is failing; inspect `/pending/*.json` for last error | +To reset one project's state, delete `state/.sqlite`; it is rebuilt from +the log on the next read, goals and external links included. To wipe the whole +journal: `rm -rf `. ## Troubleshooting | Symptom | Fix | |---------|-----| -| `task-journal-mcp: command not found` | Не сделал `cargo install`, либо `~/.cargo/bin` не в PATH. Проверь `which task-journal-mcp`. | -| Plugin виден, но MCP tools не подцепились | Перезапусти Claude Code полностью (закрыть, открыть). | -| Hook ничего не пишет в `events/` | `ANTHROPIC_API_KEY` не выставлен — экспортируй в `~/.bashrc`. | -| Pack возвращает "task not found" | Сначала `task-journal create`, потом `pack` — порядок важен. Или после нескольких ингестов запусти `task-journal rebuild-state`. | +| `task-journal-mcp: command not found` | `cargo install` didn't run, or `~/.cargo/bin` isn't on PATH (`which task-journal-mcp`). | +| The plugin is listed but the tools are missing | Restart Claude Code completely. | +| "the task-journal CLI 0.30+ was not found" in the transcript | The mod needs the binaries: `cargo install task-journal-cli task-journal-mcp --force`. | +| Codex says an MCP call "requires approval" | Add `default_tools_approval_mode = "approve"` to the server's block in `~/.codex/config.toml`. | +| `pending/` keeps growing | Auto-capture's classifier is failing; `task-journal pending list` shows the last error, `task-journal pending retry` retries. | +| A pack says "task not found" | The id is wrong or from another project; `task-journal search ""` lists this project's tasks. | +| A pack looks cut | Packs are capped at 2 KB (compact) and 32 KB (full); `task_pack` reports `truncated`. | diff --git a/README.md b/README.md index b4d7ef5..f6db340 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ Then install the Rust binaries that the plugin calls into: cargo install task-journal-cli task-journal-mcp ``` -That's it. Restart Claude Code, start working, and the journal fills itself. +That's it. Restart Claude Code, start working, and the journal fills itself. On +Claude Code 2.1.287+ the plugin also brings the **Task Journal mod** (see +[How it works](#how-it-works)). **Alternative installs:** [pre-built binaries](https://github.com/Digital-Threads/Task-Journal/releases), `cargo install` only (manual MCP wiring), or build from source — see [Manual Setup](#manual-setup). @@ -80,6 +82,11 @@ codex mcp add task-journal -- task-journal-mcp task-journal install-hooks --scope user --client codex ``` +Codex asks before every MCP tool call; add `default_tools_approval_mode = "approve"` +under `[mcp_servers.task-journal]` in `~/.codex/config.toml` so the agent can journal +without a prompt each time (in `codex exec` it is required). Any other MCP client +works with the server alone — see [INSTALL.md](INSTALL.md) for every client. + ## Why not just use the agent's own memory? Claude Code has auto-memory (`MEMORY.md`, `/memory`) and Codex has its own memory @@ -94,6 +101,7 @@ that tie it back to the code. ## How it works +- **The mod (Claude Code 2.1.287+).** A plugin of function hooks that runs inside Claude Code and calls the `task-journal` CLI. It keeps the session's active task in the system prompt (a compaction can't drop it), gives each session its own active task, shows the task in the status line, reminds the agent to log only after several prompts without an entry, and — right before a compaction — asks the model what was never logged and records it as `suggested` events. It needs no setup. When it runs, the classic hooks skip what it replaces (the reminder, per-message classification and the transcript catch-ups) and keep the rest (resume packs, push-recall, `/rewind`). Codex and Claude Code older than 2.1.287 keep using the hooks below; there the mod simply doesn't load. - **Self-tagging is the primary path (recommended).** You — the agent in the live session — record reasoning directly via the five MCP tools: open a task with a `goal`, append a typed `decision` / `finding` / `rejection` / `evidence` event at the moment of commitment, and `task_close` with a written `outcome`. This is free (it rides the interactive session), language-agnostic, and higher-fidelity than any after-the-fact classifier. The bundled `task-journal` skill drives this automatically. See [MCP tools](#mcp-tools). - **Auto-capture is an opt-in backstop — OFF by default (v0.14.0).** A fresh `install-hooks` wires only a cheap, read-only SessionStart resume hook: no per-message classifier runs, no `claude -p` is ever spawned, nothing is charged. Self-tagging is the capture mechanism. Opt in with `install-hooks --auto-capture` and Claude Code hooks also run every prompt, tool call, and reply through a two-stage classifier that lands typed events on its own: Stage 1 is a fast in-process heuristic (obvious EN+RU phrasing, zero cost); Stage 2 falls back to an LLM only when the heuristic is uncertain (and only if you pick `--backend agent-sdk` / `api`). Even opted in it is a safety net under your explicit self-tagging, not the main mechanism, and it misses real reasoning — especially non-English prose. - **Artifact extraction.** Each event scans its text for commit hashes, PR URLs, file paths, issue IDs, and branch names. Aggregated artifacts are how Task Journal links related tasks: when you start a new task touching the same issue or file, the prior task is surfaced automatically. @@ -163,14 +171,15 @@ task-journal pack tj-x9rz1f --mode full |---------|--------------| | `create [--goal "..."]` | Open a task with optional goal | | `goal <id> "..."` | Set or replace a task's goal | -| `event <id> --type X --text Y` | Append a typed event | +| `event <id> --type X --text Y [--suggested] [--session S] [--origin O]` | Append a typed event | +| `state [--session S]` | The session's active task, counts and latest entries as JSON (what the mod reads) | | `event-correct --corrects <eid> --task <id> --text "..."` | Correct an earlier event | | `external <id> "..."` | Append an external reference (URL, ticket, linked task) | | `close <id> --outcome "..." --outcome-tag done\|abandoned\|superseded` | Close with outcome | | `reopen <id> --reason "..."` | Reopen a closed task | | `pack <id> --mode compact\|full` | Render a resume pack | | `events list [--limit N]` | List recent events | -| `search <query> [--all-projects]` | Full-text search (FTS5) | +| `search <query> [--all-projects]` | Full-text search (FTS5); an empty query lists tasks | | `rejected <topic> [--all-projects] [--limit N] [--since DAYS]` | Cross-task rejection lookup — surfaces approaches already turned down | | `export-pr <id>` | Render a task as PR-description Markdown | | `statusline` | One-liner for `~/.claude/settings.json` `statusLine` (sub-100ms) | @@ -189,15 +198,21 @@ task-journal pack tj-x9rz1f --mode full ## MCP tools -The MCP server exposes five tools to Claude Code (and any MCP client): +The MCP server exposes seven tools to Claude Code, Codex and any MCP client: | Tool | Purpose | |------|---------| -| `task_create` | Open a task with optional goal | +| `task_create` | Open a task with a goal (and optional parent) | | `event_add` | Append a typed reasoning event | -| `task_pack` | Render a resume pack | -| `task_search` | Full-text search across events | +| `artifact_add` | Attach a typed link (doc, deploy, dashboard, …) | | `task_close` | Close with outcome and outcome tag | +| `task_pack` | Render a resume pack | +| `task_search` | Search tasks: full text, or `status="open"` with no query to list them | +| `task_check` | Score a task's completeness and list its gaps | + +The write tools take an optional `session_id`; Claude Code (through the mod) and +Codex (through each call's metadata) fill it in, so every session keeps its own +active task. ## Configuration @@ -225,10 +240,13 @@ chain `agent-sdk → api`, using whichever backends are available. Reorder the c with `TJ_HYBRID_LLM_ORDER` (e.g. `api,agent-sdk` to prefer the API key). With no LLM backend available, the heuristic still runs and ambiguous chunks queue in `pending/`. -The offline **`dream`** backfill (re-mining old transcripts for missed reasoning) uses -the same `agent-sdk` path: it prefers the local `claude` CLI pinned to Haiku and only -falls back to the `api` backend when no `claude` is on PATH — so dreaming over your -history needs no API key either. Override its model with `TJ_DREAM_MODEL`. +The offline **`dream`** backfill (re-mining old transcripts for missed reasoning) does +not use this chain. It runs on one LLM backend picked with `--backend` or `TJ_BACKEND`: +`claude-p` (the default — the local `claude` CLI pinned to Haiku, no API key), `codex` +(local `codex exec`, model via `TJ_CODEX_MODEL` as above), `anthropic` (needs +`ANTHROPIC_API_KEY`), `openai` (needs `OPENAI_API_KEY`) or `ollama` (free, local). +There is no fallback: when the chosen backend is unavailable (say, no `claude` on +PATH), `dream` says so and stops. Model overrides per backend are in the table below. | Env var | Effect | Default | |---------|--------|---------| @@ -236,7 +254,10 @@ history needs no API key either. Override its model with `TJ_DREAM_MODEL`. | `TJ_AGENT_SDK_MODEL` | Override the model the `agent-sdk` backend passes to `claude --model`. | `claude-haiku-4-5` | | `TJ_HYBRID_LLM_ORDER` | Comma-separated fallback order for `--backend=hybrid`. | `agent-sdk,api` | | `TJ_CLASSIFIER_MODEL` | Override the Anthropic model used by the `api` backend. | `claude-haiku-4-5-20251001` | -| `TJ_DREAM_MODEL` | Override the model used by `dream` backfill (both agent-sdk and api). | `claude-haiku-4-5` (agent-sdk) | +| `TJ_BACKEND` | LLM backend for `dream` (and `complete`, `consolidate`): `claude-p`, `codex`, `anthropic`, `openai` or `ollama`. `--backend` overrides it. | `claude-p` | +| `TJ_CONSOLIDATE_MODEL` | Model for `dream` on the `claude-p` and `anthropic` backends. | `claude-haiku-4-5` (claude-p), `claude-haiku-4-5-20251001` (anthropic) | +| `TJ_OPENAI_MODEL` / `TJ_OPENAI_BASE_URL` | Model and endpoint for `dream` on the `openai` backend (any OpenAI-compatible API). | `gpt-4o-mini` / `https://api.openai.com` | +| `TJ_OLLAMA_MODEL` / `TJ_OLLAMA_URL` | Model and endpoint for `dream` on the `ollama` backend. | `llama3.1` / `http://localhost:11434` | | `TJ_AUTO_OPEN_TASKS` | Set to `0` / `false` to disable auto-opening a task from `UserPromptSubmit` when no open task exists. | `1` | ## Event types @@ -255,6 +276,8 @@ history needs no API key either. Override its model with `TJ_DREAM_MODEL`. | `supersede` | Replaces an earlier event | | `close` | Task completed with outcome and outcome tag | | `redirect` | Task re-routed to a different task | +| `rename` | Task title changed | +| `amend` | Goal or external references changed (bookkeeping, kept out of packs and search) | ## Manual Setup @@ -296,17 +319,20 @@ Plugin install: /plugin update task-journal@task-journal ``` -Then refresh the binaries (the plugin doesn't bundle them): +Then refresh the binaries (the plugin doesn't bundle them) and restart every +open Claude Code and Codex session: ```bash cargo install task-journal-cli task-journal-mcp --force ``` +The first command after an upgrade may re-index the project once. + Restart Claude Code and verify: ```bash -task-journal --version # 0.6.3 -task-journal-mcp --version # 0.6.3 +task-journal --version # both print the same version as the plugin +task-journal-mcp --version ``` If you installed from source: diff --git a/RELEASING.md b/RELEASING.md index 173885a..56ddf6f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,9 +5,9 @@ ## Pre-release (один раз для проекта) 1. **GitHub repo создан**, origin подключён. -2. **crates.io аккаунт** заведён → Settings → API Tokens → создать токен `task-journal-publish` с scope `publish-new`. +2. **crates.io аккаунт** заведён → Settings → API Tokens → создать токен `task-journal-publish` со scopes `publish-new` и `publish-update` (без `publish-update` новые версии уже существующих crate'ов не опубликуются). 3. **GitHub Secrets** (Settings → Secrets and variables → Actions → New repository secret): - - `CRATES_IO_TOKEN` = токен из шага 2. + - `CRATES_IO_TOKEN` = токен из шага 2. `publish.yml` читает именно этот secret и передаёт его cargo как env `CARGO_REGISTRY_TOKEN` — поэтому ошибка «`CARGO_REGISTRY_TOKEN is empty`» значит, что не задан `CRATES_IO_TOKEN`. 4. **Crate names проверены** — `cargo search task-journal-core task-journal-cli task-journal-mcp` показывает что свободны. Если заняты — поменять в Cargo.toml каждого crate (`name = "..."`). ## Per-release flow @@ -26,17 +26,19 @@ cargo fmt --all --check # ВАЖНО: версия плагина НЕ наследуется из Cargo.toml — синхронь вручную, иначе # `/plugin update` будет показывать старую версию (см. историю: 0.10.3 vs 0.12.0). # Обнови до той же версии: +# - Cargo.toml [workspace.dependencies] → tj-core version (cargo publish +# заливает её как требование на core) # - plugin/.claude-plugin/plugin.json → "version" # - plugin/package.json → "version" # - .claude-plugin/marketplace.json → metadata.version И plugins[0].version # -# Проверка (падает, если хоть один манифест отстал): +# Проверка (падает, если хоть один манифест или пин отстал): cargo test -p task-journal-cli --test plugin_metadata # Плагин целиком — манифест, MCP-объявления, frontmatter скиллов и команд: claude plugin validate plugin --json -git add Cargo.toml plugin/.claude-plugin/plugin.json plugin/package.json .claude-plugin/marketplace.json +git add Cargo.toml Cargo.lock plugin/.claude-plugin/plugin.json plugin/package.json .claude-plugin/marketplace.json git commit -m "chore: bump version to v0.1.1" git push ``` @@ -56,13 +58,16 @@ git push origin v0.1.1 - **`.github/workflows/release.yml`** — собирает pre-built бинарники под Linux/macOS-x86_64/macOS-arm64/Windows и публикует GitHub Release с прикреплёнными `tar.gz`/`.zip` + `checksums.txt`. - **`.github/workflows/publish.yml`** — публикует все 3 crate'а на crates.io (нужен `CRATES_IO_TOKEN` secret). +Оба workflow сначала гоняют свой job `test` (fmt, clippy `-D warnings`, `cargo test --workspace --all-targets`); если он красный — ни бинарники, ни crate'ы не публикуются. + ### 3. Если автопубликация не нужна — руками ```bash cargo login # paste crates.io token cargo publish -p task-journal-core -sleep 30 +sleep 60 # как в publish.yml: crates.io должен проиндексировать core cargo publish -p task-journal-cli +sleep 30 cargo publish -p task-journal-mcp ``` @@ -84,7 +89,7 @@ cargo publish -p task-journal-mcp ## Post-release Для каждой следующей версии: -- Обнови `version` в `Cargo.toml` workspace +- Пройди шаг 1 целиком: `Cargo.toml` (workspace `version` и пин `tj-core`) **и** три манифеста плагина, `plugin_metadata` зелёный - `git tag vX.Y.Z && git push origin vX.Y.Z` - CI делает остальное @@ -94,5 +99,5 @@ cargo publish -p task-journal-mcp |---------|---------------| | `cargo publish` падает с "name already taken" | Имя crate занято на crates.io. Поменяй `name = "..."` в Cargo.toml на другое. | | Release workflow не запустился | Тэг должен начинаться с `v` (например `v0.1.0`, не `0.1.0` или `release-0.1.0`). | -| `cargo publish -p task-journal-cli` падает с "task-journal-core not found" | Не подождал `sleep 30` после публикации `task-journal-core` — повтори через минуту. | +| `cargo publish -p task-journal-cli` падает с "task-journal-core not found" | Не подождал `sleep 60` после публикации `task-journal-core` — повтори через минуту. | | Pre-built binary не работает на macOS — "killed" | Apple Silicon vs Intel — скачай правильный target (`aarch64-apple-darwin` для M1/M2/M3, `x86_64-apple-darwin` для Intel). | diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index c89b0a8..a76aa86 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -23,7 +23,7 @@ default = ["embed"] embed = ["tj-core/embed"] [dependencies] -tj-core = { package = "task-journal-core", version = "0.29.0", path = "../tj-core", default-features = false } +tj-core = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } @@ -36,9 +36,7 @@ chrono = { workspace = true } ratatui = { workspace = true } crossterm = { workspace = true } tempfile = { workspace = true } - -[target.'cfg(unix)'.dependencies] -libc = "0.2" +fd-lock = { workspace = true } [dev-dependencies] assert_fs = { workspace = true } diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 571a696..ec6b28e 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -14,6 +14,8 @@ struct DoctorReport { task_journal_version: &'static str, claude_in_path: bool, claude_version: Option<String>, + /// Informational only: needed just for `--backend codex`. + codex_in_path: bool, data_dir: PathBuf, events_dir: PathBuf, state_dir: PathBuf, @@ -48,6 +50,14 @@ impl DoctorReport { "NOT FOUND in PATH".into() } ); + println!( + " codex binary {}", + if self.codex_in_path { + "found" + } else { + "not found in PATH (only needed for the codex backend)" + } + ); println!(" data dir {}", self.data_dir.display()); println!( " events dir {} ({})", @@ -112,6 +122,32 @@ fn dir_writable(dir: &std::path::Path) -> bool { r } +/// Read a project's JSONL event log. Malformed lines are skipped with a +/// warning on stderr, the same policy as `rebuild_state`, so one bad line +/// cannot abort a read-only command. +fn read_events_lenient( + path: &std::path::Path, + command: &str, +) -> Result<Vec<tj_core::event::Event>> { + let body = std::fs::read_to_string(path)?; + let mut events = Vec::new(); + + for (i, line) in body.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + match serde_json::from_str(line) { + Ok(e) => events.push(e), + Err(err) => eprintln!( + "warning: skipping malformed JSONL line {} in {command}: {err}", + i + 1 + ), + } + } + + Ok(events) +} + /// Move all on-disk data for one project_hash to another. Used by the /// `migrate-project` subcommand when a project's directory has been /// moved on disk and the canonical-path hash no longer matches. @@ -131,7 +167,8 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool let state_dir = tj_core::paths::state_dir()?; let metrics_dir = tj_core::paths::metrics_dir()?; - // (source, destination) tuples to attempt to rename. + // (source, destination) tuples to attempt to rename. The SQLite runs in + // WAL mode, so its `-wal` / `-shm` sidecars travel with it. let pairs = [ ( events_dir.join(format!("{from_hash}.jsonl")), @@ -141,6 +178,18 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool state_dir.join(format!("{from_hash}.sqlite")), state_dir.join(format!("{to_hash}.sqlite")), ), + ( + state_dir.join(format!("{from_hash}.sqlite-wal")), + state_dir.join(format!("{to_hash}.sqlite-wal")), + ), + ( + state_dir.join(format!("{from_hash}.sqlite-shm")), + state_dir.join(format!("{to_hash}.sqlite-shm")), + ), + ( + state_dir.join(format!("{from_hash}.recall-shown")), + state_dir.join(format!("{to_hash}.recall-shown")), + ), ( metrics_dir.join(format!("{from_hash}.jsonl")), metrics_dir.join(format!("{to_hash}.jsonl")), @@ -159,6 +208,39 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool } } + // Fold uncheckpointed writes into the main file before it moves. When + // no one else holds the DB, closing this connection also deletes the + // sidecars; any that survive are moved with it below. + let src_state_path = state_dir.join(format!("{from_hash}.sqlite")); + if src_state_path.exists() { + let conn = rusqlite::Connection::open(&src_state_path)?; + conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .with_context(|| format!("checkpoint WAL of {src_state_path:?}"))?; + } + + // --force: the JSONL is the source of truth, so keep the journal being + // replaced. A copy, so a failed move still leaves the destination intact. + let (src_jsonl, dst_jsonl) = &pairs[0]; + let mut backup = None; + if src_jsonl.exists() && dst_jsonl.metadata().is_ok_and(|m| m.len() > 0) { + let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ"); + let bak = dst_jsonl.with_file_name(format!("{to_hash}.jsonl.bak-{stamp}")); + std::fs::copy(dst_jsonl, &bak).with_context(|| format!("back up {dst_jsonl:?}"))?; + backup = Some(bak); + } + + // --force: a destination `-wal` / `-shm` belongs to the database being + // replaced. Left next to the one moving in, SQLite would replay it into + // it; the source's own sidecars, if any, are moved in below. + if src_state_path.exists() { + for suffix in ["-wal", "-shm"] { + let stale = state_dir.join(format!("{to_hash}.sqlite{suffix}")); + if stale.exists() { + std::fs::remove_file(&stale).with_context(|| format!("remove {stale:?}"))?; + } + } + } + let mut moved: Vec<String> = Vec::new(); for (src, dst) in &pairs { if !src.exists() { @@ -167,23 +249,64 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool if let Some(parent) = dst.parent() { std::fs::create_dir_all(parent)?; } - if dst.exists() && force { - std::fs::remove_file(dst).with_context(|| format!("remove existing {dst:?}"))?; - } + // No remove-then-rename: rename replaces an existing destination + // atomically (POSIX rename, MOVEFILE_REPLACE_EXISTING on Windows), so + // a failed move under --force leaves the destination intact. std::fs::rename(src, dst).with_context(|| format!("rename {src:?} -> {dst:?}"))?; moved.push(dst.display().to_string()); } + // Queued chunks carry the hash in their name and, for the worker's + // ownership check, in their `project_hash`. + let pending_dir = events_dir.with_file_name("pending"); + if pending_dir.exists() { + let prefix = format!("{from_hash}."); + for src in std::fs::read_dir(&pending_dir)?.flatten().map(|e| e.path()) { + let Some(rest) = src + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_prefix(&prefix)) + else { + continue; + }; + let dst = pending_dir.join(format!("{to_hash}.{rest}")); + + let body = std::fs::read_to_string(&src)?; + if let Ok(mut v) = serde_json::from_str::<serde_json::Value>(&body) { + if v["project_hash"] == from_hash.as_str() { + v["project_hash"] = to_hash.clone().into(); + std::fs::write(&src, serde_json::to_string_pretty(&v)?)?; + } + } + + std::fs::rename(&src, &dst).with_context(|| format!("rename {src:?} -> {dst:?}"))?; + moved.push(dst.display().to_string()); + } + } + // Re-key the project_hash columns inside the (now renamed) SQLite. let new_state_path = state_dir.join(format!("{to_hash}.sqlite")); if new_state_path.exists() { let conn = tj_core::db::open(&new_state_path)?; - conn.execute( - "UPDATE tasks SET project_hash = ?1 WHERE project_hash = ?2", - rusqlite::params![to_hash, from_hash], - )?; - conn.execute( - "UPDATE index_state SET project_hash = ?1 WHERE project_hash = ?2", + for table in [ + "tasks", + "index_state", + "projection_state", + "embeddings", + "dream_state", + ] { + conn.execute( + &format!("UPDATE {table} SET project_hash = ?1 WHERE project_hash = ?2"), + rusqlite::params![to_hash, from_hash], + )?; + } + } + + // The global cross-project index keys its rows by project_hash too. + let memory_path = tj_core::paths::memory_db()?; + if memory_path.exists() { + tj_core::memory::open(&memory_path)?.execute( + "UPDATE global_memory SET project_hash = ?1 WHERE project_hash = ?2", rusqlite::params![to_hash, from_hash], )?; } @@ -197,6 +320,9 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool } println!(" project_hash {from_hash} -> {to_hash}"); } + if let Some(bak) = backup { + println!("kept the replaced journal as {}", bak.display()); + } Ok(()) } @@ -244,6 +370,36 @@ time { font-family: ui-monospace, monospace; color: var(--muted); margin-right: .suggested::after { content: " ?"; color: var(--muted); } "#; +/// Title and status of one task for the md/html export, folded over its +/// events with the same rules as the SQLite projection +/// (`tj_core::db::upsert_task_from_event`): the first `open` sets the title, +/// a later `rename` replaces it, and the last `close`/`reopen` decides status. +fn export_title_and_status(task_events: &[&tj_core::event::Event]) -> (String, &'static str) { + use tj_core::event::EventType; + + let mut title: Option<String> = None; + let mut status = "open"; + + for e in task_events { + let named = || { + e.meta + .get("title") + .and_then(|v| v.as_str()) + .unwrap_or(&e.text) + .to_string() + }; + match e.event_type { + EventType::Open if title.is_none() => title = Some(named()), + EventType::Rename if title.is_some() => title = Some(named()), + EventType::Close => status = "closed", + EventType::Reopen => status = "open", + _ => {} + } + } + + (title.unwrap_or_else(|| "(untitled)".into()), status) +} + fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { use std::collections::BTreeMap; @@ -266,23 +422,7 @@ fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { out.push_str("<main>"); for (task_id, task_events) in &tasks { - let title = task_events - .iter() - .find(|e| e.event_type == tj_core::event::EventType::Open) - .and_then(|e| { - e.meta - .get("title") - .and_then(|v| v.as_str()) - .map(String::from) - .or_else(|| Some(e.text.clone())) - }) - .unwrap_or_else(|| "(untitled)".into()); - - let closed = task_events - .last() - .map(|e| e.event_type == tj_core::event::EventType::Close) - .unwrap_or(false); - let status = if closed { "closed" } else { "open" }; + let (title, status) = export_title_and_status(task_events); let created = task_events .first() @@ -343,23 +483,63 @@ fn pending_dir() -> Result<std::path::PathBuf> { Ok(dir) } -fn run_pending_list() -> Result<()> { - let dir = pending_dir()?; +/// The current project's entries in the global `pending/` dir. New entries +/// are named `<project_hash>.<ulid>.json` (`….dead.json` once retries are +/// exhausted); a legacy bare `<ulid>.json` counts as ours unless its JSON +/// names another project. Callers filter schema / dead state on top. +fn project_pending_entries( + dir: &std::path::Path, + project_hash: &str, +) -> Result<Vec<std::path::PathBuf>> { if !dir.exists() { - println!("(no pending entries)"); - return Ok(()); + return Ok(Vec::new()); } - let mut entries: Vec<(String, String, String, u32)> = Vec::new(); - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("json") { + + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + let Some(stem) = path + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".json")) + else { continue; + }; + let stem = stem.strip_suffix(".dead").unwrap_or(stem); + let (ulid, ours) = match stem.split_once('.') { + Some((prefix, ulid)) => (ulid, prefix == project_hash), + None => (stem, !legacy_pending_is_foreign(&path, project_hash)), + }; + if ours { + out.push((ulid.to_string(), path)); } - let id = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("?") + } + + // ULIDs sort by creation time: oldest first, so a user prompt is + // classified before the assistant turn that answered it. + out.sort(); + Ok(out.into_iter().map(|(_, path)| path).collect()) +} + +/// A legacy (un-prefixed) entry belongs to another project only when its +/// JSON says so; one without `project_hash` stays visible here. +fn legacy_pending_is_foreign(path: &std::path::Path, project_hash: &str) -> bool { + std::fs::read_to_string(path) + .ok() + .and_then(|b| serde_json::from_str::<serde_json::Value>(&b).ok()) + .and_then(|v| v.get("project_hash")?.as_str().map(|h| h != project_hash)) + .unwrap_or(false) +} + +fn run_pending_list() -> Result<()> { + let dir = pending_dir()?; + let project_hash = tj_core::project_hash::from_path(std::env::current_dir()?)?; + let mut entries: Vec<(String, String, String, u32)> = Vec::new(); + for path in project_pending_entries(&dir, &project_hash)? { + let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?"); + let id = stem + .strip_prefix(&format!("{project_hash}.")) + .unwrap_or(stem) .to_string(); let body = std::fs::read_to_string(&path)?; let v: serde_json::Value = serde_json::from_str(&body)?; @@ -376,9 +556,7 @@ fn run_pending_list() -> Result<()> { .take(72) .collect(); let attempts = v.get("attempts").and_then(|x| x.as_u64()).unwrap_or(0) as u32; - let dead_marker = if id.ends_with(".dead") { " [DEAD]" } else { "" }; entries.push((id, queued_at, text_preview, attempts)); - let _ = dead_marker; } if entries.is_empty() { println!("(no pending entries)"); @@ -392,6 +570,7 @@ fn run_pending_list() -> Result<()> { } fn run_pending_retry( + backend: &str, mock_etype: Option<&str>, mock_tid: Option<&str>, mock_conf: Option<f64>, @@ -405,15 +584,27 @@ fn run_pending_retry( let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + // The CI-safe mock branch needs no classifier. Without a usable backend + // a retry can't do better than the attempt that queued the entry, so + // leave everything as it is rather than burn attempts toward `.dead`. + let classifier = match (mock_etype, mock_tid) { + (Some(_), Some(_)) => None, + _ => match retry_classifier(backend)? { + Some(c) => Some(c), + None => { + println!( + "pending retry: no classifier backend available for `{backend}` \ + (no `claude` on PATH, no ANTHROPIC_API_KEY) — entries left untouched" + ); + return Ok(()); + } + }, + }; + let mut succeeded = 0usize; let mut died = 0usize; let mut still_pending = 0usize; - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("json") { - continue; - } + for path in project_pending_entries(&dir, &project_hash)? { if path .file_stem() .and_then(|s| s.to_str()) @@ -436,11 +627,14 @@ fn run_pending_retry( .and_then(|x| x.as_str()) .unwrap_or("") .to_string(); + let kind = v + .get("kind") + .and_then(|x| x.as_str()) + .unwrap_or("Stop") + .to_string(); - // The real retry path would call the classifier. The CI-safe - // mock branch lets tests drive a deterministic outcome. - let outcome: anyhow::Result<()> = match (mock_etype, mock_tid) { - (Some(etype), Some(tid)) => { + let outcome: anyhow::Result<()> = match (mock_etype, mock_tid, &classifier) { + (Some(etype), Some(tid), _) => { let mut event = tj_core::event::Event::new( tid, parse_event_type(etype)?, @@ -455,9 +649,18 @@ fn run_pending_retry( writer.flush_durable()?; Ok(()) } - _ => Err(anyhow::anyhow!( - "no real classifier wired in retry path yet — pass --mock-* for tests, or run install-hooks and let the hook drain the queue" - )), + (_, _, Some(classifier)) => match classify_chunk( + classifier.as_ref(), + &events_path, + &project_hash, + &kind, + &text, + None, + )? { + ChunkOutcome::Unplaced(err) => Err(anyhow::anyhow!(err)), + ChunkOutcome::Recorded | ChunkOutcome::Dropped => Ok(()), + }, + _ => unreachable!("a missing classifier returns before the loop"), }; match outcome { @@ -493,6 +696,24 @@ fn run_pending_retry( Ok(()) } +/// The classifier `pending retry` runs, or `None` when the backend has +/// nothing beyond what already failed: hybrid without an LLM fallback, or +/// agent-sdk / api without `claude` / a key. +fn retry_classifier( + backend: &str, +) -> anyhow::Result<Option<Box<dyn tj_core::classifier::Classifier>>> { + Ok(match backend { + "hybrid" | "" => { + let hybrid = tj_core::classifier::hybrid::HybridClassifier::from_env(); + hybrid + .has_llm_fallback() + .then(|| Box::new(hybrid) as Box<dyn tj_core::classifier::Classifier>) + } + "agent-sdk" | "api" => build_classifier(backend).ok(), + other => Some(build_classifier(other)?), + }) +} + fn run_doctor() -> Result<DoctorReport> { let mut issues: Vec<String> = Vec::new(); let mut notes: Vec<String> = Vec::new(); @@ -561,6 +782,7 @@ fn run_doctor() -> Result<DoctorReport> { task_journal_version: env!("CARGO_PKG_VERSION"), claude_in_path, claude_version, + codex_in_path: tj_core::llm::codex_on_path(), data_dir, events_dir, state_dir, @@ -712,6 +934,30 @@ enum Commands { /// Optional event id this supersedes (for type=supersede). #[arg(long)] supersedes: Option<String>, + /// Record it as `suggested` by a classifier rather than confirmed + /// by a person: what a model inferred after the fact. + #[arg(long)] + suggested: bool, + /// Session the event belongs to; defaults to the live session id + /// (CLAUDE_CODE_SESSION_ID, then Codex's CODEX_THREAD_ID). + #[arg(long)] + session: Option<String>, + /// Who wrote it, kept as `meta.origin` (e.g. `mod-distill`). + #[arg(long)] + origin: Option<String>, + }, + /// Print a session's journal state as JSON: its active task (the open + /// task it last wrote to), that task's counts and latest entries, and + /// how many tasks are open. Read by the Claude Code mod; schema + /// `tj-state/1`. + State { + /// Session id; defaults to the live session id. + #[arg(long)] + session: Option<String>, + /// Keep this task as the active one while it is open (the task the + /// caller already shows), instead of the session's latest write. + #[arg(long)] + prefer: Option<String>, }, /// Close a task (writes a `close` event). Close { @@ -761,14 +1007,17 @@ enum Commands { #[arg(long, default_value_t = 7)] days: i64, }, - /// Garbage-collect the pending classifier queue. Removes entries - /// older than N days OR marked dead by retry exhaustion. Run after - /// classifier auth was broken for a while and the queue grew - /// stale. + /// Garbage-collect the current project's pending classifier queue. + /// Removes entries older than N days OR marked dead by retry + /// exhaustion. Run after classifier auth was broken for a while and + /// the queue grew stale. PendingGc { /// Age threshold in days. Default 7. #[arg(long, default_value_t = 7)] days: i64, + /// Collect every project's entries, not just the current one's. + #[arg(long)] + all: bool, }, /// Set or update the goal of an existing task. Goal { @@ -794,7 +1043,7 @@ enum Commands { Reclassify { task_id: String }, /// Full-text search across events (FTS5). Search { - /// Query string. + /// Query string. Empty lists the tasks instead, newest first. query: String, #[arg(long, default_value_t = 20)] limit: usize, @@ -1104,7 +1353,12 @@ enum PendingCmd { List, /// Re-feed every pending entry through the classifier. Marks an /// entry as `<id>.dead.json` after PENDING_MAX_ATTEMPTS failures. + /// With no usable backend, entries are left untouched. Retry { + /// Classifier backend: "hybrid", "agent-sdk", "api", or "heuristic". + /// Defaults to hybrid. + #[arg(long, default_value = "hybrid")] + backend: String, /// Test/dev override: bypass classifier and force this event /// type. Hidden from --help. #[arg(long, hide = true)] @@ -1182,24 +1436,17 @@ fn real_main() -> Result<()> { if let Some(ref parent_id) = parent { meta["parent_id"] = serde_json::Value::String(parent_id.clone()); } + // The goal rides in the open event, so ingest (and a rebuild + // from the JSONL) restores it. + if let Some(g) = goal { + meta["goal"] = serde_json::Value::String(g); + } event.meta = meta; let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; - // If --goal was provided, ingest the open event into SQLite - // (so the row exists) and write the goal column. Skipping - // this when --goal is absent keeps the SQLite hot path - // exclusive to ingest-hook / pack callers. - if let Some(g) = goal { - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::set_task_goal(&conn, &task_id, &g)?; - } - println!("{}", task_id); } Commands::List { tree } => { @@ -1234,12 +1481,7 @@ fn real_main() -> Result<()> { println!("(no events yet)"); return Ok(()); } - let body = std::fs::read_to_string(&events_path)?; - let mut events: Vec<tj_core::event::Event> = body - .lines() - .filter(|l| !l.trim().is_empty()) - .map(serde_json::from_str) - .collect::<Result<_, _>>()?; + let mut events = read_events_lenient(&events_path, "events list")?; events.reverse(); for e in events.into_iter().take(limit) { let title = e @@ -1248,7 +1490,11 @@ fn real_main() -> Result<()> { .and_then(|v| v.as_str()) .map(|s| s.to_string()) .unwrap_or_else(|| e.text.clone()); - println!("{} [{:?}] {}", e.timestamp, e.event_type, title); + let etype = serde_json::to_value(e.event_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "?".into()); + println!("{} [{etype}] {}", e.timestamp, title); } } }, @@ -1469,28 +1715,80 @@ fn real_main() -> Result<()> { text, corrects, supersedes, + suggested, + session, + origin, } => { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); - std::fs::create_dir_all(events_path.parent().unwrap())?; + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + // Same guard as the MCP tool: a mistyped id must not leave an + // orphan event behind. let event_type = parse_event_type(&r#type)?; - let mut event = tj_core::event::Event::new( - &task_id, - event_type, - tj_core::event::Author::User, - tj_core::event::Source::Cli, - text, - ); + if !events_path.exists() { + anyhow::bail!("task not found: {task_id}"); + } + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + if !tj_core::db::task_exists(&conn, &task_id)? { + anyhow::bail!("task not found: {task_id}"); + } + + let (author, source) = if suggested { + ( + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + ) + } else { + (tj_core::event::Author::User, tj_core::event::Source::Cli) + }; + let mut event = tj_core::event::Event::new(&task_id, event_type, author, source, text); event.corrects = corrects; event.supersedes = supersedes; + if suggested { + event.status = tj_core::event::EventStatus::Suggested; + } + if let Some(origin) = origin { + event.meta["origin"] = serde_json::Value::String(origin); + } + let session = session.or_else(tj_core::session_id::session_id_from_env); + tj_core::session_id::stamp_session_id(&mut event.meta, session.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; println!("{}", event.event_id); } + Commands::State { session, prefer } => { + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let session = session.or_else(tj_core::session_id::session_id_from_env); + + // No journal yet: an empty state, and no state DB created for it. + let state = if events_path.exists() { + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + tj_core::session_state::session_state( + &conn, + &project_hash, + session.as_deref(), + prefer.as_deref(), + )? + } else { + tj_core::session_state::SessionState { + schema: tj_core::session_state::SCHEMA, + session_id: session, + open_tasks: 0, + active: None, + } + }; + + println!("{}", serde_json::to_string(&state)?); + } Commands::ArtifactAdd { task_id, kind, @@ -1550,13 +1848,6 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - // Persist outcome BEFORE the close event so the cache wipe - // inside set_task_outcome doesn't compete with subsequent - // assemble calls. Both columns optional — caller can pass - // neither and just get the close event. - if let Some(o) = outcome.as_deref() { - tj_core::db::set_task_outcome(&conn, &task_id, o, outcome_tag.as_deref())?; - } let open_kids = tj_core::db::count_open_children(&conn, &task_id)?; drop(conn); @@ -1571,6 +1862,15 @@ fn real_main() -> Result<()> { if let Some(r) = reason { meta.insert("reason".into(), serde_json::Value::String(r)); } + // outcome + tag ride in the close event's meta and reach the task + // row only when that event is ingested, so a failed append never + // leaves an open task with an outcome and a rebuild keeps it. + if let Some(o) = outcome { + meta.insert("outcome".into(), serde_json::Value::String(o)); + } + if let Some(t) = outcome_tag { + meta.insert("outcome_tag".into(), serde_json::Value::String(t)); + } // Layer-2 close harvest: stamp deterministic git/gh refs (commit, // branch, PR) so the closed pack reads as a clickable ledger of // what shipped. Best-effort; structured artifacts are merged in @@ -1642,7 +1942,7 @@ fn real_main() -> Result<()> { ); } } - Commands::PendingGc { days } => { + Commands::PendingGc { days, all } => { let pending_dir = tj_core::paths::events_dir()? .parent() .ok_or_else(|| anyhow::anyhow!("events_dir has no parent"))? @@ -1651,20 +1951,23 @@ fn real_main() -> Result<()> { println!("(no pending dir — nothing to gc)"); return Ok(()); } + let entries: Vec<std::path::PathBuf> = if all { + std::fs::read_dir(&pending_dir)? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("json")) + .collect() + } else { + let project_hash = tj_core::project_hash::from_path(std::env::current_dir()?)?; + project_pending_entries(&pending_dir, &project_hash)? + }; let cutoff = chrono::Utc::now() - chrono::Duration::days(days); let mut removed = 0usize; - for entry in std::fs::read_dir(&pending_dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) != Some("json") { - continue; - } + for path in entries { // Prefer the file's mtime over JSON parsing — pending // payloads include their own queued_at but are not // guaranteed parseable when the classifier corrupted // input mid-stream. - let mtime = entry - .metadata() + let mtime = std::fs::metadata(&path) .and_then(|m| m.modified()) .ok() .and_then(|t| { @@ -1729,7 +2032,22 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - tj_core::db::set_task_goal(&conn, &task_id, &text)?; + + // An amend event carries the change, so a rebuild from the + // JSONL restores it; ingest applies it to the task row. + let mut event = tj_core::event::Event::new( + &task_id, + tj_core::event::EventType::Amend, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + format!("goal: {text}"), + ); + event.meta = serde_json::json!({ "goal": text }); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; println!("ok"); } Commands::External { task_id, add } => { @@ -1745,7 +2063,20 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - tj_core::db::add_task_external(&conn, &task_id, &add)?; + + let mut event = tj_core::event::Event::new( + &task_id, + tj_core::event::EventType::Amend, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + format!("external: +{add}"), + ); + event.meta = serde_json::json!({ "external_add": [add] }); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; println!("ok"); } Commands::Reclassify { task_id } => { @@ -2134,11 +2465,13 @@ fn real_main() -> Result<()> { run_pending_list()?; } PendingCmd::Retry { + backend, mock_event_type, mock_task_id, mock_confidence, } => { run_pending_retry( + &backend, mock_event_type.as_deref(), mock_task_id.as_deref(), mock_confidence, @@ -2177,6 +2510,17 @@ fn real_main() -> Result<()> { _ => parse_hook_stdin()?, }; + // The Claude Code mod captures in-process and marks the hooks it + // starts; the classic capture stands down where the mod does the + // same work: the Stop / PreCompact / SessionEnd transcript + // catch-up (and the compaction marker) here, per-message queueing + // and auto-open further down. Push-recall, the `/rewind` + // correction, resume packs and model switches stay ours. + let mod_on = mod_active(); + if mod_on && matches!(kind.as_str(), "Stop" | "PreCompact" | "SessionEnd") { + return Ok(()); + } + // Emergency capture kill-switch: a `.capture-disabled` marker in the // data dir no-ops realtime capture (the read-only SessionStart // resume still runs). Because the hook re-invokes this binary on @@ -2232,6 +2576,12 @@ fn real_main() -> Result<()> { &text, tj_core::recall::DEFAULT_MAX_HITS, ) { + let hits = fresh_recall_hits( + hits, + live_session_id.as_deref(), + &events_path, + &project_hash, + ); if !hits.is_empty() { let mut ctx = String::new(); for h in &hits { @@ -2264,8 +2614,12 @@ fn real_main() -> Result<()> { // mcp__ tools) — gated MCP-only, falls through to the queue path so // event capture is unaffected. Disabled by TJ_PUSH_RECALL=0. if kind == "PostToolUse" && std::env::var("TJ_PUSH_RECALL").as_deref() != Ok("0") { - if let Some(envelope) = push_recall_envelope(&payload, &events_path, &project_hash) - { + if let Some(envelope) = push_recall_envelope( + &payload, + &events_path, + &project_hash, + live_session_id.as_deref(), + ) { println!("{}", serde_json::to_string(&envelope)?); } } @@ -2294,7 +2648,8 @@ fn real_main() -> Result<()> { tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 3)?; + let recent = + recent_task_contexts(&conn, &project_hash, 3, live_session_id.as_deref())?; if recent.is_empty() { if !prefs_block.is_empty() { emit_session_context(&prefs_block); @@ -2370,33 +2725,6 @@ runs in the background and won't block you; it only fills gaps and never closes bundle.push_str("\n\n"); } - // v0.10.2 X4: emit `watchPaths` so Claude Code starts - // monitoring our marker files (CLAUDE.md, README.md, - // .docs/plans). When any of them changes, Claude Code - // fires a FileChanged hook event — our ingest-hook - // handler below treats those as `evidence` entries on - // the active task so the journal captures - // "instructions were updated mid-session" without the - // user manually logging it. Only paths that exist at - // SessionStart time are emitted (no point watching a - // non-existent file — Claude Code logs `watcher error` - // and gives up on it). Gated by TJ_WATCH_PATHS=0. - let allow_watch_paths = std::env::var("TJ_WATCH_PATHS").as_deref() != Ok("0"); - let watch_candidates = [ - cwd.join("CLAUDE.md"), - cwd.join("README.md"), - cwd.join(".docs").join("plans"), - ]; - let watch_paths: Vec<String> = if allow_watch_paths { - watch_candidates - .iter() - .filter(|p| p.exists()) - .map(|p| p.to_string_lossy().to_string()) - .collect() - } else { - Vec::new() - }; - // We deliberately DO NOT emit `sessionTitle` or // `initialUserMessage` here. The v0.10.1 X2 experiment set // `sessionTitle` to "TJ — <task_id> (<n> open)", which @@ -2408,18 +2736,10 @@ runs in the background and won't block you; it only fills gaps and never closes // title. The resume context the model actually needs already // rides in `additionalContext`; the tab label belongs to // Claude Code, not to us. (0.14.3) - let mut hook_specific = serde_json::json!({ + let hook_specific = serde_json::json!({ "hookEventName": "SessionStart", "additionalContext": bundle.trim_end(), }); - if !watch_paths.is_empty() { - hook_specific["watchPaths"] = serde_json::Value::Array( - watch_paths - .into_iter() - .map(serde_json::Value::String) - .collect(), - ); - } let envelope = serde_json::json!({ "hookSpecificOutput": hook_specific, }); @@ -2427,65 +2747,6 @@ runs in the background and won't block you; it only fills gaps and never closes return Ok(()); } - // v0.10.2 X4: FileChanged. Claude Code 2.1.x fires this - // event whenever a path in `watchPaths` (emitted on - // SessionStart) changes. Payload: { file_path, event: - // "change"|"add"|"unlink" }. We translate it into an - // `evidence` event on the active task — captures - // "the user/agent edited CLAUDE.md mid-session" without - // anyone typing anything. Schema verified in 2.1.160: - // `literal("FileChanged"), file_path: y.string(), event: - // y.enum(["change","add","unlink"])`. - // - // No active task → drop silently (we're not opening a - // task just because a watched file moved). No events_path - // → ditto, fresh project. - if kind == "FileChanged" { - if !events_path.exists() { - return Ok(()); - } - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; - let Some(tc) = recent.into_iter().next() else { - return Ok(()); - }; - let file_path = payload - .get("file_path") - .and_then(|v| v.as_str()) - .unwrap_or("(unknown)"); - let change = payload - .get("event") - .and_then(|v| v.as_str()) - .unwrap_or("change"); - // Trim noisy absolute paths to project-relative when - // possible — the journal is per-project so the prefix - // is redundant and just steals tokens from the pack. - let display_path = cwd - .to_str() - .and_then(|c| file_path.strip_prefix(c)) - .map(|s| s.trim_start_matches('/').to_string()) - .unwrap_or_else(|| file_path.to_string()); - let evidence_text = format!("FileChanged ({change}): {display_path}"); - let mut event = tj_core::event::Event::new( - &tc.task_id, - tj_core::event::EventType::Evidence, - tj_core::event::Author::Classifier, - tj_core::event::Source::Hook, - evidence_text, - ); - event.confidence = Some(0.9); - event.status = tj_core::event::EventStatus::Confirmed; - tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); - let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - println!("{}", event.event_id); - return Ok(()); - } - // PostModelSwitch (Claude Code 2.1.251+). Payload: { from_model, // to_model, source: "user"|"auto"|"resume", requested_model }. // Which model did the work is a fact about the task that neither @@ -2493,7 +2754,7 @@ runs in the background and won't block you; it only fills gaps and never closes // "why did this stretch come out shallow" is usually answered by // "it ran on a fallback model". Recorded as a `constraint`: it's an // external condition the work happened under, not a decision. - // No active task → drop silently, same rule as FileChanged. + // No active task → drop silently. if kind == "PostModelSwitch" { if !events_path.exists() { return Ok(()); @@ -2502,7 +2763,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2521,7 +2783,10 @@ runs in the background and won't block you; it only fills gaps and never closes .get("source") .and_then(|v| v.as_str()) .unwrap_or("user"); - let text = format!("Model switched ({switch_source}): {from_model} → {to_model}"); + let text = format!( + "{}{switch_source}): {from_model} → {to_model}", + tj_core::reminder::MODEL_SWITCH_TEXT_PREFIX + ); let mut event = tj_core::event::Event::new( &tc.task_id, tj_core::event::EventType::Constraint, @@ -2531,6 +2796,10 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(0.9); event.status = tj_core::event::EventStatus::Confirmed; + // Kept in the journal, but left out of the constraint lists + // (resume reminder, classifier context) — see + // MODEL_SWITCH_TEXT_PREFIX. + event.meta = serde_json::json!({ "kind": "model_switch" }); tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -2556,7 +2825,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2646,6 +2916,9 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(1.0); event.status = tj_core::event::EventStatus::Confirmed; + // Bookkeeping, not a decision: kept out of active decisions, + // export-pr, recall and global memory — see db::is_bookkeeping. + event.meta = serde_json::json!({ "kind": "compaction_marker" }); tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -2693,7 +2966,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2745,9 +3019,10 @@ runs in the background and won't block you; it only fills gaps and never closes ); } - // Drain any pending entries first (Task 10 fills the real-classifier branch). + // Mock path only: drain legacy pending entries first. drain_pending( &events_path, + &project_hash, mock_event_type.as_deref(), mock_task_id.as_deref(), mock_confidence, @@ -2779,7 +3054,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2792,6 +3068,7 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(1.0); event.status = tj_core::event::EventStatus::Confirmed; + tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; @@ -2799,6 +3076,19 @@ runs in the background and won't block you; it only fills gaps and never closes return Ok(()); } + // The mod classifies prompts and tool calls itself (see above). + if mod_on && matches!(kind.as_str(), "UserPromptSubmit" | "PostToolUse") { + return Ok(()); + } + + // A tool call arrives as its full input + response JSON; cap it + // so one big file read or command output can't flood the queue. + let text: String = if kind == "PostToolUse" { + text.chars().take(POST_TOOL_USE_TEXT_MAX).collect() + } else { + text + }; + // v0.6.2 fork-bomb fix. The real-classifier path used to run // `claude -p` synchronously inside the hook, blocking each // UserPromptSubmit/PostToolUse/Stop for 5-30s. Symptoms: @@ -2828,25 +3118,8 @@ runs in the background and won't block you; it only fills gaps and never closes // Fire-and-forget worker. Errors here are best-effort — // a failure to spawn just means the entry sits in // pending/ until the next hook fires another spawn. - let _ = spawn_classify_worker(&backend); - - // v0.10.0 asyncRewake backlog signal. Only the PostToolUse - // hook runs as asyncRewake (hooks.json sets TJ_ASYNC_REWAKE=1 - // there), so other kinds — and direct CLI invocations — - // never exit 2 even on overflow. Exit code 2 from a sync - // hook would BLOCK the operation; only asyncRewake hooks - // treat code 2 as "wake the model with rewakeMessage". stdout - // is appended to the wake message, so the user sees the - // drain command without us reaching into stderr. - let allow_wake = std::env::var("TJ_ASYNC_REWAKE").as_deref() == Ok("1"); - if allow_wake && kind == "PostToolUse" { - let pending_count = count_pending_entries(&events_path).unwrap_or(0); - if pending_count > PENDING_OVERFLOW_THRESHOLD { - println!( - "Task Journal pending queue: {pending_count} entries. Classifier behind — run `task-journal pending-gc --days 0` to drain.", - ); - std::process::exit(2); - } + if std::env::var("TJ_DISABLE_CLASSIFY_SPAWN").is_err() { + let _ = spawn_classify_worker(&backend); } return Ok(()); } @@ -2858,159 +3131,130 @@ runs in the background and won't block you; it only fills gaps and never closes "assistant" }; - let (etype, task_id, confidence, evidence_strength, suggested_text) = if let ( - Some(t), - Some(tid), - ) = - (mock_event_type.as_deref(), mock_task_id.as_deref()) - { - ( - parse_event_type(t)?, - tid.to_string(), - mock_confidence.unwrap_or(1.0), - None, - None, - ) - } else { - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } - let mut recent = recent_task_contexts(&conn, 5)?; - if recent.is_empty() { - // No open tasks. v0.5.0 Phase A: auto-open a new - // task from the user's prompt so subsequent - // events have somewhere to land. Without this - // every fresh session was a black hole — events - // dropped silently because there was nothing to - // classify against. Opt-out via - // TJ_AUTO_OPEN_TASKS=0; only fires for - // UserPromptSubmit (assistant tool calls - // shouldn't conjure tasks). - let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") - .ok() - .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) - .unwrap_or(false); - if auto_open_disabled || !kind.contains("UserPrompt") { - return Ok(()); - } - let Some(new_task) = - auto_open_task_from_prompt(&events_path, &project_hash, &conn, &text)? - else { - // Prompt was only machine noise — nothing worth a task. - return Ok(()); - }; - recent.push(new_task); - } - - use tj_core::classifier::Classifier; - let classifier: Box<dyn Classifier> = match backend.as_str() { - // v0.8.0: hybrid is the new default. Heuristic - // pattern-matching first (free), Anthropic API - // fallback when uncertain (requires ANTHROPIC_API_KEY). - // No background spawn of `claude -p` — that subprocess - // now bills tokens separately from Pro/Max. - "hybrid" | "" => { - Box::new(tj_core::classifier::hybrid::HybridClassifier::from_env()) + let (etype, task_id, confidence, evidence_strength, suggested_text) = + if let (Some(t), Some(tid)) = (mock_event_type.as_deref(), mock_task_id.as_deref()) + { + ( + parse_event_type(t)?, + tid.to_string(), + mock_confidence.unwrap_or(1.0), + None, + None, + ) + } else { + let state_path = + tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } - "api" => Box::new(tj_core::classifier::http::AnthropicClassifier::from_env()?), - "agent-sdk" => Box::new( - tj_core::classifier::agent_sdk::ClaudeCliClassifier::from_env() - .ok_or_else(|| { - anyhow::anyhow!( - "agent-sdk backend selected but no `claude` binary on PATH — \ - install Claude Code (https://claude.com/claude-code) or pick another --backend" - ) - })?, - ), - "heuristic" => { - // Heuristic-only: no LLM at all. Trades coverage - // for absolute zero-cost / offline operation. - use tj_core::classifier::heuristic::try_heuristic; - use tj_core::classifier::{ClassifyInput, ClassifyOutput}; - struct HeuristicOnly; - impl Classifier for HeuristicOnly { - fn classify( - &self, - input: &ClassifyInput, - ) -> anyhow::Result<ClassifyOutput> { - try_heuristic(input).ok_or_else(|| { - anyhow::anyhow!( - "heuristic uncertain (heuristic-only mode has no LLM fallback)" - ) - }) - } + let mut recent = + recent_task_contexts(&conn, &project_hash, 5, live_session_id.as_deref())?; + if recent.is_empty() { + // No open tasks. v0.5.0 Phase A: auto-open a new + // task from the user's prompt so subsequent + // events have somewhere to land. Without this + // every fresh session was a black hole — events + // dropped silently because there was nothing to + // classify against. Opt-out via + // TJ_AUTO_OPEN_TASKS=0; only fires for + // UserPromptSubmit (assistant tool calls + // shouldn't conjure tasks). + let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") + .ok() + .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) + .unwrap_or(false); + if auto_open_disabled || !kind.contains("UserPrompt") { + return Ok(()); } - Box::new(HeuristicOnly) - } - other => anyhow::bail!( - "unknown backend: {other} (expected `hybrid`, `agent-sdk`, `api`, or `heuristic`)" - ), - }; - let input = tj_core::classifier::ClassifyInput { - text: text.clone(), - author_hint: author_hint.into(), - recent_tasks: recent, - }; - let out = match classifier.classify(&input) { - Ok(o) => o, - Err(e) => { - persist_pending(&events_path, &text, &e.to_string())?; - return Ok(()); + let Some(new_task) = auto_open_task_from_prompt( + &events_path, + &project_hash, + &conn, + &text, + live_session_id.as_deref(), + )? + else { + // Prompt was only machine noise — nothing worth a task. + return Ok(()); + }; + recent.push(new_task); } - }; - let Some(tid) = out.task_id_guess else { - return Ok(()); - }; + let classifier = build_classifier(&backend)?; + let input = tj_core::classifier::ClassifyInput { + text: text.clone(), + author_hint: author_hint.into(), + recent_tasks: recent, + tool_output: kind == "PostToolUse", + }; + let out = match classifier.classify(&input) { + Ok(o) => o, + Err(e) => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &e.to_string(), + )?; + return Ok(()); + } + }; - // Journal-integrity safeguards. The classifier sometimes - // mis-attributes events to old or closed tasks (no fault - // of the model — its prompt only sees recent_tasks). We - // reject three patterns that produce confusing journals: - // - // 1. Stop-hook → Close event. The Stop hook fires at - // every Claude Code session end. Session ending - // != task done. Closes happen via explicit - // `task-journal close <id>` only. - // 2. task_id_guess pointing at a non-existent task — - // route to pending so the user can decide later. - // 3. task_id_guess pointing at a CLOSED task — same - // treatment; closed tasks must stay closed. - use tj_core::event::EventType; - if matches!(out.event_type, EventType::Close) && kind == "Stop" { - return Ok(()); - } - match tj_core::db::task_status(&conn, &tid)? { - None => { - persist_pending( - &events_path, - &text, - &format!("task_id_guess `{tid}` not found"), - )?; + let Some(tid) = out.task_id_guess else { return Ok(()); - } - Some(s) if s == "closed" => { - persist_pending( - &events_path, - &text, - &format!("task_id_guess `{tid}` is closed"), - )?; + }; + + // Journal-integrity safeguards. The classifier sometimes + // mis-attributes events to old or closed tasks (no fault + // of the model — its prompt only sees recent_tasks). We + // reject three patterns that produce confusing journals: + // + // 1. Stop-hook → Close event. The Stop hook fires at + // every Claude Code session end. Session ending + // != task done. Closes happen via explicit + // `task-journal close <id>` only. + // 2. task_id_guess pointing at a non-existent task — + // route to pending so the user can decide later. + // 3. task_id_guess pointing at a CLOSED task — same + // treatment; closed tasks must stay closed. + use tj_core::event::EventType; + if matches!(out.event_type, EventType::Close) && kind == "Stop" { return Ok(()); } - _ => {} - } + match tj_core::db::task_status(&conn, &tid)? { + None => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &format!("task_id_guess `{tid}` not found"), + )?; + return Ok(()); + } + Some(s) if s == "closed" => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &format!("task_id_guess `{tid}` is closed"), + )?; + return Ok(()); + } + _ => {} + } - ( - out.event_type, - tid, - out.confidence, - out.evidence_strength, - Some(out.suggested_text), - ) - }; + ( + out.event_type, + tid, + out.confidence, + out.evidence_strength, + Some(out.suggested_text), + ) + }; // Use classifier's suggested_text if available (it's more concise and specific), // fall back to raw hook text for mock/manual events. @@ -3026,6 +3270,7 @@ runs in the background and won't block you; it only fills gaps and never closes event.confidence = Some(confidence); event.status = tj_core::classifier::decide_status(confidence); event.evidence_strength = evidence_strength; + tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -3095,12 +3340,7 @@ runs in the background and won't block you; it only fills gaps and never closes anyhow::bail!("no events file at {events_path:?}"); } - let body = std::fs::read_to_string(&events_path)?; - let all_events: Vec<tj_core::event::Event> = body - .lines() - .filter(|l| !l.trim().is_empty()) - .map(serde_json::from_str) - .collect::<Result<_, _>>()?; + let all_events = read_events_lenient(&events_path, "export")?; // Filter to specific task if requested. let events: Vec<&tj_core::event::Event> = if let Some(ref tid) = task { @@ -3133,29 +3373,7 @@ runs in the background and won't block you; it only fills gaps and never closes } for (task_id, task_events) in &tasks { - // Derive title from the first open event's meta, or text. - let title = task_events - .iter() - .find(|e| e.event_type == tj_core::event::EventType::Open) - .and_then(|e| { - e.meta - .get("title") - .and_then(|v| v.as_str()) - .map(String::from) - .or_else(|| Some(e.text.clone())) - }) - .unwrap_or_else(|| "(untitled)".into()); - - // Determine status: closed if last event is close, else open. - let status = if task_events - .last() - .map(|e| e.event_type == tj_core::event::EventType::Close) - .unwrap_or(false) - { - "closed" - } else { - "open" - }; + let (title, status) = export_title_and_status(task_events); // Created timestamp from first event. let created = task_events @@ -3231,7 +3449,10 @@ runs in the background and won't block you; it only fills gaps and never closes let path = state_dir.join(format!("{hash}.sqlite")); let conn = match rusqlite::Connection::open(&path) { Ok(c) => c, - Err(_) => continue, + Err(e) => { + warn_skipped_project(&hash, e); + continue; + } }; let ids = match run_search( &conn, @@ -3241,7 +3462,10 @@ runs in the background and won't block you; it only fills gaps and never closes limit, ) { Ok(v) => v, - Err(_) => continue, + Err(e) => { + warn_skipped_project(&hash, e); + continue; + } }; for id in ids { println!("{hash}\t{id}"); @@ -3254,11 +3478,14 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + // No journal, no tasks — and no empty state DB left behind to + // show up in `--all-projects` and project lists later. + if !events_path.exists() { + return Ok(()); + } let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; let ids = run_search(&conn, &fts_query, &like_query, event_type.as_deref(), limit)?; for id in ids { println!("{id}"); @@ -3319,7 +3546,8 @@ runs in the background and won't block you; it only fills gaps and never closes Some(d) => d, None => { eprintln!( - "No Claude Code sessions found for: {}", + "No Claude Code sessions found for: {} — backfill reads Claude Code \ +transcripts only (Codex sessions are not read yet)", project_path.display() ); eprintln!( @@ -3332,10 +3560,37 @@ runs in the background and won't block you; it only fills gaps and never closes } }; - // List available sessions. + // Check which sessions are already imported (idempotent): a session + // counts once some event is tagged with it in `meta.session_id` — + // not merely mentioned in some unrelated event's text. + let already_imported: std::collections::HashSet<String> = + std::fs::read_to_string(&events_path) + .unwrap_or_default() + .lines() + .filter_map(|l| serde_json::from_str::<tj_core::event::Event>(l).ok()) + .filter_map(|e| { + e.meta + .get("session_id") + .and_then(|v| v.as_str()) + .map(String::from) + }) + .collect(); + + // List available sessions. --limit counts only sessions not yet + // imported, so a rerun reaches older ones; imported ones stay in + // the list and are reported as skipped below. let mut sessions = discovery::list_sessions(&proj_dir)?; if let Some(max) = limit { - sessions.truncate(max); + let mut fresh = 0; + sessions.retain(|p| { + let id = p.file_stem().and_then(|s| s.to_str()).unwrap_or("?"); + if already_imported.contains(id) { + return true; + } + + fresh += 1; + fresh <= max + }); } if sessions.is_empty() { @@ -3349,18 +3604,6 @@ runs in the background and won't block you; it only fills gaps and never closes project_path.display() ); - // Check which sessions are already imported (idempotent). - let already_imported = if events_path.exists() { - let content = std::fs::read_to_string(&events_path).unwrap_or_default(); - sessions - .iter() - .filter_map(|p| p.file_stem().and_then(|s| s.to_str()).map(String::from)) - .filter(|sid| content.contains(sid)) - .collect::<std::collections::HashSet<_>>() - } else { - std::collections::HashSet::new() - }; - let mut total_tasks = 0; let mut total_events = 0; @@ -3452,6 +3695,13 @@ runs in the background and won't block you; it only fills gaps and never closes ); eprintln!("Run without --dry-run to import."); } else { + // Index the appended events so search / pack see them now. + if total_tasks > 0 { + let state_path = + tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } eprintln!("\nImported {total_tasks} task(s) with {total_events} event(s)."); } } @@ -3556,20 +3806,19 @@ fn run_statusline() -> anyhow::Result<String> { }) .count(); - // Pending dir is global — one entry per queued classifier failure. - // No project filter (matches the brief; per-project counting would - // need extra metadata in each pending file). + // Pending dir is global; this project's entries carry its hash as a + // filename prefix, so counting stays a cheap readdir with no JSON + // parsing. Legacy un-prefixed entries are not counted. + let prefix = format!("{project_hash}."); let pending_count = pending_dir() .ok() .and_then(|d| std::fs::read_dir(&d).ok()) .map(|rd| { rd.filter_map(|e| e.ok()) .filter(|e| { - e.path() - .extension() - .and_then(|x| x.to_str()) - .map(|x| x == "json") - .unwrap_or(false) + let name = e.file_name(); + let name = name.to_string_lossy(); + name.starts_with(&prefix) && name.ends_with(".json") }) .count() }) @@ -3613,6 +3862,23 @@ fn run_search( event_type: Option<&str>, limit: usize, ) -> Result<Vec<String>> { + // No query: list the tasks, newest first, like MCP task_search. FTS5 + // rejects an empty MATCH, so it must not reach it. + if fts_query.is_empty() { + let mut stmt = conn.prepare( + "SELECT task_id FROM tasks \ + WHERE ?1 IS NULL OR task_id IN (SELECT task_id FROM events_index WHERE type = ?1) \ + ORDER BY last_event_at DESC, rowid DESC LIMIT ?2", + )?; + let ids = stmt + .query_map(rusqlite::params![event_type, limit as i64], |r| { + r.get::<_, String>(0) + })? + .collect::<rusqlite::Result<_>>()?; + + return Ok(ids); + } + let (fts_sql, fts_uses_type) = match event_type { Some(_) => ( "SELECT DISTINCT task_id FROM search_fts \ @@ -3672,6 +3938,12 @@ fn run_search( Ok(ids_like) } +/// Cross-project reads keep going past a project they cannot read, but say +/// so on stderr instead of dropping it silently. +fn warn_skipped_project(hash: &str, err: impl std::fmt::Display) { + eprintln!("warning: skipping project {hash}: {err}"); +} + fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option<i64>) -> Result<()> { let cutoff: Option<String> = since.map(|d| { (chrono::Utc::now() - chrono::Duration::days(d)) @@ -3704,11 +3976,21 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option<i64 // attached DBs would be faster but rusqlite's bundled build doesn't // ship ATTACH-friendly ergonomics; per-project loop is fine here. let mut hits: Vec<(String, String, String, String, String)> = Vec::new(); + // Only the --all-projects sweep reports a skipped project: the current + // project of a fresh clone legitimately has no tables yet. + let skip = |hash: &str, e: rusqlite::Error| { + if all_projects { + warn_skipped_project(hash, e); + } + }; for hash in hashes { let path = state_dir.join(format!("{hash}.sqlite")); let conn = match rusqlite::Connection::open(&path) { Ok(c) => c, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; let use_fts = topic_is_fts_safe(topic); @@ -3734,7 +4016,10 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option<i64 let mut stmt = match conn.prepare(sql) { Ok(s) => s, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; let bind_q = if use_fts { topic.to_string() @@ -3751,10 +4036,23 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option<i64 )) }) { Ok(r) => r, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; - for row in rows.flatten() { - hits.push(row); + // Keep going past a row that fails to map, but say so once. + let mut first_err = None; + for row in rows { + match row { + Ok(row) => hits.push(row), + Err(e) => { + first_err.get_or_insert(e); + } + } + } + if let Some(e) = first_err { + eprintln!("warning: skipped unreadable rejection row(s) in project {hash}: {e}"); } } @@ -3809,32 +4107,37 @@ fn run_export_pr(task_id: &str) -> Result<()> { let meta = tj_core::db::task_metadata(&conn, task_id)?.unwrap_or_default(); let summary = meta.goal.unwrap_or_else(|| title.clone()); - // Pull all events ordered ASC so the PR description reads like a - // narrative (oldest decision first → newest). + // Changes / Why list exactly the pack's Active decisions / Rejected, + // reversed so the PR description reads like a narrative (oldest first). + let one_lines = |choices: Vec<tj_core::pack::Choice>| -> Vec<String> { + choices + .iter() + .rev() + .filter_map(|c| { + let one_line = c.text.lines().next().unwrap_or("").trim(); + (!one_line.is_empty()).then(|| format!("{one_line}{}", c.marker())) + }) + .collect() + }; + let decisions = one_lines(tj_core::pack::active_decisions(&conn, task_id)?); + let rejections = one_lines(tj_core::pack::rejections(&conn, task_id)?); + let mut stmt = conn.prepare( - "SELECT ei.type, sf.text FROM events_index ei + "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 ORDER BY ei.timestamp ASC", + WHERE ei.task_id = ?1 AND ei.type = 'evidence' AND ei.corrected_by IS NULL + ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { - let ty: String = r.get(0)?; - let txt: Option<String> = r.get(1)?; - Ok((ty, txt.unwrap_or_default())) + let txt: Option<String> = r.get(0)?; + Ok(txt.unwrap_or_default()) })?; - let mut decisions: Vec<String> = Vec::new(); - let mut rejections: Vec<String> = Vec::new(); let mut evidence: Vec<String> = Vec::new(); for row in rows { - let (ty, text) = row?; + let text = row?; let one_line: String = text.lines().next().unwrap_or("").trim().to_string(); - if one_line.is_empty() { - continue; - } - match ty.as_str() { - "decision" => decisions.push(one_line), - "rejection" => rejections.push(one_line), - "evidence" => evidence.push(one_line), - _ => {} + if !one_line.is_empty() { + evidence.push(one_line); } } @@ -4061,31 +4364,35 @@ fn run_export_memory(task: Option<&str>, _all_closed: bool, dry_run: bool) -> Re )?; let meta = tj_core::db::task_metadata(&conn, id)?.unwrap_or_default(); - // decision + constraint one-liners, oldest-first (== run_export_pr style). + // One-liners, oldest-first (== run_export_pr style). Decisions are the + // pack's Active decisions; constraints skip bookkeeping (model + // switches) and corrected ones by the same rules. + let decisions: Vec<String> = tj_core::pack::active_decisions(&conn, id)? + .iter() + .rev() + .filter_map(|c| { + let line = c.text.lines().next().unwrap_or("").trim(); + (!line.is_empty()).then(|| format!("{line}{}", c.marker())) + }) + .take(MAX_ITEMS) + .collect(); + let mut stmt = conn.prepare( - "SELECT ei.type, sf.text FROM events_index ei + "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 AND ei.type IN ('decision','constraint') + WHERE ei.task_id = ?1 AND ei.type = 'constraint' + AND ei.bookkeeping = 0 AND ei.corrected_by IS NULL ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![id], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option<String>>(1)?.unwrap_or_default(), - )) + Ok(r.get::<_, Option<String>>(0)?.unwrap_or_default()) })?; - let mut decisions = Vec::new(); let mut constraints = Vec::new(); for row in rows { - let (ty, text) = row?; + let text = row?; let line = text.lines().next().unwrap_or("").trim().to_string(); - if line.is_empty() { - continue; - } - match ty.as_str() { - "decision" if decisions.len() < MAX_ITEMS => decisions.push(line), - "constraint" if constraints.len() < MAX_ITEMS => constraints.push(line), - _ => {} + if !line.is_empty() && constraints.len() < MAX_ITEMS { + constraints.push(line); } } @@ -4114,15 +4421,26 @@ fn run_export_memory(task: Option<&str>, _all_closed: bool, dry_run: bool) -> Re /// the classifier prompt. Kept small so the prompt stays bounded. const CONSTRAINT_CONTEXT_LIMIT: i64 = 5; +/// The open tasks a hook works with, newest first — except that the task +/// this session last wrote to comes first, so two sessions in one project +/// each see (and classify into) their own task, not just the newest one. fn recent_task_contexts( conn: &rusqlite::Connection, + project_hash: &str, limit: usize, + session: Option<&str>, ) -> anyhow::Result<Vec<tj_core::classifier::TaskContext>> { + let own = match session { + Some(sid) => tj_core::db::active_task_for_session(conn, project_hash, sid)?, + None => None, + }; + let mut stmt = conn.prepare( - "SELECT task_id, title FROM tasks WHERE status='open' ORDER BY last_event_at DESC LIMIT ?1", + "SELECT task_id, title FROM tasks WHERE status='open' + ORDER BY (task_id = ?2) DESC, last_event_at DESC LIMIT ?1", )?; let task_rows: Vec<(String, String)> = stmt - .query_map(rusqlite::params![limit as i64], |r| { + .query_map(rusqlite::params![limit as i64, own], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) })? .collect::<Result<_, _>>()?; @@ -4153,17 +4471,22 @@ fn recent_task_contexts( "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id WHERE ei.task_id = ?1 AND ei.type = 'constraint' + AND COALESCE(sf.text, '') NOT LIKE ?3 ORDER BY ei.timestamp DESC LIMIT ?2", )?; + let model_switch = format!("{}%", tj_core::reminder::MODEL_SWITCH_TEXT_PREFIX); let constraints: Vec<String> = c_stmt - .query_map(rusqlite::params![task_id, CONSTRAINT_CONTEXT_LIMIT], |r| { - let txt: Option<String> = r.get(0)?; - Ok(txt - .unwrap_or_default() - .chars() - .take(120) - .collect::<String>()) - })? + .query_map( + rusqlite::params![task_id, CONSTRAINT_CONTEXT_LIMIT, model_switch], + |r| { + let txt: Option<String> = r.get(0)?; + Ok(txt + .unwrap_or_default() + .chars() + .take(120) + .collect::<String>()) + }, + )? .collect::<Result<Vec<String>, _>>()? .into_iter() .filter(|s| !s.is_empty()) @@ -4241,11 +4564,27 @@ fn count_session_events_tail(path: &std::path::Path, sid: &str, tail_lines: usiz .count() } +/// True when the Claude Code mod (`plugin/hooks/register.ts`) runs in this +/// session: it sets `TJ_MOD_ACTIVE` for every hook it starts. Codex never does. +fn mod_active() -> bool { + std::env::var("TJ_MOD_ACTIVE").is_ok_and(|v| !v.is_empty() && v != "0") +} + /// Adaptive UserPromptSubmit nudge (caveman pattern, non-blocking, free): always /// emit the base "record as you go" reminder, and — when the session has done /// substantial work but logged little — escalate. All signals are cheap (a file /// size + a tail scan); no model, never blocks the prompt. fn run_nudge() -> anyhow::Result<()> { + // Recursion guard, same as recall-hook: never inject into our own + // classifier child (`claude -p` / `codex exec` re-run the user's hooks). + if std::env::var(tj_core::classifier::agent_sdk::IN_CLASSIFIER_ENV).is_ok() { + return Ok(()); + } + // The Claude Code mod nudges by itself (after N turns without an entry). + if mod_active() { + return Ok(()); + } + let mut ctx = NUDGE_BASE.to_string(); let escalation = (|| -> Option<String> { use std::io::{IsTerminal, Read}; @@ -4303,7 +4642,10 @@ fn run_session_end_catchup( let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; - let Some(tc) = recent_task_contexts(&conn, 1)?.into_iter().next() else { + let Some(tc) = recent_task_contexts(&conn, project_hash, 1, live_session_id)? + .into_iter() + .next() + else { return Ok(()); }; let last_event_ts: Option<String> = conn @@ -4498,7 +4840,10 @@ fn run_dream_op( // 1. Resolve session files in scope. let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; let Some(project_dir) = project_dir else { - println!("dream: no Claude Code sessions found for this project"); + println!( + "dream: no Claude Code session directory for this project — dream mines \ +Claude Code transcripts only (Codex sessions are not read yet)" + ); return Ok(()); }; let session_paths = tj_core::session::discovery::list_sessions(&project_dir)?; @@ -4524,11 +4869,13 @@ fn run_dream_op( Some(tj_core::dream::scope::SessionFile { path: p, mtime }) }) .collect(); + let mtimes: std::collections::HashMap<std::path::PathBuf, std::time::SystemTime> = + scoped.iter().map(|s| (s.path.clone(), s.mtime)).collect(); let in_scope = tj_core::dream::scope::in_scope(scoped, since_time, limit); // 2. Assemble (session_id, BackfillInput) per session. let run_id = ulid::Ulid::new().to_string(); - let sessions = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; + let (sessions, unreadable) = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; let opts = tj_core::dream::DreamOptions { project_hash: project_hash.clone(), @@ -4561,16 +4908,42 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l &run_id, )?; - // 4. Advance watermark to now (only reached on success). - tj_core::dream::state::set_last_dream_at( - &conn, - &project_hash, - &chrono::Utc::now().to_rfc3339(), - )?; - println!( + // 4. Advance the watermark — only on an unscoped run (--task / --limit / + // --since skip sessions they never looked at), and only to the newest + // session such that it and every older in-scope one were mined cleanly. + if since.is_none() && task.is_none() && limit.is_none() { + let mined: Vec<(std::time::SystemTime, bool)> = in_scope + .iter() + .map(|p| { + let id = p.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + let clean = !unreadable.iter().any(|u| u == id) + && !report.failed_sessions.iter().any(|f| f == id); + (mtimes[p], clean) + }) + .collect(); + if let Some(t) = tj_core::dream::scope::next_watermark(&mined) { + let at = chrono::DateTime::<chrono::Utc>::from(t).to_rfc3339(); + tj_core::dream::state::set_last_dream_at(&conn, &project_hash, &at)?; + } + } + + let mut summary = format!( "dream: {} session(s) processed, {} event(s) backfilled", report.sessions_processed, report.events_backfilled ); + if report.events_dropped_unknown_task > 0 { + summary.push_str(&format!( + ", {} dropped (unknown task id)", + report.events_dropped_unknown_task + )); + } + if !report.failed_sessions.is_empty() { + summary.push_str(&format!( + ", {} only partly mined (retried next run)", + report.failed_sessions.len() + )); + } + println!("{summary}"); Ok(()) } @@ -4702,7 +5075,7 @@ fn task_sessions( }) .collect(); let in_scope = tj_core::dream::scope::in_scope(scoped, None, None); - build_dream_inputs(events_path, &in_scope, Some(task_id)) + Ok(build_dream_inputs(events_path, &in_scope, Some(task_id))?.0) } /// Enrich a single task from every session that touched it. Unlike `dream`, @@ -4930,6 +5303,23 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l ); } +/// The Claude Code session dir that `--enrich` reads. With `--enrich` and no +/// such dir, say why nothing gets enriched instead of staying silent. +fn complete_project_dir( + cwd: &std::path::Path, + enrich: bool, +) -> anyhow::Result<Option<std::path::PathBuf>> { + let dir = tj_core::session::discovery::find_project_dir(cwd)?; + if enrich && dir.is_none() { + eprintln!( + "complete: no Claude Code session directory for this project — --enrich reads \ +Claude Code transcripts only (Codex sessions are not read yet)" + ); + } + + Ok(dir) +} + /// `complete <id>` — finalize a single task. fn run_complete_single( task_id: &str, @@ -4948,7 +5338,7 @@ fn run_complete_single( if !tj_core::db::task_exists(&conn, task_id)? { anyhow::bail!("task not found: {task_id}"); } - let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; + let project_dir = complete_project_dir(&cwd, enrich)?; let ctx = ProjectCtx { conn: &conn, events_path: &events_path, @@ -4994,7 +5384,7 @@ fn run_complete_batch( return Ok(()); } - let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; + let project_dir = complete_project_dir(&cwd, enrich)?; // Show the numbered list with event/session counts so the user can judge // what to keep before anything is touched. @@ -5254,6 +5644,7 @@ fn auto_open_task_from_prompt( project_hash: &str, conn: &rusqlite::Connection, prompt: &str, + session_id: Option<&str>, ) -> anyhow::Result<Option<tj_core::classifier::TaskContext>> { // Title/goal must read like a human wrote them on purpose. When the // prompt is only machine noise — session-start scrollback @@ -5266,41 +5657,20 @@ fn auto_open_task_from_prompt( }; let goal: String = tj_core::title::humanize_goal(prompt, 200).unwrap_or_else(|| title.clone()); - let task_id = tj_core::new_task_id(); - let mut event = tj_core::event::Event::new( - task_id.clone(), - tj_core::event::EventType::Open, - tj_core::event::Author::User, - tj_core::event::Source::Cli, - title.clone(), - ); - event.meta = serde_json::json!({ "title": title, "auto_opened": true }); - - let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - - tj_core::db::ingest_new_events(conn, events_path, project_hash)?; - if !goal.is_empty() { - tj_core::db::set_task_goal(conn, &task_id, &goal)?; - } - // v0.5.0 Phase C / v0.6.0: score-based linking. Pull artifacts // from the prompt — ticket ids, commit hashes, file paths — then // ask the journal which prior tasks share enough signal to be a // probable continuation. Anything with score > 0 gets linked via // External; the strongest closed match also triggers a stderr // hint so the user can reopen instead of accumulating duplicates. + // Resolved before the open event is written so the links ride in it. + let mut linked: Vec<String> = Vec::new(); let prompt_arts = tj_core::artifacts::extract(prompt); if !prompt_arts.is_empty() { let related = tj_core::db::find_related_tasks(conn, &prompt_arts)?; let mut warned = false; for r in related.iter().take(5) { - if r.task_id == task_id { - continue; - } - let _ = - tj_core::db::add_task_external(conn, &task_id, &format!("linked:{}", r.task_id)); + linked.push(format!("linked:{}", r.task_id)); if !warned && r.status == "closed" { eprintln!( "task-journal: this prompt looks like a continuation of closed task {} \ @@ -5312,6 +5682,30 @@ fn auto_open_task_from_prompt( } } + let task_id = tj_core::new_task_id(); + let mut event = tj_core::event::Event::new( + task_id.clone(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + title.clone(), + ); + // Goal and links ride in the open event, so a rebuild restores them. + event.meta = serde_json::json!({ "title": title, "auto_opened": true }); + if !goal.is_empty() { + event.meta["goal"] = serde_json::Value::String(goal); + } + if !linked.is_empty() { + event.meta["external"] = serde_json::json!(linked); + } + tj_core::session_id::stamp_session_id(&mut event.meta, session_id); + + let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(conn, events_path, project_hash)?; + Ok(Some(tj_core::classifier::TaskContext { task_id, title, @@ -5320,7 +5714,13 @@ fn auto_open_task_from_prompt( })) } -fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyhow::Result<()> { +fn persist_pending( + events_path: &std::path::Path, + project_hash: &str, + kind: &str, + text: &str, + err: &str, +) -> anyhow::Result<()> { let pending_dir = events_path .parent() .unwrap() @@ -5329,9 +5729,10 @@ fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyh .join("pending"); std::fs::create_dir_all(&pending_dir)?; let id = ulid::Ulid::new().to_string(); - let payload = serde_json::json!({"text": text, "error": err, "queued_at": chrono::Utc::now().to_rfc3339()}); + // `kind` lets `pending retry` classify the chunk the way the hook would. + let payload = serde_json::json!({"kind": kind, "text": text, "error": err, "queued_at": chrono::Utc::now().to_rfc3339()}); std::fs::write( - pending_dir.join(format!("{id}.json")), + pending_dir.join(format!("{project_hash}.{id}.json")), serde_json::to_string_pretty(&payload)?, )?; Ok(()) @@ -5340,41 +5741,6 @@ fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyh /// v0.6.2: queue an ingest event for the detached classify-worker. The /// hook returns immediately after writing this entry so it does not /// block Claude Code's hook timeout (was 5-30s, now <100ms). Schema "v2" -/// Threshold for the v0.10.0 asyncRewake backlog signal. When the -/// PostToolUse hook (configured with `asyncRewake: true` in -/// `hooks.json`) finds more than this many entries already queued -/// in `pending/`, it exits with code 2 to wake the model with a -/// system reminder pointing at `task-journal pending-gc`. Tuned so -/// that normal load (<5 in-flight at any moment) never trips, but -/// a stuck classifier surfaces visibly before the queue grows into -/// the hundreds (the v0.6.2 fork-bomb era saw 515 entries before a -/// user noticed). -const PENDING_OVERFLOW_THRESHOLD: usize = 25; - -/// Count `.json` (and `.json.dead`) entries currently sitting in -/// `pending/` next to `events_path`. Best-effort: any IO error -/// returns 0 so a borked filesystem never wakes the model with -/// noise. Used by the asyncRewake backlog signal. -fn count_pending_entries(events_path: &std::path::Path) -> anyhow::Result<usize> { - let dir = events_path - .parent() - .and_then(|p| p.parent()) - .ok_or_else(|| anyhow::anyhow!("events_path has no grandparent"))? - .join("pending"); - if !dir.exists() { - return Ok(0); - } - let mut count = 0usize; - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if let Some("json") = path.extension().and_then(|e| e.to_str()) { - count += 1; - } - } - Ok(count) -} - /// distinguishes async-ingest entries from legacy v1 (text+error) ones /// the `pending retry` path knows how to handle. fn persist_pending_v2( @@ -5405,7 +5771,9 @@ fn persist_pending_v2( if let Some(sid) = session_id { payload["session_id"] = serde_json::Value::String(sid.to_string()); } - let path = pending_dir.join(format!("{id}.json")); + // `pending/` is shared by every project: the hash prefix tells each + // project's worker which entries are its own. + let path = pending_dir.join(format!("{project_hash}.{id}.json")); std::fs::write(&path, serde_json::to_string_pretty(&payload)?)?; Ok(path) } @@ -5492,83 +5860,36 @@ fn spawn_classify_worker(backend: &str) -> anyhow::Result<()> { Ok(()) } -/// File-lock guard for the classify-worker. Holds the lockfile until -/// dropped; ensures cleanup on panic. One worker per project_hash. -struct WorkerLock { - path: std::path::PathBuf, -} - -impl WorkerLock { - /// Try to acquire the lock. Returns Ok(Some(_)) on success, Ok(None) - /// if another live worker holds it, Err on filesystem failure. - fn try_acquire(project_hash: &str) -> anyhow::Result<Option<Self>> { - let dir = tj_core::paths::state_dir()?; - std::fs::create_dir_all(&dir)?; - let path = dir.join(format!("classifier-{project_hash}.lock")); - - loop { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&path) - { - Ok(mut f) => { - use std::io::Write; - let _ = writeln!(f, "{}", std::process::id()); - return Ok(Some(Self { path })); - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Inspect existing lockfile. If PID is alive → another - // worker is running; back off. If dead/missing → - // remove stale file and retry. - let body = std::fs::read_to_string(&path).unwrap_or_default(); - let pid: Option<u32> = body.trim().parse().ok(); - if let Some(pid) = pid { - if pid_is_alive(pid) { - return Ok(None); - } - } - // Stale (no PID, or dead PID) — remove and retry. - let _ = std::fs::remove_file(&path); - continue; - } - Err(e) => return Err(e.into()), - } - } - } -} - -impl Drop for WorkerLock { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } -} - -#[cfg(unix)] -fn pid_is_alive(pid: u32) -> bool { - // kill(pid, 0) probes existence without sending a signal. - // SAFETY: libc::kill is a thin syscall wrapper, no aliasing concerns. - unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } -} - -#[cfg(not(unix))] -fn pid_is_alive(_pid: u32) -> bool { - // Conservative on non-Unix: assume alive so we don't double-spawn. - // The lockfile gets cleaned up on Drop in the normal exit path. - true -} +/// Characters of a PostToolUse chunk (tool input + response) that get queued. +const POST_TOOL_USE_TEXT_MAX: usize = 2000; /// classify-worker: drain pending v2 entries by running the real /// classifier. v1 entries (legacy text+error shape) are left for -/// `pending retry`. Holds a project-scoped file lock so only one +/// `pending retry`. Holds a project-scoped OS lock so only one /// worker per project runs at a time. fn run_classify_worker(backend: &str) -> Result<()> { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; - let lock = match WorkerLock::try_acquire(&project_hash)? { - Some(l) => l, - None => return Ok(()), // another worker is running + // An advisory lock held for the worker's lifetime. The OS releases it + // when the process exits or dies, so a leftover file never blocks the + // next worker and needs no stale-pid check. The file is never deleted: + // a worker could lock the unlinked file while another creates and + // locks a fresh one. + let state_dir = tj_core::paths::state_dir()?; + std::fs::create_dir_all(&state_dir)?; + let lock_file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(state_dir.join(format!("classifier-{project_hash}.lock")))?; + let mut lock = fd_lock::RwLock::new(lock_file); + let _held = match lock.try_write() { + Ok(guard) => guard, + // Another worker of this project is running. + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(e) => return Err(e.into()), }; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); @@ -5578,20 +5899,33 @@ fn run_classify_worker(backend: &str) -> Result<()> { .ok_or_else(|| anyhow::anyhow!("events_dir has no grandparent"))? .join("pending"); if !pending.exists() { - drop(lock); return Ok(()); } - // Snapshot entries up front so concurrent re-queues don't loop us. - let mut entries: Vec<std::path::PathBuf> = Vec::new(); - for e in std::fs::read_dir(&pending)? { - let e = e?; - let p = e.path(); - if p.extension().and_then(|s| s.to_str()) == Some("json") { - entries.push(p); + // A worker that died mid-entry left it claimed. We hold the lock, so no + // other worker of this project is on it: put it back in the queue. + for claimed in std::fs::read_dir(&pending)?.flatten().map(|e| e.path()) { + let Some(name) = claimed + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".processing")) + else { + continue; + }; + let ours = std::fs::read_to_string(&claimed) + .ok() + .and_then(|b| serde_json::from_str::<serde_json::Value>(&b).ok()) + .is_some_and(|v| v["project_hash"] == project_hash.as_str()); + if ours { + let _ = std::fs::rename(&claimed, pending.join(name)); } } + // Snapshot entries up front so concurrent re-queues don't loop us. + // Only this project's entries: the lock is per project, so a worker + // of another project may be draining the same directory right now. + let entries = project_pending_entries(&pending, &project_hash)?; + for path in entries { if let Err(err) = process_pending_entry(&path, &events_path, &project_hash, backend) { // Non-fatal: leave the file in place; pending-retry / next @@ -5601,7 +5935,6 @@ fn run_classify_worker(backend: &str) -> Result<()> { } } - drop(lock); Ok(()) } @@ -5620,6 +5953,11 @@ fn process_pending_entry( if schema != "v2" { return Ok(()); // legacy entry, handled by `pending retry` } + // A legacy un-prefixed entry is ours only when it names this project; + // one without `project_hash` is left for `pending retry`. + if v.get("project_hash").and_then(|x| x.as_str()) != Some(project_hash) { + return Ok(()); + } let kind = v .get("kind") @@ -5635,41 +5973,54 @@ fn process_pending_entry( // Inherit the session id queued on the v2 chunk (additive; absent → None). let chunk_session_id = tj_core::session_id::session_id_from_payload(&v); - // Mirror the synchronous flow that used to live in IngestHook — - // see commit history of v0.6.1 for the original. Auto-open, run - // classifier, apply integrity safeguards, persist event, telemetry. - let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + // Claim the entry before the classifier call: a rename succeeds for one + // process only, so an entry is never classified (and recorded) twice. + let claimed = path.with_extension("json.processing"); + if std::fs::rename(path, &claimed).is_err() { + return Ok(()); // someone else took it } - let mut recent = recent_task_contexts(&conn, 5)?; - if recent.is_empty() { - let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") - .ok() - .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) - .unwrap_or(false); - if auto_open_disabled || !kind.contains("UserPrompt") { - // Nothing to do — drop the entry silently. - std::fs::remove_file(path)?; - return Ok(()); + let classified = (|| -> anyhow::Result<()> { + let classifier = build_classifier(backend)?; + let outcome = classify_chunk( + classifier.as_ref(), + events_path, + project_hash, + &kind, + &text, + chunk_session_id.as_deref(), + )?; + if let ChunkOutcome::Unplaced(err) = outcome { + // Persist as legacy v1 pending entry so `pending retry` + // surfaces it; remove the v2 source. + persist_pending(events_path, project_hash, &kind, &text, &err)?; } - let Some(new_task) = auto_open_task_from_prompt(events_path, project_hash, &conn, &text)? - else { - // Prompt was only machine noise — drop the entry silently. - std::fs::remove_file(path)?; - return Ok(()); - }; - recent.push(new_task); + Ok(()) + })(); + if classified.is_err() { + // Back in the queue under its own name for the next worker. + let _ = std::fs::rename(&claimed, path); + return classified; } - let author_hint = if kind.contains("UserPrompt") { - "user" - } else { - "assistant" - }; + std::fs::remove_file(&claimed)?; + Ok(()) +} +/// What became of one classified chunk. +enum ChunkOutcome { + /// An event was written to the journal. + Recorded, + /// Nothing worth recording: no task to attach to, machine noise, no + /// task guess, or a session-end "close". + Dropped, + /// The classifier failed or guessed a missing / closed task; the reason + /// goes back into `pending/` with the chunk. + Unplaced(String), +} + +/// The classifier behind a `--backend` name. +fn build_classifier(backend: &str) -> anyhow::Result<Box<dyn tj_core::classifier::Classifier>> { use tj_core::classifier::Classifier; let classifier: Box<dyn Classifier> = match backend { "hybrid" | "" => Box::new(tj_core::classifier::hybrid::HybridClassifier::from_env()), @@ -5701,50 +6052,83 @@ fn process_pending_entry( "unknown backend: {other} (expected `hybrid`, `agent-sdk`, `api`, or `heuristic`)" ), }; + Ok(classifier) +} + +/// Classify one chunk against the project's open tasks and record the +/// event. Shared by classify-worker and `pending retry`, so both auto-open, +/// check attribution, stamp the session and write telemetry the same way. +fn classify_chunk( + classifier: &dyn tj_core::classifier::Classifier, + events_path: &std::path::Path, + project_hash: &str, + kind: &str, + text: &str, + session_id: Option<&str>, +) -> anyhow::Result<ChunkOutcome> { + // Mirror the synchronous flow that used to live in IngestHook — + // see commit history of v0.6.1 for the original. Auto-open, run + // classifier, apply integrity safeguards, persist event, telemetry. + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + } + + let mut recent = recent_task_contexts(&conn, project_hash, 5, session_id)?; + if recent.is_empty() { + let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") + .ok() + .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) + .unwrap_or(false); + if auto_open_disabled || !kind.contains("UserPrompt") { + // Nothing to do — drop the entry silently. + return Ok(ChunkOutcome::Dropped); + } + let Some(new_task) = + auto_open_task_from_prompt(events_path, project_hash, &conn, text, session_id)? + else { + // Prompt was only machine noise — drop the entry silently. + return Ok(ChunkOutcome::Dropped); + }; + recent.push(new_task); + } + + let author_hint = if kind.contains("UserPrompt") { + "user" + } else { + "assistant" + }; + let input = tj_core::classifier::ClassifyInput { - text: text.clone(), + text: text.to_string(), author_hint: author_hint.into(), recent_tasks: recent, + tool_output: kind == "PostToolUse", }; let out = match classifier.classify(&input) { Ok(o) => o, - Err(e) => { - // Persist as legacy v1 pending entry so `pending retry` - // surfaces it; remove the v2 source. - persist_pending(events_path, &text, &e.to_string())?; - std::fs::remove_file(path)?; - return Ok(()); - } + Err(e) => return Ok(ChunkOutcome::Unplaced(e.to_string())), }; let Some(tid) = out.task_id_guess else { - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Dropped); }; use tj_core::event::EventType; if matches!(out.event_type, EventType::Close) && kind == "Stop" { - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Dropped); } match tj_core::db::task_status(&conn, &tid)? { None => { - persist_pending( - events_path, - &text, - &format!("task_id_guess `{tid}` not found"), - )?; - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Unplaced(format!( + "task_id_guess `{tid}` not found" + ))) } Some(s) if s == "closed" => { - persist_pending( - events_path, - &text, - &format!("task_id_guess `{tid}` is closed"), - )?; - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Unplaced(format!( + "task_id_guess `{tid}` is closed" + ))) } _ => {} } @@ -5764,7 +6148,7 @@ fn process_pending_entry( event.confidence = Some(confidence); event.status = tj_core::classifier::decide_status(confidence); event.evidence_strength = evidence_strength; - tj_core::session_id::stamp_session_id(&mut event.meta, chunk_session_id.as_deref()); + tj_core::session_id::stamp_session_id(&mut event.meta, session_id); let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; writer.append(&event)?; @@ -5792,33 +6176,32 @@ fn process_pending_entry( }, ); - std::fs::remove_file(path)?; - Ok(()) + Ok(ChunkOutcome::Recorded) } +/// Mock-only drain: with the mock flags, turn this project's legacy (v1) +/// pending entries into events. Without them it does nothing — a v1 entry is +/// a classifier failure waiting for `pending retry`, and v2 entries belong to +/// classify-worker. An entry is removed only after its event is written. fn drain_pending( events_path: &std::path::Path, + project_hash: &str, mock_etype: Option<&str>, mock_tid: Option<&str>, mock_conf: Option<f64>, ) -> anyhow::Result<()> { + let (Some(t), Some(tid)) = (mock_etype, mock_tid) else { + return Ok(()); + }; let pending_dir = events_path .parent() .unwrap() .parent() .unwrap() .join("pending"); - if !pending_dir.exists() { - return Ok(()); - } - - for entry in std::fs::read_dir(&pending_dir)? { - let entry = entry?; - if entry.path().extension().and_then(|e| e.to_str()) != Some("json") { - continue; - } - let body = std::fs::read_to_string(entry.path())?; + for path in project_pending_entries(&pending_dir, project_hash)? { + let body = std::fs::read_to_string(&path)?; let v: serde_json::Value = serde_json::from_str(&body)?; // v0.6.2: skip v2 entries — those are owned by classify-worker. // Removing them here would silently drop async-queued events. @@ -5830,23 +6213,24 @@ fn drain_pending( .and_then(|x| x.as_str()) .unwrap_or("") .to_string(); - if !text.is_empty() { - if let (Some(t), Some(tid)) = (mock_etype, mock_tid) { - let mut event = tj_core::event::Event::new( - tid, - parse_event_type(t)?, - tj_core::event::Author::Classifier, - tj_core::event::Source::Hook, - text, - ); - event.confidence = mock_conf; - event.status = tj_core::classifier::decide_status(mock_conf.unwrap_or(1.0)); - let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - } + if text.is_empty() { + continue; } - std::fs::remove_file(entry.path())?; + + let mut event = tj_core::event::Event::new( + tid, + parse_event_type(t)?, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text, + ); + event.confidence = mock_conf; + event.status = tj_core::classifier::decide_status(mock_conf.unwrap_or(1.0)); + let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + std::fs::remove_file(&path)?; } Ok(()) } @@ -5879,6 +6263,7 @@ fn push_recall_envelope( payload: &serde_json::Value, events_path: &std::path::Path, project_hash: &str, + session_id: Option<&str>, ) -> Option<serde_json::Value> { // MCP-only gate: Claude Code prefixes MCP tools `mcp__<server>__<tool>`. let tool_name = payload.get("tool_name").and_then(|v| v.as_str())?; @@ -5909,6 +6294,7 @@ fn push_recall_envelope( let hits = tj_core::recall::relevant_recall(&conn, &query_text, tj_core::recall::DEFAULT_MAX_HITS) .ok()?; + let hits = fresh_recall_hits(hits, session_id, events_path, project_hash); if hits.is_empty() { return None; } @@ -5921,6 +6307,83 @@ fn push_recall_envelope( })) } +/// Most `<session> <event_id>` lines the shown-recall log keeps. +const RECALL_SHOWN_CAP: usize = 2000; + +/// The recall hits still worth pushing to `session_id`: each one at most once +/// per session (remembered in `<state_dir>/<project>.recall-shown`, newest +/// [`RECALL_SHOWN_CAP`] lines), and never an event the session wrote on its +/// current task — the agent just wrote it. Without a session id every hit +/// passes, as before. Best-effort: an unreadable log never hides a hit. +fn fresh_recall_hits( + hits: Vec<tj_core::recall::RecallHit>, + session_id: Option<&str>, + events_path: &std::path::Path, + project_hash: &str, +) -> Vec<tj_core::recall::RecallHit> { + let Some(sid) = session_id else { + return hits; + }; + if hits.is_empty() { + return hits; + } + let Ok(log) = + tj_core::paths::state_dir().map(|d| d.join(format!("{project_hash}.recall-shown"))) + else { + return hits; + }; + + let own = session_task_event_ids(events_path, sid); + let body = std::fs::read_to_string(&log).unwrap_or_default(); + let shown: std::collections::HashSet<&str> = body.lines().collect(); + let fresh: Vec<_> = hits + .into_iter() + .filter(|h| !own.contains(&h.event_id)) + .filter(|h| !shown.contains(format!("{sid} {}", h.event_id).as_str())) + .collect(); + if fresh.is_empty() { + return fresh; + } + + // ponytail: read-modify-write without a lock; two parallel hooks can + // re-show a hit once. Add a file lock if that ever shows up in practice. + let mut lines: Vec<String> = body.lines().map(str::to_string).collect(); + lines.extend(fresh.iter().map(|h| format!("{sid} {}", h.event_id))); + let keep = &lines[lines.len().saturating_sub(RECALL_SHOWN_CAP)..]; + let _ = std::fs::write(&log, keep.join("\n") + "\n"); + + fresh +} + +/// Ids of the events session `sid` wrote on its current task — the task of +/// its latest event — read from the journal by `meta.session_id`. +fn session_task_event_ids( + events_path: &std::path::Path, + sid: &str, +) -> std::collections::HashSet<String> { + let body = std::fs::read_to_string(events_path).unwrap_or_default(); + let mine: Vec<(String, String)> = body + .lines() + .filter(|l| l.contains(sid)) + .filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok()) + .filter(|e| e["meta"]["session_id"].as_str() == Some(sid)) + .filter_map(|e| { + Some(( + e["event_id"].as_str()?.into(), + e["task_id"].as_str()?.into(), + )) + }) + .collect(); + + let Some((_, current)) = mine.last() else { + return Default::default(); + }; + mine.iter() + .filter(|(_, task)| task == current) + .map(|(id, _)| id.clone()) + .collect() +} + /// One ⚠ line per recall hit (mirrors the close-gate / SessionStart convention). fn render_recall_banner(hits: &[tj_core::recall::RecallHit]) -> String { let mut s = String::from("\u{26a0} Task Journal recall — you may be repeating a prior path:"); @@ -6064,26 +6527,18 @@ fn task_matches_session( }) } -/// Read the project's events from `events_path`, group by `task_id`, and -/// return candidate task contexts for sessions whose events match this -/// session (precise session_id, or legacy time-window). Each context -/// carries the task title and up to the last ~20 event texts (dedup -/// context for the backend). -fn candidate_tasks_for_session( +/// Read the project's events from `events_path`, grouped by `task_id`. +/// Read once per run and shared by every session's candidate lookup. +fn events_by_task( events_path: &std::path::Path, - session_id: &str, - first_ts: Option<&str>, - last_ts: Option<&str>, -) -> anyhow::Result<Vec<tj_core::dream::backend::BackfillTaskContext>> { - use std::collections::BTreeMap; - use tj_core::dream::backend::BackfillTaskContext; - use tj_core::event::{Event, EventType}; +) -> anyhow::Result<std::collections::BTreeMap<String, Vec<tj_core::event::Event>>> { + use tj_core::event::Event; + let mut by_task = std::collections::BTreeMap::new(); if !events_path.exists() { - return Ok(Vec::new()); + return Ok(by_task); } let body = std::fs::read_to_string(events_path)?; - let mut by_task: BTreeMap<String, Vec<Event>> = BTreeMap::new(); for line in body.lines() { if line.trim().is_empty() { continue; @@ -6092,10 +6547,25 @@ fn candidate_tasks_for_session( by_task.entry(e.task_id.clone()).or_default().push(e); } } + Ok(by_task) +} + +/// Candidate task contexts for the tasks whose events match this session +/// (precise session_id, or legacy time-window). Each context carries the +/// task title and up to the last ~20 event texts (dedup context for the +/// backend). +fn candidate_tasks_for_session( + by_task: &std::collections::BTreeMap<String, Vec<tj_core::event::Event>>, + session_id: &str, + first_ts: Option<&str>, + last_ts: Option<&str>, +) -> Vec<tj_core::dream::backend::BackfillTaskContext> { + use tj_core::dream::backend::BackfillTaskContext; + use tj_core::event::EventType; let mut out = Vec::new(); for (task_id, events) in by_task { - if !task_matches_session(&events, session_id, first_ts, last_ts) { + if !task_matches_session(events, session_id, first_ts, last_ts) { continue; } // Title from the Open event when present, else the first event's text. @@ -6113,39 +6583,56 @@ fn candidate_tasks_for_session( .map(|e| e.text.clone()) .collect(); out.push(BackfillTaskContext { - task_id, + task_id: task_id.clone(), title, existing_events, }); } - Ok(out) + out } +/// Per-session `(session_id, BackfillInput)` pairs fed to `run_dream`. +type DreamInputs = Vec<(String, tj_core::dream::backend::BackfillInput)>; + /// Assemble per-session `(session_id, BackfillInput)` from the in-scope -/// session transcripts and the project's existing events. +/// session transcripts and the project's existing events. Also returns the +/// ids of sessions skipped as unreadable. fn build_dream_inputs( events_path: &std::path::Path, sessions: &[std::path::PathBuf], task_filter: Option<&str>, -) -> anyhow::Result<Vec<(String, tj_core::dream::backend::BackfillInput)>> { +) -> anyhow::Result<(DreamInputs, Vec<String>)> { use tj_core::dream::backend::BackfillInput; use tj_core::session::parser::parse_session; + let by_task = events_by_task(events_path)?; let mut out = Vec::new(); + let mut unreadable = Vec::new(); for path in sessions { let session_id = path .file_stem() .and_then(|s| s.to_str()) .unwrap_or("") .to_string(); - let parsed = parse_session(path)?; + // One unreadable transcript must not abort mining the rest. + let parsed = match parse_session(path) { + Ok(p) => p, + Err(e) => { + eprintln!( + "dream: skipping unreadable session {}: {e:#}", + path.display() + ); + unreadable.push(session_id); + continue; + } + }; let candidates = candidate_tasks_for_session( - events_path, + &by_task, &session_id, parsed.first_timestamp.as_deref(), parsed.last_timestamp.as_deref(), - )?; + ); let tasks: Vec<_> = candidates .into_iter() .filter(|t| task_filter.is_none_or(|f| f == t.task_id)) @@ -6157,7 +6644,7 @@ fn build_dream_inputs( let transcript = flatten_transcript(&parsed); out.push((session_id, BackfillInput { tasks, transcript })); } - Ok(out) + Ok((out, unreadable)) } #[cfg(test)] @@ -6309,6 +6796,61 @@ mod inline_tests { )); } + #[test] + fn candidate_tasks_come_from_events_loaded_once() { + // The events log is grouped once per run and reused for every + // session, so candidate lookup takes the grouped map, not a path. + use tj_core::event::{Author, Event, EventType, Source}; + let mut tagged = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "Task one".into(), + ); + tagged.meta = serde_json::json!({"session_id": "sess-1"}); + let by_task = std::collections::BTreeMap::from([("tj-1".to_string(), vec![tagged])]); + + let hit = candidate_tasks_for_session(&by_task, "sess-1", None, None); + let miss = candidate_tasks_for_session(&by_task, "sess-2", None, None); + + assert_eq!(hit.len(), 1); + assert_eq!(hit[0].task_id, "tj-1"); + assert_eq!(hit[0].title, "Task one"); + assert!(miss.is_empty()); + } + + #[test] + fn build_dream_inputs_skips_an_unreadable_session() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("h.jsonl"); + let mut ev = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "task".into(), + ); + ev.meta = serde_json::json!({"session_id": "good"}); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + writer.append(&ev).unwrap(); + writer.flush_durable().unwrap(); + + // Invalid UTF-8 makes parse_session fail for this one file. + let bad = dir.path().join("bad.jsonl"); + std::fs::write(&bad, [0xff, 0xfe, b'\n']).unwrap(); + let good = dir.path().join("good.jsonl"); + std::fs::write(&good, + "{\"type\":\"user\",\"uuid\":\"u1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{\"content\":\"hi\"}}\n").unwrap(); + + let (inputs, unreadable) = build_dream_inputs(&events_path, &[bad, good], None).unwrap(); + + assert_eq!(inputs.len(), 1); + assert_eq!(inputs[0].0, "good"); + assert_eq!(unreadable, vec!["bad".to_string()]); + } + #[test] fn persist_pending_v2_includes_session_id_when_present() { let dir = tempfile::tempdir().unwrap(); @@ -6344,6 +6886,59 @@ mod inline_tests { assert!(v.get("session_id").is_none()); } + #[test] + fn pending_entries_are_named_after_their_project() { + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + + persist_pending_v2(&events_path, "PostToolUse", "txt", "h", "hybrid", None).unwrap(); + persist_pending(&events_path, "h", "Stop", "txt", "err").unwrap(); + + let pending = dir.path().join("pending"); + let names: Vec<String> = std::fs::read_dir(&pending) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.iter().all(|n| n.starts_with("h.")), "{names:?}"); + assert_eq!( + project_pending_entries(&pending, "h").unwrap().len(), + 2, + "both entries belong to project h" + ); + assert!(project_pending_entries(&pending, "other") + .unwrap() + .is_empty()); + } + + #[test] + fn pending_entries_come_back_oldest_first() { + let dir = tempfile::tempdir().unwrap(); + // ULID order == queue order; legacy un-prefixed names interleave by + // their ULID, not by the hash prefix of the new names. + let mut expected = Vec::new(); + for i in 0..20u32 { + let ulid = format!("01JA{i:022}"); + let name = if i % 5 == 0 { + format!("{ulid}.json") + } else { + format!("h.{ulid}.json") + }; + expected.push(name); + } + for name in expected.iter().rev() { + std::fs::write(dir.path().join(name), "{}").unwrap(); + } + + let got: Vec<String> = project_pending_entries(dir.path(), "h") + .unwrap() + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(got, expected); + } + #[test] fn is_rewind_prompt_simple() { assert!(is_rewind_prompt("/rewind")); @@ -6366,6 +6961,51 @@ mod inline_tests { assert!(!is_rewind_prompt("/rewinder")); } + #[test] + fn recent_task_contexts_puts_the_sessions_own_task_first() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = dir.path().join("h.sqlite"); + + // s1 works on the older task; another session touched tj-2 later. + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + for (task, at, session) in [ + ("tj-1", "2026-01-01T00:00:00Z", Some("s1")), + ("tj-2", "2026-01-01T00:00:05Z", Some("s2")), + ] { + let mut open = Event::new( + task, + EventType::Open, + Author::Agent, + Source::Chat, + task.into(), + ); + open.timestamp = at.into(); + tj_core::session_id::stamp_session_id(&mut open.meta, session); + writer.append(&open).unwrap(); + } + writer.flush_durable().unwrap(); + + let conn = tj_core::db::open(&state_path).unwrap(); + tj_core::db::ingest_new_events(&conn, &events_path, "h").unwrap(); + + let first = |session| { + recent_task_contexts(&conn, "h", 2, session).unwrap()[0] + .task_id + .clone() + }; + assert_eq!(first(Some("s1")), "tj-1", "s1 sees its own task first"); + assert_eq!(first(Some("s2")), "tj-2"); + assert_eq!( + first(Some("s9")), + "tj-2", + "an unknown session falls back to the newest" + ); + assert_eq!(first(None), "tj-2"); + } + #[test] fn recent_task_contexts_gathers_constraints() { use tj_core::event::{Author, Event, EventType, Source}; @@ -6409,7 +7049,7 @@ mod inline_tests { let conn = tj_core::db::open(&state_path).unwrap(); tj_core::db::ingest_new_events(&conn, &events_path, project_hash).unwrap(); - let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); assert!( ctx.constraints @@ -6462,7 +7102,7 @@ mod inline_tests { let conn = tj_core::db::open(&state_path).unwrap(); tj_core::db::ingest_new_events(&conn, &events_path, project_hash).unwrap(); - let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); assert_eq!( ctx.constraints.len(), @@ -6484,6 +7124,59 @@ mod inline_tests { .any(|s| s.contains("constraint number 1"))); } + #[test] + fn recent_task_contexts_skips_model_switch_constraints() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = dir.path().join("h.sqlite"); + + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + let mut open = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "task one".into(), + ); + open.meta = serde_json::json!({ "title": "task one" }); + open.timestamp = "2026-01-01T00:00:00Z".into(); + writer.append(&open).unwrap(); + for i in 0..8 { + // Five real constraints, then three newer model switches. + let text = if i < 5 { + format!("constraint number {i}") + } else { + format!("Model switched (auto): opus → haiku {i}") + }; + let mut cons = Event::new( + "tj-1", + EventType::Constraint, + Author::Agent, + Source::Hook, + text, + ); + cons.timestamp = format!("2026-01-01T00:00:1{i}Z"); + writer.append(&cons).unwrap(); + } + writer.flush_durable().unwrap(); + + let conn = tj_core::db::open(&state_path).unwrap(); + tj_core::db::ingest_new_events(&conn, &events_path, "h").unwrap(); + + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); + let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); + assert_eq!(ctx.constraints.len(), 5, "{:?}", ctx.constraints); + assert!( + ctx.constraints + .iter() + .all(|s| s.starts_with("constraint number")), + "{:?}", + ctx.constraints + ); + } + #[test] fn topic_is_fts_safe_basic() { assert!(topic_is_fts_safe("oauth")); diff --git a/crates/tj-cli/src/tui/app.rs b/crates/tj-cli/src/tui/app.rs index 0845924..8518e64 100644 --- a/crates/tj-cli/src/tui/app.rs +++ b/crates/tj-cli/src/tui/app.rs @@ -10,13 +10,13 @@ use anyhow::Result; use crossterm::{ - event::{self, Event, KeyCode, KeyModifiers}, + event::{self, Event, KeyCode, KeyEvent, KeyEventKind, KeyModifiers}, execute, terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, }; use ratatui::{backend::CrosstermBackend, Terminal}; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use tj_core::session::{discovery, parser}; use super::chat_view::ChatView; @@ -42,6 +42,8 @@ pub struct App { pub session_list: Option<SessionList>, pub chat_view: Option<ChatView>, pub should_quit: bool, + /// The project this UI browses (`ui --project`, else cwd). + project_path: PathBuf, } impl App { @@ -68,6 +70,7 @@ impl App { session_list: None, chat_view: None, should_quit: false, + project_path: project_path.to_path_buf(), }) } @@ -108,6 +111,7 @@ impl App { session_list: Some(SessionList::new(items, project_str)), chat_view: None, should_quit: false, + project_path: project_path.to_path_buf(), }) } @@ -120,9 +124,14 @@ impl App { let result = self.main_loop(&mut terminal); - disable_raw_mode()?; - execute!(terminal.backend_mut(), LeaveAlternateScreen)?; - terminal.show_cursor()?; + run_restore_steps( + &mut terminal, + &[ + |_| disable_raw_mode(), + |t| execute!(t.backend_mut(), LeaveAlternateScreen), + |t| t.show_cursor(), + ], + )?; result } @@ -154,18 +163,7 @@ impl App { if event::poll(std::time::Duration::from_millis(100))? { if let Event::Key(key) = event::read()? { - if key.modifiers.contains(KeyModifiers::CONTROL) - && key.code == KeyCode::Char('c') - { - self.should_quit = true; - } - - match &self.screen { - Screen::TaskList => self.handle_task_list_input(key.code), - Screen::TaskDetail => self.handle_task_detail_input(key.code), - Screen::SessionList => self.handle_session_list_input(key.code), - Screen::Chat => self.handle_chat_input(key.code), - } + self.handle_key(key); } } @@ -176,6 +174,25 @@ impl App { Ok(()) } + fn handle_key(&mut self, key: KeyEvent) { + // Terminals that report key releases (Windows) would otherwise fire + // every key twice. + if key.kind != KeyEventKind::Press { + return; + } + + if key.modifiers.contains(KeyModifiers::CONTROL) && key.code == KeyCode::Char('c') { + self.should_quit = true; + } + + match &self.screen { + Screen::TaskList => self.handle_task_list_input(key.code), + Screen::TaskDetail => self.handle_task_detail_input(key.code), + Screen::SessionList => self.handle_session_list_input(key.code), + Screen::Chat => self.handle_chat_input(key.code), + } + } + fn handle_task_list_input(&mut self, key: KeyCode) { let Some(ref mut tl) = self.task_list else { return; @@ -226,8 +243,7 @@ impl App { // The state SQLite already exists (App::new opened it). // Re-resolve through paths to avoid storing the connection // on App and dealing with !Send across the render loop. - let cwd = std::env::current_dir()?; - let project_hash = tj_core::project_hash::from_path(&cwd)?; + let project_hash = tj_core::project_hash::from_path(&self.project_path)?; let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; // Full mode: show the complete reasoning chain — every event, @@ -325,3 +341,99 @@ impl App { } } } + +/// Terminal restore sequence on exit. Every step runs even when an earlier +/// one fails — a half-restored terminal (raw mode, alternate screen, hidden +/// cursor) is worse than a late error — and the first error is reported. +fn run_restore_steps<T>(target: &mut T, steps: &[fn(&mut T) -> io::Result<()>]) -> io::Result<()> { + let mut first = Ok(()); + + for step in steps { + let result = step(target); + if first.is_ok() { + first = result; + } + } + + first +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restore_runs_every_step_and_reports_the_first_error() { + let mut log: Vec<&str> = Vec::new(); + let err = run_restore_steps( + &mut log, + &[ + |log| { + log.push("raw mode"); + Err(io::Error::other("raw mode failed")) + }, + |log| { + log.push("alt screen"); + Err(io::Error::other("alt screen failed")) + }, + |log| { + log.push("cursor"); + Ok(()) + }, + ], + ) + .unwrap_err(); + + assert_eq!(log, ["raw mode", "alt screen", "cursor"]); + assert_eq!(err.to_string(), "raw mode failed"); + } + + #[test] + fn task_detail_reads_the_project_passed_to_ui_not_cwd() { + let data = tempfile::TempDir::new().unwrap(); + let proj = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(proj.path().join(".git")).unwrap(); + // The only test in this binary that resolves the data dir. + std::env::set_var("TASK_JOURNAL_DATA_DIR", data.path()); + + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + let state = tj_core::paths::state_dir() + .unwrap() + .join(format!("{hash}.sqlite")); + let conn = tj_core::db::open(&state).unwrap(); + let open = tj_core::event::Event::new( + "tj-ui1".to_string(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + "Picked via --project".to_string(), + ); + tj_core::db::upsert_task_from_event(&conn, &open, &hash).unwrap(); + tj_core::db::index_event(&conn, &open).unwrap(); + drop(conn); + + let app = App::new(proj.path()).unwrap(); + let pack = app.assemble_pack("tj-ui1").unwrap(); + assert!(pack.contains("Picked via --project"), "{pack}"); + } + + #[test] + fn key_release_events_are_ignored() { + let mut app = App { + screen: Screen::TaskList, + task_list: Some(TaskList::new(vec![], "proj".into())), + task_detail: None, + session_list: None, + chat_view: None, + should_quit: false, + project_path: PathBuf::from("proj"), + }; + let q = |kind| KeyEvent::new_with_kind(KeyCode::Char('q'), KeyModifiers::NONE, kind); + + app.handle_key(q(KeyEventKind::Release)); + assert!(!app.should_quit, "a key release must not act as a press"); + + app.handle_key(q(KeyEventKind::Press)); + assert!(app.should_quit); + } +} diff --git a/crates/tj-cli/src/tui/chat_view.rs b/crates/tj-cli/src/tui/chat_view.rs index 07adeb7..2f2a67f 100644 --- a/crates/tj-cli/src/tui/chat_view.rs +++ b/crates/tj-cli/src/tui/chat_view.rs @@ -118,7 +118,7 @@ impl ChatView { .unwrap_or(&clean); truncate(line.trim(), 60) } else { - format!("Session {}", &session.session_id[..8]) + format!("Session {}", super::short_session_id(&session.session_id)) }; ChatView { @@ -347,6 +347,31 @@ fn format_ts(ts: &str) -> String { mod tests { use super::*; + // --- from_session() title fallback --- + + fn session_with_id(id: &str) -> ParsedSession { + ParsedSession { + session_id: id.into(), + file_path: String::new(), + entries: vec![], + first_timestamp: None, + last_timestamp: None, + } + } + + #[test] + fn title_fallback_survives_short_session_id() { + let view = ChatView::from_session(&session_with_id("abc")); + assert_eq!(view.title, "Session abc"); + } + + #[test] + fn title_fallback_cuts_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let view = ChatView::from_session(&session_with_id("aééééé")); + assert_eq!(view.title, "Session aééééé"); + } + // --- word_wrap() tests --- #[test] diff --git a/crates/tj-cli/src/tui/mod.rs b/crates/tj-cli/src/tui/mod.rs index f11dd56..69732c8 100644 --- a/crates/tj-cli/src/tui/mod.rs +++ b/crates/tj-cli/src/tui/mod.rs @@ -6,3 +6,8 @@ pub mod chat_view; pub mod session_list; pub mod task_detail; pub mod task_list; + +/// First 8 characters of a session id, cut on a char boundary. +fn short_session_id(id: &str) -> String { + id.chars().take(8).collect() +} diff --git a/crates/tj-cli/src/tui/session_list.rs b/crates/tj-cli/src/tui/session_list.rs index 21fb1a6..437b1df 100644 --- a/crates/tj-cli/src/tui/session_list.rs +++ b/crates/tj-cli/src/tui/session_list.rs @@ -268,7 +268,7 @@ impl SessionList { s.assistant_message_count() ); let duration = format_duration(&s.first_timestamp, &s.last_timestamp); - let id_short = &s.session_id[..8.min(s.session_id.len())]; + let id_short = super::short_session_id(&s.session_id); let line = Line::from(vec![ Span::styled(format!("{date} "), Style::default().fg(Color::DarkGray)), @@ -355,8 +355,7 @@ fn session_title(s: &ParsedSession) -> String { .unwrap_or(&clean); truncate_with_ellipsis(line.trim(), 80) } else { - let head: String = s.session_id.chars().take(8).collect(); - format!("Session {head}") + format!("Session {}", super::short_session_id(&s.session_id)) } } @@ -431,7 +430,25 @@ fn shorten_path(path: &str) -> String { #[cfg(test)] mod tests { - use super::truncate_with_ellipsis; + use super::{truncate_with_ellipsis, SessionList}; + use ratatui::{backend::TestBackend, Terminal}; + use tj_core::session::parser::ParsedSession; + + #[test] + fn render_list_cuts_non_ascii_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let session = ParsedSession { + session_id: "aééééé".into(), + file_path: String::new(), + entries: vec![], + first_timestamp: None, + last_timestamp: None, + }; + let list = SessionList::new(vec![session], String::new()); + let mut terminal = Terminal::new(TestBackend::new(80, 12)).unwrap(); + + terminal.draw(|frame| list.render(frame)).unwrap(); + } #[test] fn truncate_ascii_under_limit_returns_input_as_is() { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index c768714..139d00d 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1,7 +1,64 @@ -use assert_cmd::Command; use predicates::prelude::*; use predicates::str::contains; +/// Every `Command::cargo_bin` in this file goes through this wrapper, not +/// `assert_cmd`'s, so a test can't reach a real model or the developer's live +/// session by accident: PATH loses every directory holding a `claude` or +/// `codex` binary, API keys and backend overrides are cleared, and the Claude +/// Code session / mod markers of the shell running `cargo test` are dropped. +/// A test that needs a model installs its own fake and sets PATH itself. +struct Command; + +impl Command { + fn cargo_bin(name: &str) -> Result<assert_cmd::Command, assert_cmd::cargo::CargoError> { + let mut cmd = assert_cmd::Command::cargo_bin(name)?; + cmd.env("PATH", path_without_llm_clis()); + for var in [ + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "TJ_BACKEND", + "TJ_HYBRID_LLM_ORDER", + "TASK_JOURNAL_DATA_DIR", + "CLAUDE_CODE_SESSION_ID", + "TJ_IN_CLASSIFIER", + "TJ_MOD_ACTIVE", + ] { + cmd.env_remove(var); + } + Ok(cmd) + } +} + +/// The test process's PATH minus any directory that holds `claude` or `codex`, +/// led by a directory whose `gh` always fails, so `close`'s artifact harvest +/// never asks GitHub about this repository. Computed once: on WSL the PATH +/// carries dozens of slow `/mnt/c` directories. +fn path_without_llm_clis() -> &'static std::ffi::OsStr { + static PATH: std::sync::OnceLock<std::ffi::OsString> = std::sync::OnceLock::new(); + PATH.get_or_init(|| { + let shims = std::env::temp_dir().join(format!("tj-cli-test-shims-{}", std::process::id())); + std::fs::create_dir_all(&shims).unwrap(); + std::fs::write(shims.join("gh.cmd"), "@exit /b 1\r\n").unwrap(); + std::fs::write(shims.join("gh"), "#!/bin/sh\nexit 1\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(shims.join("gh"), std::fs::Permissions::from_mode(0o755)) + .unwrap(); + } + + let path = std::env::var_os("PATH").unwrap_or_default(); + let keep = std::iter::once(shims).chain(std::env::split_paths(&path).filter(|dir| { + ["claude", "codex"].iter().all(|bin| { + ["", ".exe", ".cmd"] + .iter() + .all(|ext| !dir.join(format!("{bin}{ext}")).exists()) + }) + })); + std::env::join_paths(keep).unwrap() + }) +} + #[test] fn pack_command_prints_markdown_for_existing_task() { let dir = assert_fs::TempDir::new().unwrap(); @@ -255,6 +312,74 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +/// `doctor` with PATH limited to `bin`, so only what the test puts there +/// resolves. +fn doctor_with_path(xdg: &std::path::Path, bin: &std::path::Path, json: bool) -> String { + let mut args = vec!["doctor"]; + if json { + args.push("--json"); + } + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg) + .env("PATH", bin) + .args(args) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn doctor_reports_missing_codex_and_claude_as_information_only() { + let xdg = assert_fs::TempDir::new().unwrap(); + let empty_bin = assert_fs::TempDir::new().unwrap(); + + let v: serde_json::Value = + serde_json::from_str(&doctor_with_path(xdg.path(), empty_bin.path(), true)).unwrap(); + assert_eq!(v["codex_in_path"], false); + assert_eq!(v["claude_in_path"], false); + assert_eq!(v["issues"].as_array().unwrap().len(), 0, "{v}"); + + let human = doctor_with_path(xdg.path(), empty_bin.path(), false); + assert!(human.contains("codex binary"), "{human}"); +} + +/// The suite must never reach a real model: a test command sees no `claude` +/// or `codex` unless the test installs its own fake on PATH. +#[test] +fn test_commands_see_no_real_claude_or_codex() { + let xdg = assert_fs::TempDir::new().unwrap(); + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args(["doctor", "--json"]) + .output() + .unwrap(); + + let v: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["claude_in_path"], false, "{v}"); + assert_eq!(v["codex_in_path"], false, "{v}"); +} + +#[cfg(unix)] +#[test] +fn doctor_detects_codex_on_path() { + use std::os::unix::fs::PermissionsExt; + + let xdg = assert_fs::TempDir::new().unwrap(); + let bin = assert_fs::TempDir::new().unwrap(); + let codex = bin.path().join("codex"); + std::fs::write(&codex, "#!/bin/sh\necho codex-cli 9.9.9\n").unwrap(); + std::fs::set_permissions(&codex, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let v: serde_json::Value = + serde_json::from_str(&doctor_with_path(xdg.path(), bin.path(), true)).unwrap(); + assert_eq!(v["codex_in_path"], true); +} + fn write_pending(xdg: &std::path::Path, id: &str, text: &str, attempts: u32) { let dir = xdg.join("task-journal").join("pending"); std::fs::create_dir_all(&dir).unwrap(); @@ -361,13 +486,22 @@ fn pending_retry_marks_dead_after_max_attempts() { let proj = assert_fs::TempDir::new().unwrap(); // Already at attempts=2; one more failure should rename to *.dead.json. write_pending(xdg.path(), "tj-dying", "any text", 2); + // An open task, so the chunk reaches the classifier at all. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Dying host"]) + .assert() + .success(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", xdg.path()) .current_dir(proj.path()) - // No --mock-* flags → retry fails → attempts becomes 3 → dead. - .args(["pending", "retry"]) + // The heuristic can't place "any text" → a real classifier failure + // → attempts becomes 3 → dead. + .args(["pending", "retry", "--backend", "heuristic"]) .assert() .success() .stdout(contains("1 marked dead")); @@ -379,6 +513,81 @@ fn pending_retry_marks_dead_after_max_attempts() { assert!(dead.exists(), "dead file must exist: {dead:?}"); } +#[test] +fn pending_retry_without_a_backend_leaves_entries_untouched() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "tj-waiting", "any text", 2); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + // Hybrid with no LLM fallback can only repeat the heuristic that + // already failed — that is "no backend", not a failed attempt. + .env("TJ_HYBRID_LLM_ORDER", "none") + .env_remove("ANTHROPIC_API_KEY") + .current_dir(proj.path()) + .args(["pending", "retry"]) + .assert() + .success() + .stdout(contains("no classifier backend")); + + let pending_dir = xdg.path().join("task-journal").join("pending"); + let live = pending_dir.join("tj-waiting.json"); + assert!(live.exists(), "entry must stay queued"); + assert!(!pending_dir.join("tj-waiting.dead.json").exists()); + let v: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&live).unwrap()).unwrap(); + assert_eq!(v["attempts"], 2, "no attempt is burned without a backend"); +} + +#[test] +fn pending_retry_classifies_through_the_real_backend() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Retry host"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + write_pending( + xdg.path(), + "tj-retry-real", + "After review we decided to use postgres for the journal store", + 1, + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pending", "retry", "--backend", "heuristic"]) + .assert() + .success() + .stdout(contains("1 drained")); + + let pending_dir = xdg.path().join("task-journal").join("pending"); + assert!(!pending_dir.join("tj-retry-real.json").exists()); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("[decision]").and(contains("postgres"))); +} + #[test] fn export_sqlite_round_trips_through_pack() { // Setup A: write a project + task in xdg_a/proj_a. @@ -531,6 +740,111 @@ fn export_html_emits_self_contained_document() { assert!(!html.contains("https://"), "external https url leaked"); } +/// Append a raw line to the only JSONL event log under `xdg`. +fn append_jsonl_line(xdg: &std::path::Path, line: &str) { + use std::io::Write; + + let events = xdg.join("task-journal").join("events"); + let log = std::fs::read_dir(&events) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("jsonl")) + .expect("events log present"); + let mut f = std::fs::OpenOptions::new().append(true).open(log).unwrap(); + writeln!(f, "{line}").unwrap(); +} + +#[test] +fn export_md_and_html_follow_close_reopen_and_rename_like_the_projection() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Original title"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + // Closed, then renamed: the projection says closed + new title, while the + // last event alone (a rename) would read as open with the old title. + for (etype, text) in [ + (tj_core::event::EventType::Close, "done"), + (tj_core::event::EventType::Reopen, "one more fix"), + (tj_core::event::EventType::Close, "done again"), + (tj_core::event::EventType::Rename, "Renamed title"), + ] { + let e = tj_core::event::Event::new( + task_id.clone(), + etype, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + text.to_string(), + ); + append_jsonl_line(xdg.path(), &serde_json::to_string(&e).unwrap()); + } + + let export = |format: &str| { + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["export", "--format", format]) + .output() + .unwrap(); + String::from_utf8(out.stdout).unwrap() + }; + + let md = export("md"); + assert!( + md.contains(&format!("## [{task_id}] Renamed title")), + "md title must follow the rename: {md}" + ); + assert!(md.contains("**Status**: closed"), "md status: {md}"); + + let html = export("html"); + assert!( + html.contains(&format!("{task_id}</span>Renamed title</h2>")), + "html title must follow the rename: {html}" + ); + assert!(html.contains("status: closed"), "html status: {html}"); +} + +#[test] +fn export_skips_malformed_jsonl_lines_with_a_warning() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Survives a bad line"]) + .assert() + .success(); + append_jsonl_line(xdg.path(), "{not json"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["export", "--format", "json"]) + .assert() + .success() + .stdout(contains("Survives a bad line")) + .stderr(contains("skipping malformed JSONL line 2")); +} + #[test] fn migrate_project_round_trips_data_to_new_path() { let xdg = assert_fs::TempDir::new().unwrap(); @@ -619,63 +933,516 @@ fn migrate_project_refuses_overwrite_without_force() { .stderr(contains("destination already exists")); } -#[test] -fn close_unknown_task_id_returns_error() { - let dir = assert_fs::TempDir::new().unwrap(); +/// Two project dirs that each hash to themselves, plus their hashes. +fn two_projects() -> (assert_fs::TempDir, assert_fs::TempDir, String, String) { + let proj_a = assert_fs::TempDir::new().unwrap(); + let proj_b = assert_fs::TempDir::new().unwrap(); + std::fs::create_dir(proj_a.path().join(".git")).unwrap(); + std::fs::create_dir(proj_b.path().join(".git")).unwrap(); + let from_hash = tj_core::project_hash::from_path(proj_a.path()).unwrap(); + let to_hash = tj_core::project_hash::from_path(proj_b.path()).unwrap(); + + (proj_a, proj_b, from_hash, to_hash) +} + +fn migrate(xdg: &std::path::Path, from: &std::path::Path, to: &std::path::Path, force: bool) { + let mut args = vec![ + "migrate-project", + "--from", + from.to_str().unwrap(), + "--to", + to.to_str().unwrap(), + ]; + if force { + args.push("--force"); + } Command::cargo_bin("task-journal") .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["close", "tj-doesnotexist", "--reason", "shipped"]) + .env("XDG_DATA_HOME", xdg) + .args(args) .assert() - .failure() - .stderr(contains("task not found: tj-doesnotexist")); + .success(); } #[test] -fn search_all_projects_finds_match_in_other_project_hash() { - let dir = assert_fs::TempDir::new().unwrap(); +fn migrate_project_rekeys_embeddings_and_dream_state() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); - let state = dir.path().join("task-journal").join("state"); - std::fs::create_dir_all(&state).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Re-key everything"]) + .assert() + .success(); + { + let conn = tj_core::db::open(state.join(format!("{from_hash}.sqlite"))).unwrap(); + conn.execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES (?1, 't', 't')", + [&from_hash], + ) + .unwrap(); + conn.execute( + "INSERT INTO embeddings(event_id, task_id, project_hash, model, dim, vec, created_at) + VALUES ('e1', 'tj-1', ?1, 'hash', 1, x'00000000', 't')", + [&from_hash], + ) + .unwrap(); + } - for hash in ["aaaa1111aaaa1111", "bbbb2222bbbb2222"] { - let db_path = state.join(format!("{hash}.sqlite")); - let conn = tj_core::db::open(&db_path).unwrap(); - let mut e = tj_core::event::Event::new( - format!("tj-{}", &hash[..6]), - tj_core::event::EventType::Open, - tj_core::event::Author::User, - tj_core::event::Source::Cli, - format!("Marker {hash}"), + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + let conn = rusqlite::Connection::open(state.join(format!("{to_hash}.sqlite"))).unwrap(); + for table in ["dream_state", "embeddings"] { + let rekeyed: i64 = conn + .query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE project_hash = ?1"), + [&to_hash], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + rekeyed, 1, + "{table} row not re-keyed to the new project_hash" ); - e.meta = serde_json::json!({"title": format!("Title {hash}")}); - tj_core::db::upsert_task_from_event(&conn, &e, hash).unwrap(); - tj_core::db::index_event(&conn, &e).unwrap(); } +} + +#[test] +fn migrate_project_rekeys_only_its_rows_in_global_memory() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); Command::cargo_bin("task-journal") .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["search", "Marker", "--all-projects"]) + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Re-key global memory"]) .assert() - .success() - .stdout(contains("aaaa1111").and(contains("bbbb2222"))); + .success(); + let memory_path = xdg.path().join("task-journal").join("memory.sqlite"); + { + let global = tj_core::memory::open(&memory_path).unwrap(); + for (event_id, hash) in [("e-mine", from_hash.as_str()), ("e-other", "otherhash")] { + global + .execute( + "INSERT INTO global_memory(event_id, project_hash, task_id, type, text, model, dim, vec, created_at) + VALUES (?1, ?2, 'tj-1', 'decision', 't', 'hash', 1, x'00000000', 't')", + [event_id, hash], + ) + .unwrap(); + } + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + let global = rusqlite::Connection::open(&memory_path).unwrap(); + let hash_of = |event_id: &str| -> String { + global + .query_row( + "SELECT project_hash FROM global_memory WHERE event_id = ?1", + [event_id], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!(hash_of("e-mine"), to_hash); + assert_eq!(hash_of("e-other"), "otherhash"); } +/// A write still sitting in `<hash>.sqlite-wal` (another process holds the +/// DB open, so nothing checkpointed it) must reach the new project, and no +/// sidecar may stay behind under the old hash. Unix-only: Windows refuses to +/// rename a file another handle keeps open. +#[cfg(unix)] #[test] -fn search_command_finds_task_by_event_text() { - let dir = assert_fs::TempDir::new().unwrap(); - let task_id = String::from_utf8( - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["create", "OAuth thing"]) - .assert() - .success() - .get_output() - .stdout - .clone(), - ) +fn migrate_project_carries_uncheckpointed_wal_writes() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "WAL survives migration"]) + .assert() + .success(); + + let src = state.join(format!("{from_hash}.sqlite")); + let holder = tj_core::db::open(&src).unwrap(); + holder + .execute_batch("PRAGMA wal_autocheckpoint=0;") + .unwrap(); + holder + .execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES (?1, 't', 't')", + [&from_hash], + ) + .unwrap(); + // Keep the connection open for the rest of the test so its WAL is never + // checkpointed or deleted on close. + std::mem::forget(holder); + let wal = |hash: &str| state.join(format!("{hash}.sqlite-wal")); + assert!(wal(&from_hash).exists(), "precondition: source WAL present"); + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + for suffix in ["-wal", "-shm"] { + let left = state.join(format!("{from_hash}.sqlite{suffix}")); + assert!(!left.exists(), "sidecar left behind: {left:?}"); + } + let conn = rusqlite::Connection::open(state.join(format!("{to_hash}.sqlite"))).unwrap(); + let rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dream_state WHERE project_hash = ?1", + [&to_hash], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(rows, 1, "the WAL-only write was lost in migration"); +} + +// Unix only: the stand-in for a failing move (renaming a directory over a +// file) is refused on Unix but allowed on Windows, where the real failure is a +// sharing violation that a test can't stage portably. +#[cfg(unix)] +#[test] +fn migrate_project_force_keeps_destination_when_the_move_fails() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let events = xdg.path().join("task-journal").join("events"); + + // A directory cannot be renamed over a file, so the move must fail. + std::fs::create_dir_all(events.join(format!("{from_hash}.jsonl"))).unwrap(); + let dst = events.join(format!("{to_hash}.jsonl")); + std::fs::write(&dst, "destination data\n").unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + "--force", + ]) + .assert() + .failure(); + + assert_eq!( + std::fs::read_to_string(&dst).unwrap(), + "destination data\n", + "a failed --force move must leave the destination intact" + ); +} + +/// The JSONL is the source of truth: `--force` keeps the journal it replaces +/// as `<dst>.jsonl.bak-<timestamp>` and says where it went. +#[test] +fn migrate_project_force_replaces_existing_destination() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, _, to_hash) = two_projects(); + + for (proj, title) in [(&proj_a, "Source task"), (&proj_b, "Overwritten task")] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", title]) + .assert() + .success(); + } + + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + "--force", + ]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_b.path()) + .args(["events", "list"]) + .assert() + .success() + .stdout(contains("Source task").and(contains("Overwritten task").not())); + + let events = xdg.path().join("task-journal").join("events"); + let backup = std::fs::read_dir(&events) + .unwrap() + .map(|e| e.unwrap().path()) + .find(|p| { + p.file_name() + .unwrap() + .to_string_lossy() + .starts_with(&format!("{to_hash}.jsonl.bak-")) + }) + .unwrap_or_else(|| panic!("no backup of the replaced journal; stdout: {out}")); + assert!(std::fs::read_to_string(&backup) + .unwrap() + .contains("Overwritten task")); + assert!(out.contains(&backup.display().to_string()), "{out}"); +} + +/// A destination WAL left behind by the database `--force` replaces must not +/// be replayed into the one moving in. +#[test] +fn migrate_project_force_drops_the_replaced_databases_wal() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); + + for (proj, title) in [(&proj_a, "Source task"), (&proj_b, "Overwritten task")] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", title]) + .assert() + .success(); + } + drop(tj_core::db::open(state.join(format!("{from_hash}.sqlite"))).unwrap()); + + // A crash leftover: the destination's WAL with a write never folded in. + let dst_db = state.join(format!("{to_hash}.sqlite")); + let dst_wal = state.join(format!("{to_hash}.sqlite-wal")); + { + let conn = tj_core::db::open(&dst_db).unwrap(); + conn.execute_batch("PRAGMA wal_autocheckpoint=0;").unwrap(); + conn.execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES ('stale-wal', 't', 't')", + [], + ) + .unwrap(); + let wal = std::fs::read(&dst_wal).unwrap(); + drop(conn); + std::fs::write(&dst_wal, wal).unwrap(); + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), true); + + let conn = rusqlite::Connection::open(&dst_db).unwrap(); + let check: String = conn + .query_row("PRAGMA integrity_check", [], |r| r.get(0)) + .unwrap(); + assert_eq!(check, "ok"); + let stale: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dream_state WHERE project_hash = 'stale-wal'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(stale, 0, "the replaced database's WAL was replayed"); +} + +/// Queued chunks and the push-recall log are keyed by the project hash too. +/// After a move the new project's worker classifies what the old one queued. +#[test] +fn migrate_project_rekeys_pending_entries_and_the_recall_log() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let pending = xdg.path().join("task-journal").join("pending"); + let state = xdg.path().join("task-journal").join("state"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Queue host"]) + .assert() + .success(); + let queued = write_pending_entry( + &pending, + &format!("{from_hash}.01JD0000000000000000000000.json"), + Some(&from_hash), + true, + "UserPromptSubmit", + "We decided to use store-moved for the cache", + ); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write(state.join(format!("{from_hash}.recall-shown")), "s1 e1\n").unwrap(); + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + assert!(!queued.exists(), "entry left under the old hash"); + assert_eq!( + std::fs::read_to_string(state.join(format!("{to_hash}.recall-shown"))).unwrap(), + "s1 e1\n" + ); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_b.path()) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + let journal = project_events(xdg.path(), &to_hash); + assert!(journal.contains("store-moved"), "{journal}"); +} + +#[test] +fn close_unknown_task_id_returns_error() { + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["close", "tj-doesnotexist", "--reason", "shipped"]) + .assert() + .failure() + .stderr(contains("task not found: tj-doesnotexist")); +} + +#[test] +fn search_all_projects_finds_match_in_other_project_hash() { + let dir = assert_fs::TempDir::new().unwrap(); + + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + + for hash in ["aaaa1111aaaa1111", "bbbb2222bbbb2222"] { + let db_path = state.join(format!("{hash}.sqlite")); + let conn = tj_core::db::open(&db_path).unwrap(); + let mut e = tj_core::event::Event::new( + format!("tj-{}", &hash[..6]), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + format!("Marker {hash}"), + ); + e.meta = serde_json::json!({"title": format!("Title {hash}")}); + tj_core::db::upsert_task_from_event(&conn, &e, hash).unwrap(); + tj_core::db::index_event(&conn, &e).unwrap(); + } + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["search", "Marker", "--all-projects"]) + .assert() + .success() + .stdout(contains("aaaa1111").and(contains("bbbb2222"))); +} + +#[test] +fn all_projects_search_and_rejected_warn_about_unreadable_projects() { + let dir = assert_fs::TempDir::new().unwrap(); + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + + let good = "aaaa1111aaaa1111"; + let conn = tj_core::db::open(state.join(format!("{good}.sqlite"))).unwrap(); + let mut e = tj_core::event::Event::new( + "tj-good1".to_string(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + "Marker in a healthy project".to_string(), + ); + e.meta = serde_json::json!({"title": "Healthy"}); + tj_core::db::upsert_task_from_event(&conn, &e, good).unwrap(); + tj_core::db::index_event(&conn, &e).unwrap(); + drop(conn); + std::fs::write(state.join("cccc3333cccc3333.sqlite"), b"not a database").unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["search", "Marker", "--all-projects"]) + .assert() + .success() + .stdout(contains(good)) + .stderr(contains("warning: skipping project cccc3333cccc3333")); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["rejected", "Marker", "--all-projects"]) + .assert() + .success() + .stderr(contains("warning: skipping project cccc3333cccc3333")); +} + +#[test] +fn rejected_warns_about_a_row_it_cannot_read_and_keeps_the_rest() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = assert_fs::TempDir::new().unwrap(); + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + let hash = "aaaa1111aaaa1111"; + let conn = tj_core::db::open(state.join(format!("{hash}.sqlite"))).unwrap(); + + for task_id in ["tj-good1", "tj-bad1"] { + let mut open = Event::new( + task_id, + EventType::Open, + Author::User, + Source::Cli, + "t".into(), + ); + open.meta = serde_json::json!({"title": task_id}); + let rejection = Event::new( + task_id, + EventType::Rejection, + Author::User, + Source::Cli, + format!("Marker rejected in {task_id}"), + ); + for e in [&open, &rejection] { + tj_core::db::upsert_task_from_event(&conn, e, hash).unwrap(); + tj_core::db::index_event(&conn, e).unwrap(); + } + } + // A BLOB title cannot be read as text, so this task's row fails to map. + conn.execute( + "UPDATE tasks SET title = x'00' WHERE task_id = 'tj-bad1'", + [], + ) + .unwrap(); + drop(conn); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["rejected", "Marker", "--all-projects"]) + .assert() + .success() + .stdout(contains("tj-good1")) + .stderr(contains("warning:").and(contains(hash))); +} + +#[test] +fn search_command_finds_task_by_event_text() { + let dir = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "OAuth thing"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) .unwrap() .trim() .to_string(); @@ -1109,6 +1876,17 @@ fn post_model_switch_records_a_constraint_on_the_active_task() { .stdout(contains( "Model switched (auto): claude-opus-5 → claude-haiku-4-5", )); + + // Marked so readers of the journal can tell it from a real constraint. + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + let journal = project_events(dir.path(), &hash); + let switch: serde_json::Value = journal + .lines() + .map(|l| serde_json::from_str::<serde_json::Value>(l).unwrap()) + .find(|e| e["type"] == "constraint") + .expect("model switch recorded"); + assert_eq!(switch["meta"]["kind"], "model_switch", "{switch}"); + assert_eq!(switch["meta"]["session_id"], "s-switch", "{switch}"); } /// No open task → a model switch is not a reason to start one. @@ -1489,39 +2267,286 @@ fn ingest_hook_drains_pending_queue_via_mock() { ); } +/// A legacy entry is a classifier failure waiting for `pending retry`; a +/// live hook must not throw it away. #[test] -fn stats_command_shows_classifier_counts() { - let dir = assert_fs::TempDir::new().unwrap(); - let metrics = dir.path().join("task-journal").join("metrics"); - std::fs::create_dir_all(&metrics).unwrap(); - let body = [r#"{"timestamp":"2026-04-30T00:00:00Z","project_hash":"feedface","task_id_guess":"tj-x","event_type":"decision","confidence":0.95,"status":"confirmed","error":null}"#, - r#"{"timestamp":"2026-04-30T00:00:00Z","project_hash":"feedface","task_id_guess":"tj-x","event_type":"finding","confidence":0.65,"status":"suggested","error":null}"#].join("\n"); - std::fs::write(metrics.join("feedface.jsonl"), body).unwrap(); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["stats"]) - .assert() - .success() - .stdout( - contains("classified: 2") - .and(contains("confirmed: 1")) - .and(contains("suggested: 1")), - ); -} +fn ingest_hook_keeps_legacy_pending_entries_for_pending_retry() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "01failed", "We decided to adopt PKCE flow.", 1); -#[test] -fn ingest_hook_writes_telemetry_record() { - let dir = assert_fs::TempDir::new().unwrap(); - let task_id = String::from_utf8( + for kind in ["UserPromptSubmit", "PostToolUse"] { Command::cargo_bin("task-journal") .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["create", "Tel"]) + .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .current_dir(proj.path()) + .args(["ingest-hook", "--kind", kind, "--text", "next chunk"]) .assert() - .success() - .get_output() + .success(); + } + + let entry = xdg + .path() + .join("task-journal") + .join("pending") + .join("01failed.json"); + assert!( + entry.exists(), + "the failed chunk must wait for pending retry" + ); +} + +/// The mock drain only removes an entry it turned into an event. +#[test] +fn ingest_hook_mock_drain_keeps_entries_it_cannot_record() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "01empty", "", 0); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args([ + "ingest-hook", + "--kind", + "Stop", + "--text", + "Live chunk", + "--mock-event-type", + "decision", + "--mock-task-id", + "tj-mock", + ]) + .assert() + .success(); + + let entry = xdg + .path() + .join("task-journal") + .join("pending") + .join("01empty.json"); + assert!( + entry.exists(), + "an entry that wrote no event must stay queued" + ); +} + +/// Run one `ingest-hook` payload of `kind` with `TJ_MOD_ACTIVE` set to +/// `mod_value` (None = unset), in a fresh project with an open task and a +/// transcript. True when the hook left any trace: stdout, a pending entry, +/// or a new journal line. +fn hook_leaves_a_trace(kind: &str, mod_value: Option<&str>) -> bool { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Mod host"]) + .assert() + .success(); + let transcript = proj.path().join("session.jsonl"); + std::fs::write( + &transcript, + r#"{"type":"user","uuid":"u1","timestamp":"2099-01-01T00:00:00.000Z","sessionId":"s1","message":{"content":"the refund flow needs idempotency keys"}}"#, + ) + .unwrap(); + let payload = serde_json::json!({ + "hook_event_name": kind, + "session_id": "s1", + "transcript_path": transcript.to_str().unwrap(), + "reason": "clear", + "prompt": "We decided to adopt the PKCE flow.", + "tool_name": "Bash", + "tool_input": { "command": "cargo test" }, + "tool_response": { "output": "ok" }, + "from_model": "claude-opus", + "to_model": "claude-sonnet", + }); + let journal = xdg.path().join("task-journal").join("events"); + let journal_len = || -> u64 { + std::fs::read_dir(&journal) + .unwrap() + .map(|e| e.unwrap().metadata().unwrap().len()) + .sum() + }; + let before = journal_len(); + + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .current_dir(proj.path()); + if let Some(v) = mod_value { + cmd.env("TJ_MOD_ACTIVE", v); + } + let out = cmd + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + + let pending = std::fs::read_dir(xdg.path().join("task-journal").join("pending")) + .map(|d| d.count()) + .unwrap_or(0); + !out.is_empty() || pending > 0 || journal_len() != before +} + +/// The Claude Code mod captures in-process and sets TJ_MOD_ACTIVE for the +/// hooks it starts; the classic capture must then stand down. (Nothing here +/// echoes a prior entry, so push-recall stays quiet too.) +#[test] +fn mod_active_silences_the_classic_capture_hooks() { + for kind in [ + "UserPromptSubmit", + "PostToolUse", + "Stop", + "PreCompact", + "SessionEnd", + ] { + assert!(!hook_leaves_a_trace(kind, Some("1")), "{kind}, mod on"); + assert!(hook_leaves_a_trace(kind, Some("0")), "{kind}, mod \"0\""); + assert!(hook_leaves_a_trace(kind, None), "{kind}, mod unset"); + } +} + +/// Resume packs and the model-switch constraint are not the mod's job. +#[test] +fn mod_active_keeps_session_start_and_model_switch() { + for kind in ["SessionStart", "PostModelSwitch"] { + assert!(hook_leaves_a_trace(kind, Some("1")), "{kind}, mod on"); + } +} + +/// The mod takes over capture, not push-recall: a tool call that echoes a +/// prior rejection still gets the recall on both paths, and nothing is queued. +#[test] +fn mod_active_keeps_push_recall() { + let (dir, _task_id) = seed_axum_rejection(); + + for tool in ["Bash", "mcp__x__do"] { + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": tool, + "tool_input": { "command": "let's switch the server to axum" }, + "tool_response": { "output": "" } + }); + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .env("TJ_MOD_ACTIVE", "1") + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); + assert!(out.contains("previously rejected"), "{tool}: {out}"); + } + + let pending = std::fs::read_dir(dir.path().join("task-journal").join("pending")) + .map(|d| d.count()) + .unwrap_or(0); + assert_eq!(pending, 0, "the mod queues tool calls itself"); +} + +/// `/rewind` is the user rolling back a path — the mod doesn't record that, +/// so the correction event still lands. +#[test] +fn mod_active_keeps_the_rewind_correction() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Rewind host"]) + .assert() + .success(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .env("TJ_MOD_ACTIVE", "1") + .current_dir(proj.path()) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin( + serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "prompt": "/rewind go back to plan A", + }) + .to_string(), + ) + .assert() + .success(); + + let journal = project_events(xdg.path(), &hash); + assert_eq!(session_ids_of(&journal, "correction").len(), 1, "{journal}"); +} + +#[test] +fn mod_active_silences_the_nudge() { + for (mod_value, silent) in [(Some("1"), true), (Some("0"), false), (None, false)] { + let xdg = assert_fs::TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", xdg.path()); + if let Some(v) = mod_value { + cmd.env("TJ_MOD_ACTIVE", v); + } + let out = cmd + .arg("nudge") + .write_stdin("{}") + .assert() + .success() + .get_output() + .stdout + .clone(); + assert_eq!(out.is_empty(), silent, "TJ_MOD_ACTIVE={mod_value:?}"); + } +} + +#[test] +fn stats_command_shows_classifier_counts() { + let dir = assert_fs::TempDir::new().unwrap(); + let metrics = dir.path().join("task-journal").join("metrics"); + std::fs::create_dir_all(&metrics).unwrap(); + let body = [r#"{"timestamp":"2026-04-30T00:00:00Z","project_hash":"feedface","task_id_guess":"tj-x","event_type":"decision","confidence":0.95,"status":"confirmed","error":null}"#, + r#"{"timestamp":"2026-04-30T00:00:00Z","project_hash":"feedface","task_id_guess":"tj-x","event_type":"finding","confidence":0.65,"status":"suggested","error":null}"#].join("\n"); + std::fs::write(metrics.join("feedface.jsonl"), body).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["stats"]) + .assert() + .success() + .stdout( + contains("classified: 2") + .and(contains("confirmed: 1")) + .and(contains("suggested: 1")), + ); +} + +#[test] +fn ingest_hook_writes_telemetry_record() { + let dir = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Tel"]) + .assert() + .success() + .get_output() .stdout .clone(), ) @@ -1645,8 +2670,8 @@ fn ingest_hook_session_start_emits_resume_pack_json() { #[test] fn session_start_emits_neither_session_title_nor_initial_message() { - // 0.14.3: the SessionStart envelope carries ONLY additionalContext - // (+ optional watchPaths). It must never set sessionTitle (which + // 0.14.3: the SessionStart envelope carries ONLY additionalContext. + // It must never set sessionTitle (which // overrode Claude Code's own session name with our task id) nor // initialUserMessage (which seeded garbage "[Task Journal resumed: …]" // task titles). @@ -1804,6 +2829,70 @@ fn close_with_outcome_renders_outcome_block() { .stdout(contains("**Outcome** [done]: Shipped in v0.4.0")); } +#[test] +fn close_records_the_outcome_only_through_the_close_event() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let out = tj() + .args(["create", "Close me", "--goal", "g"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + let close = |outcome: &str| { + let mut cmd = tj(); + cmd.args([ + "close", + &task_id, + "--outcome", + outcome, + "--outcome-tag", + "done", + ]); + cmd + }; + + // The close append fails: the journal is read-only. + let journal = std::fs::read_dir(dir.path().join("task-journal").join("events")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let writable = std::fs::metadata(&journal).unwrap().permissions(); + let mut read_only = writable.clone(); + read_only.set_readonly(true); + std::fs::set_permissions(&journal, read_only).unwrap(); + close("must not stick").assert().failure(); + std::fs::set_permissions(&journal, writable).unwrap(); + + let state = std::fs::read_dir(dir.path().join("task-journal").join("state")) + .unwrap() + .map(|e| e.unwrap().path()) + .find(|p| p.extension().is_some_and(|x| x == "sqlite")) + .unwrap(); + let conn = tj_core::db::open(&state).unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task_id) + .unwrap() + .unwrap(); + assert_eq!(meta.outcome, None, "failed close left an outcome behind"); + drop(conn); + + // A real close keeps its outcome through a rebuild from the journal. + close("shipped").assert().success(); + std::fs::remove_dir_all(dir.path().join("task-journal").join("state")).unwrap(); + tj().args(["pack", &task_id]) + .assert() + .success() + .stdout(contains("**Outcome** [done]: shipped")); +} + #[test] fn close_rejects_invalid_outcome_tag() { let dir = assert_fs::TempDir::new().unwrap(); @@ -1905,6 +2994,56 @@ fn external_add_appends_references() { .stdout(contains("**External**: beads:claude-memory-rsw,github:#42")); } +#[test] +fn goal_and_external_survive_deleting_the_state_db() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let wipe_state = || { + let _ = std::fs::remove_dir_all(dir.path().join("task-journal").join("state")); + }; + let out = tj() + .args(["create", "Rebuildable", "--goal", "first goal"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + + wipe_state(); + tj().args(["pack", &task_id]) + .assert() + .success() + .stdout(contains("**Goal**: first goal")); + + tj().args(["goal", &task_id, "second goal"]) + .assert() + .success(); + tj().args(["external", &task_id, "--add", "beads:rb-1"]) + .assert() + .success(); + wipe_state(); + + tj().args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("**Goal**: second goal")) + .stdout(contains("**External**: beads:rb-1")) + .stdout(contains("amend").not()); + tj().args(["pack", "--external", "beads:rb-1"]) + .assert() + .success() + .stdout(contains("Rebuildable")); + tj().args(["events", "list"]) + .assert() + .success() + .stdout(contains("goal: second goal")); +} + #[test] fn ingest_hook_short_circuits_when_in_classifier_env_set() { // Recursion guard: classifier sets TJ_IN_CLASSIFIER=1 before @@ -1966,6 +3105,22 @@ fn ingest_hook_short_circuits_when_in_classifier_env_set() { ); } +#[test] +fn nudge_is_silent_inside_a_classifier_child() { + // A `claude -p` / `codex exec` spawned by the classifier re-runs the + // user's UserPromptSubmit hooks; the nudge must not inject into it. + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_IN_CLASSIFIER", "1") + .args(["nudge"]) + .write_stdin("") + .assert() + .success() + .stdout(""); +} + #[test] fn ingest_hook_reads_user_prompt_submit_payload_from_stdin() { // Real Claude Code passes hook input as JSON over stdin, NOT via env @@ -2225,24 +3380,72 @@ fn events_list_shows_recent_events() { } #[test] -fn rebuild_state_creates_sqlite_with_one_task() { - let dir = assert_fs::TempDir::new().unwrap(); +fn events_list_skips_malformed_jsonl_lines_with_a_warning() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["create", "Build it"]) + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Listed despite a bad line"]) .assert() .success(); + append_jsonl_line(xdg.path(), "{not json"); Command::cargo_bin("task-journal") .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .args(["rebuild-state"]) + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["events", "list"]) .assert() .success() - .stdout(contains("rebuilt")); + .stdout(contains("Listed despite a bad line")) + .stderr(contains("skipping malformed JSONL line 2")); +} - let state_dir = dir.path().join("task-journal").join("state"); +#[test] +fn events_list_prints_snake_case_event_types() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Typed listing"]) + .assert() + .success(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["events", "list"]) + .assert() + .success() + .stdout(contains("[open]").and(contains("[Open]").not())); +} + +#[test] +fn rebuild_state_creates_sqlite_with_one_task() { + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Build it"]) + .assert() + .success(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["rebuild-state"]) + .assert() + .success() + .stdout(contains("rebuilt")); + + let state_dir = dir.path().join("task-journal").join("state"); let mut found = 0; for entry in std::fs::read_dir(&state_dir).unwrap() { let p = entry.unwrap().path(); @@ -2539,6 +3742,17 @@ fn auto_open_links_to_prior_task_referencing_same_issue() { // current status annotated next to the id. .stdout(contains("**Linked**:")) .stdout(contains(format!("- {} [closed]", prior))); + + // Goal and link ride in the journal: a state rebuilt from it keeps both. + std::fs::remove_dir_all(dir.path().join("task-journal").join("state")).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["pack", &new_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("**Goal**: (not set)").not()) + .stdout(contains(format!("- {} [closed]", prior))); } #[test] @@ -2728,7 +3942,14 @@ fn ingest_hook_returns_fast_in_async_mode() { let entries: Vec<_> = std::fs::read_dir(&pending) .unwrap() .filter_map(|e| e.ok()) - .filter(|e| e.path().extension().and_then(|s| s.to_str()) == Some("json")) + // A worker that already claimed the entry renamed it to `.processing`: + // still queued, just being classified. + .filter(|e| { + matches!( + e.path().extension().and_then(|s| s.to_str()), + Some("json" | "processing") + ) + }) .collect(); // Worker may have already drained; in that case at least the // worker should have left a v1 pending entry from /bin/false @@ -2765,105 +3986,600 @@ fn classify_worker_handles_classifier_failure_cleanly() { .join(format!("{project_hash}.jsonl")); std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); - let entry = pending.join("01worker.json"); - let body = serde_json::json!({ - "schema": "v2", - "kind": "UserPromptSubmit", - "text": "worker test marker", - "project_hash": project_hash, - "events_path": events_path.to_string_lossy(), - "backend": "hybrid", - "queued_at": "2026-05-08T00:00:00Z", - }); - std::fs::write(&entry, body.to_string()).unwrap(); + let entry = pending.join("01worker.json"); + let body = serde_json::json!({ + "schema": "v2", + "kind": "UserPromptSubmit", + "text": "worker test marker", + "project_hash": project_hash, + "events_path": events_path.to_string_lossy(), + "backend": "hybrid", + "queued_at": "2026-05-08T00:00:00Z", + }); + std::fs::write(&entry, body.to_string()).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_CLASSIFIER_CLI", "/bin/false") + .args(["classify-worker", "--backend", "hybrid"]) + .assert() + .success(); +} + +/// One worker per project, enforced by an OS lock on the lock file, not by +/// what the file says: while another process holds the lock, a worker +/// leaves the queue alone — even when the file names a dead pid. +#[test] +fn classify_worker_respects_existing_lock() { + let dir = assert_fs::TempDir::new().unwrap(); + let cwd = std::env::current_dir().unwrap(); + let project_hash = tj_core::project_hash::from_path(&cwd).expect("compute project hash"); + + // Pre-create a v2 pending entry. + let pending = dir.path().join("task-journal").join("pending"); + std::fs::create_dir_all(&pending).unwrap(); + let events_path = dir + .path() + .join("task-journal") + .join("events") + .join(format!("{project_hash}.jsonl")); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let entry = pending.join("01locked.json"); + std::fs::write( + &entry, + serde_json::json!({ + "schema": "v2", + "kind": "UserPromptSubmit", + "text": "locked marker", + "project_hash": project_hash, + "events_path": events_path.to_string_lossy(), + "backend": "hybrid", + "queued_at": "2026-05-08T00:00:00Z", + }) + .to_string(), + ) + .unwrap(); + + // Hold the lock from this process, over a file naming a dead pid. + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + let lock_path = state.join(format!("classifier-{project_hash}.lock")); + std::fs::write(&lock_path, format!("{}\n", i32::MAX)).unwrap(); + let mut lock = fd_lock::RwLock::new( + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(), + ); + let _held = lock.try_write().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_CLASSIFIER_CLI", "/bin/false") + .args(["classify-worker", "--backend", "hybrid"]) + .assert() + .success(); + + // Pending entry must still be there — the second worker bailed. + assert!( + entry.exists(), + "pending entry must survive — second worker must not have drained it" + ); + // Our hand-rolled lockfile must still be there too — the bailing + // worker must NOT remove a lock it didn't acquire. + assert!( + lock_path.exists(), + "lockfile must survive — bailing worker must not delete others' locks" + ); +} + +/// A lock file nobody holds is a finished or crashed worker's leftover — +/// whatever it says (empty, or a pid that is alive again under another +/// program) the next worker takes over: the OS released the lock with the +/// process. +#[test] +fn classify_worker_takes_over_a_leftover_lock_file() { + let dir = assert_fs::TempDir::new().unwrap(); + let cwd = std::env::current_dir().unwrap(); + let project_hash = tj_core::project_hash::from_path(&cwd).expect("compute project hash"); + + let pending = dir.path().join("task-journal").join("pending"); + std::fs::create_dir_all(&pending).unwrap(); + let events_path = dir + .path() + .join("task-journal") + .join("events") + .join(format!("{project_hash}.jsonl")); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + let lock_path = state.join(format!("classifier-{project_hash}.lock")); + + for leftover in [String::new(), format!("{}\n", std::process::id())] { + let entry = pending.join("01leftoverlock.json"); + std::fs::write( + &entry, + serde_json::json!({ + "schema": "v2", + "kind": "PostToolUse", + "text": "leftover lock marker", + "project_hash": project_hash, + "events_path": events_path.to_string_lossy(), + "backend": "heuristic", + "queued_at": "2026-05-08T00:00:00Z", + }) + .to_string(), + ) + .unwrap(); + std::fs::write(&lock_path, &leftover).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + assert!(!entry.exists(), "lock file {leftover:?} is taken over"); + } +} + +/// Workers started at once on one queue — over a crashed worker's leftover +/// lock — record each entry exactly once: no duplicate classifier calls, no +/// duplicate events. +#[test] +fn classify_workers_started_together_record_each_entry_once() { + const ENTRIES: usize = 12; + const WORKERS: usize = 8; + + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Worker race host"]) + .assert() + .success(); + + let pending = dir.path().join("task-journal").join("pending"); + for i in 0..ENTRIES { + write_pending_entry( + &pending, + &format!("{hash}.01JB{i:022}.json"), + Some(&hash), + true, + "UserPromptSubmit", + &format!("We decided to use store-{i:02} for the cache"), + ); + } + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write( + state.join(format!("classifier-{hash}.lock")), + format!("{}\n", i32::MAX), + ) + .unwrap(); + + let start = std::sync::Barrier::new(WORKERS); + std::thread::scope(|s| { + for _ in 0..WORKERS { + s.spawn(|| { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]); + start.wait(); + cmd.assert().success(); + }); + } + }); + + // The race can leave an entry to the next worker, never record it twice. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + let journal = project_events(dir.path(), &hash); + for i in 0..ENTRIES { + let marker = format!("store-{i:02}"); + let n = journal.lines().filter(|l| l.contains(&marker)).count(); + assert_eq!(n, 1, "{marker} recorded {n} times:\n{journal}"); + } +} + +/// A worker claims an entry (`<name>.processing`) before classifying it. A +/// failed entry goes back under its own name; one a crashed worker left +/// claimed is put back in the queue by the next worker and recorded. +#[test] +fn classify_worker_requeues_failed_and_orphaned_claims() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Claim host"]) + .assert() + .success(); + let pending = dir.path().join("task-journal").join("pending"); + let worker = |backend: &str| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", backend]) + .assert() + .success(); + }; + + let queued = write_pending_entry( + &pending, + &format!("{hash}.01JC0000000000000000000000.json"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use store-failed for the cache", + ); + worker("no-such-backend"); + assert!(queued.exists(), "a failed entry is back under its own name"); + + let orphan = write_pending_entry( + &pending, + &format!("{hash}.01JC0000000000000000000001.json.processing"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use store-orphan for the cache", + ); + worker("heuristic"); + + assert!(!queued.exists() && !orphan.exists()); + let journal = project_events(dir.path(), &hash); + for marker in ["store-failed", "store-orphan"] { + let n = journal.lines().filter(|l| l.contains(marker)).count(); + assert_eq!(n, 1, "{marker} recorded {n} times:\n{journal}"); + } +} + +// ---------------- pending/ is global: every consumer is project-scoped ---------------- + +/// Write a pending entry. `project_hash: None` leaves the field out (an +/// entry from before the queue knew about projects). +fn write_pending_entry( + pending: &std::path::Path, + file_name: &str, + project_hash: Option<&str>, + schema_v2: bool, + kind: &str, + text: &str, +) -> std::path::PathBuf { + std::fs::create_dir_all(pending).unwrap(); + let mut body = serde_json::json!({ + "kind": kind, + "text": text, + "backend": "heuristic", + "queued_at": "2026-05-08T00:00:00Z", + }); + if schema_v2 { + body["schema"] = serde_json::json!("v2"); + } + if let Some(h) = project_hash { + body["project_hash"] = serde_json::json!(h); + body["events_path"] = serde_json::json!(format!("/nowhere/{h}.jsonl")); + } + let path = pending.join(file_name); + std::fs::write(&path, body.to_string()).unwrap(); + path +} + +fn project_events(xdg: &std::path::Path, project_hash: &str) -> String { + std::fs::read_to_string( + xdg.join("task-journal") + .join("events") + .join(format!("{project_hash}.jsonl")), + ) + .unwrap_or_default() +} + +const OTHER_PROJECT: &str = "0123456789abcdef"; + +#[test] +fn classify_worker_only_processes_its_own_projects_entries() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + let pending = dir.path().join("task-journal").join("pending"); + + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + true, + "UserPromptSubmit", + "We decided to use sqlite for the project A store", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + true, + "UserPromptSubmit", + "We decided to use postgres for the project B store", + ); + let legacy_foreign = write_pending_entry( + &pending, + "01JA0000000000000000000002.json", + Some(OTHER_PROJECT), + true, + "UserPromptSubmit", + "We decided to use mongodb for the project B cache", + ); + let legacy_no_project = write_pending_entry( + &pending, + "01JA0000000000000000000003.json", + None, + true, + "UserPromptSubmit", + "We decided to use redis for the unknown project cache", + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + assert!(!own.exists(), "the project's own entry is processed"); + assert!( + foreign.exists(), + "another project's entry must be left alone" + ); + assert!( + legacy_foreign.exists(), + "a legacy entry of another project too" + ); + assert!( + legacy_no_project.exists(), + "a legacy entry without project_hash is left for `pending retry`" + ); + let journal = project_events(dir.path(), &hash_a); + assert!(journal.contains("sqlite"), "own entry recorded: {journal}"); + for other in ["postgres", "mongodb", "redis"] { + assert!( + !journal.contains(other), + "{other} belongs to another project: {journal}" + ); + } +} + +#[test] +fn statusline_counts_only_this_projects_pending_entries() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Statusline scope"]) + .assert() + .success(); + + let pending = dir.path().join("task-journal").join("pending"); + for i in 0..2 { + write_pending_entry( + &pending, + &format!("{hash_a}.01JA000000000000000000000{i}.json"), + Some(&hash_a), + true, + "PostToolUse", + "own", + ); + } + for i in 0..3 { + write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA000000000000000000000{i}.json"), + Some(OTHER_PROJECT), + true, + "PostToolUse", + "foreign", + ); + } + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["statusline"]) + .assert() + .success() + .stdout(contains("pending: 2")); +} + +#[test] +fn pending_list_retry_and_hook_drain_skip_other_projects() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Project A"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + let pending = dir.path().join("task-journal").join("pending"); + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + false, + "Stop", + "own legacy-v1 chunk alpha", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + false, + "Stop", + "foreign chunk bravo", + ); + let legacy_foreign = write_pending_entry( + &pending, + "01JA0000000000000000000002.json", + Some(OTHER_PROJECT), + false, + "Stop", + "foreign legacy chunk charlie", + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["pending", "list"]) + .assert() + .success() + .stdout(contains("alpha")) + .stdout(contains("bravo").not()) + .stdout(contains("charlie").not()); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) - .env("TJ_CLASSIFIER_CLI", "/bin/false") - .args(["classify-worker", "--backend", "hybrid"]) + .current_dir(&workdir) + .args([ + "pending", + "retry", + "--mock-event-type", + "decision", + "--mock-task-id", + &task_id, + ]) + .assert() + .success() + .stdout(contains("1 drained")); + assert!(!own.exists(), "own entry drained"); + + // The hook's mock drain is project-scoped too. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args([ + "ingest-hook", + "--kind", + "Stop", + "--text", + "Live chunk", + "--mock-event-type", + "decision", + "--mock-task-id", + &task_id, + ]) .assert() .success(); - // Lockfile must not be left behind. - let state = dir.path().join("task-journal").join("state"); - if state.exists() { - for e in std::fs::read_dir(&state).unwrap() { - let p = e.unwrap().path(); - assert!( - !p.file_name() - .and_then(|n| n.to_str()) - .unwrap_or("") - .ends_with(".lock"), - "lockfile must be removed after worker exits, found: {p:?}" - ); - } - } + assert!( + foreign.exists(), + "another project's entry must be left alone" + ); + assert!( + legacy_foreign.exists(), + "a legacy entry of another project too" + ); + let journal = project_events(dir.path(), &hash_a); + assert!(journal.contains("alpha"), "own entry recorded: {journal}"); + assert!( + !journal.contains("bravo") && !journal.contains("charlie"), + "foreign chunks leaked into project A: {journal}" + ); } -/// Lockfile prevents concurrent workers in the same project. We can't -/// easily race two real spawns deterministically in a unit test, so -/// instead simulate a held lock by writing a lockfile with our own -/// (live) PID, then run classify-worker and assert it exits cleanly -/// without draining the queue. #[test] -fn classify_worker_respects_existing_lock() { +fn pending_gc_is_project_scoped_unless_all() { let dir = assert_fs::TempDir::new().unwrap(); - let cwd = std::env::current_dir().unwrap(); - let project_hash = tj_core::project_hash::from_path(&cwd).expect("compute project hash"); - - // Pre-create a v2 pending entry. + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); let pending = dir.path().join("task-journal").join("pending"); - std::fs::create_dir_all(&pending).unwrap(); - let events_path = dir - .path() - .join("task-journal") - .join("events") - .join(format!("{project_hash}.jsonl")); - std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); - let entry = pending.join("01locked.json"); - std::fs::write( - &entry, - serde_json::json!({ - "schema": "v2", - "kind": "UserPromptSubmit", - "text": "locked marker", - "project_hash": project_hash, - "events_path": events_path.to_string_lossy(), - "backend": "hybrid", - "queued_at": "2026-05-08T00:00:00Z", - }) - .to_string(), - ) - .unwrap(); - // Hand-roll a lockfile with this process's (live) PID. The - // worker should see the live PID and bail without touching the - // pending entry. - let state = dir.path().join("task-journal").join("state"); - std::fs::create_dir_all(&state).unwrap(); - let lock_path = state.join(format!("classifier-{project_hash}.lock")); - std::fs::write(&lock_path, format!("{}\n", std::process::id())).unwrap(); + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + true, + "PostToolUse", + "own", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + true, + "PostToolUse", + "foreign", + ); + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(3 * 86_400); + for p in [&own, &foreign] { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(old) + .unwrap(); + } - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .env("TJ_CLASSIFIER_CLI", "/bin/false") - .args(["classify-worker", "--backend", "hybrid"]) - .assert() - .success(); + let gc = |extra: &[&str]| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["pending-gc", "--days", "1"]) + .args(extra) + .assert() + .success(); + }; - // Pending entry must still be there — the second worker bailed. - assert!( - entry.exists(), - "pending entry must survive — second worker must not have drained it" - ); - // Our hand-rolled lockfile must still be there too — the bailing - // worker must NOT remove a lock it didn't acquire. + gc(&[]); + assert!(!own.exists(), "own stale entry collected"); assert!( - lock_path.exists(), - "lockfile must survive — bailing worker must not delete others' locks" + foreign.exists(), + "another project's entry is not ours to collect" ); + + gc(&["--all"]); + assert!(!foreign.exists(), "--all collects every project"); } // ===================================================================== @@ -2977,6 +4693,7 @@ fn precompact_hook_appends_marker_decision_to_open_task() { .unwrap_or("") .contains("Conversation compacted at") { + assert_eq!(v["meta"]["kind"], "compaction_marker", "{v}"); marker_lines += 1; } } @@ -3000,6 +4717,10 @@ fn precompact_hook_appends_marker_decision_to_open_task() { .not() .and(contains("single reasoning unit").not()), ); + + // It is bookkeeping, not a change the PR made. + let pr = tj_stdout(dir.path(), &workdir, &["export-pr", &task_id]); + assert!(!pr.contains("Conversation compacted at"), "{pr}"); } #[test] @@ -3308,6 +5029,92 @@ fn rewind_prompt_appends_correction_event() { .stdout(contains("[correction]").and(contains("/rewind"))); } +/// `meta.session_id` of the journal's events of `etype`. +fn session_ids_of(journal: &str, etype: &str) -> Vec<Option<String>> { + journal + .lines() + .map(|l| serde_json::from_str::<serde_json::Value>(l).unwrap()) + .filter(|e| e["type"] == etype) + .map(|e| e["meta"]["session_id"].as_str().map(str::to_string)) + .collect() +} + +#[test] +fn hook_written_events_carry_the_session_id() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + let hook = |prompt: &str| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_INGEST_SYNC", "1") + .env_remove("CLAUDE_CODE_SESSION_ID") + .current_dir(&workdir) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin( + serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": "sess-hook", + "prompt": prompt, + }) + .to_string(), + ) + .assert() + .success(); + }; + + // No open task: the prompt auto-opens one and is classified inline. + hook("We decided to use postgres for the journal store going forward"); + hook("/rewind go back to plan A"); + + let journal = project_events(dir.path(), &hash); + let want = vec![Some("sess-hook".to_string())]; + assert_eq!(session_ids_of(&journal, "open"), want, "auto-open event"); + assert_eq!( + session_ids_of(&journal, "decision"), + want, + "sync-classified" + ); + assert_eq!(session_ids_of(&journal, "correction"), want, "/rewind"); +} + +#[test] +fn worker_auto_open_carries_the_chunk_session_id() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + + let pending = dir.path().join("task-journal").join("pending"); + let entry = write_pending_entry( + &pending, + &format!("{hash}.01JA0000000000000000000000.json"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use postgres for the journal store going forward", + ); + let mut v: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&entry).unwrap()).unwrap(); + v["session_id"] = serde_json::json!("sess-chunk"); + std::fs::write(&entry, v.to_string()).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + let journal = project_events(dir.path(), &hash); + let want = vec![Some("sess-chunk".to_string())]; + assert_eq!(session_ids_of(&journal, "open"), want, "auto-open event"); + assert_eq!(session_ids_of(&journal, "decision"), want, "classified"); +} + #[test] fn install_hooks_wires_precompact_event() { let dir = assert_fs::TempDir::new().unwrap(); @@ -3530,6 +5337,123 @@ fn export_pr_omits_optional_sections_when_no_data() { ); } +/// Run `task-journal <args>` in `workdir` under `xdg` and return trimmed stdout. +fn tj_stdout(xdg: &std::path::Path, workdir: &std::path::Path, args: &[&str]) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg) + .current_dir(workdir) + .args(args) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap().trim().to_string() +} + +#[test] +fn export_pr_leaves_out_corrected_events() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let xdg = dir.path(); + + let task = tj_stdout(xdg, &workdir, &["create", "Corrected PR", "--goal", "g"]); + for (ty, text) in [ + ("decision", "Wrong decision: use Mongo"), + ("rejection", "Wrong rejection: SQLite too slow"), + ("evidence", "Wrong evidence: bench 5ms"), + ] { + let id = tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", ty, "--text", text], + ); + tj_stdout( + xdg, + &workdir, + &[ + "event-correct", + "--corrects", + &id, + "--task", + &task, + "--text", + "That was a mistake", + ], + ); + } + tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", "decision", "--text", "Use SQLite"], + ); + + let pr = tj_stdout(xdg, &workdir, &["export-pr", &task]); + assert!(!pr.contains("Wrong "), "{pr}"); + assert!(pr.contains("- Use SQLite"), "{pr}"); +} + +#[test] +fn export_pr_lists_the_same_decisions_and_rejections_as_the_pack() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let xdg = dir.path(); + + let task = tj_stdout(xdg, &workdir, &["create", "Same selection", "--goal", "g"]); + let event = |ty: &str, text: &str| { + tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", ty, "--text", text], + ) + }; + let ts = event("decision", "Adopt TypeScript"); + tj_stdout( + xdg, + &workdir, + &[ + "event", + &task, + "--type", + "supersede", + "--text", + "TS replaced", + "--supersedes", + &ts, + ], + ); + event("decision", "Adopt Rust"); + event("decision", "Adopt Rust"); + // Classifier guesses, not yet confirmed. + for (ty, text) in [ + (tj_core::event::EventType::Decision, "Maybe cache packs"), + (tj_core::event::EventType::Rejection, "Maybe drop Postgres"), + ] { + let mut e = tj_core::event::Event::new( + task.clone(), + ty, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text.to_string(), + ); + e.status = tj_core::event::EventStatus::Suggested; + append_jsonl_line(xdg, &serde_json::to_string(&e).unwrap()); + } + + let pr = tj_stdout(xdg, &workdir, &["export-pr", &task]); + let changes = pr.split("## ").find(|s| s.starts_with("Changes")).unwrap(); + assert!(!changes.contains("Adopt TypeScript"), "{pr}"); + assert_eq!(changes.matches("Adopt Rust").count(), 1, "{pr}"); + assert!( + changes.contains("- Maybe cache packs _(unconfirmed)_\n"), + "{pr}" + ); + assert!(pr.contains("- Maybe drop Postgres _(unconfirmed)_"), "{pr}"); +} + #[test] fn export_pr_unknown_task_id_exits_one_with_stderr_message() { let dir = assert_fs::TempDir::new().unwrap(); @@ -3555,12 +5479,9 @@ fn export_pr_unknown_task_id_exits_one_with_stderr_message() { .stderr(contains("task not found: tj-zzzz")); } -// v0.10.0: asyncRewake backlog signal. PostToolUse hook configured with -// asyncRewake:true in hooks.json sets TJ_ASYNC_REWAKE=1; when pending/ -// has more than PENDING_OVERFLOW_THRESHOLD (25) entries already queued, -// ingest-hook exits 2 with a wake-message on stdout. Sync hooks (or -// CLI invocations without the env var) must NEVER exit 2 — that would -// block the operation in Claude Code's hook contract. +// The PostToolUse hook used to exit 2 on a large pending backlog to wake the +// model (TJ_ASYNC_REWAKE). Nothing installs that env var, so the signal was +// dead code; a backlog must never turn the hook into a non-zero exit. fn seed_pending_chunks(pending_dir: &std::path::Path, count: usize) { std::fs::create_dir_all(pending_dir).unwrap(); for i in 0..count { @@ -3592,82 +5513,19 @@ fn posttooluse_payload() -> String { } #[test] -fn session_start_emits_watch_paths_for_existing_marker_files() { - // v0.10.2 X4: SessionStart envelope must include `watchPaths` with - // existing marker files (CLAUDE.md, README.md, .docs/plans). Files - // that don't exist are skipped — Claude Code's watcher logs an - // error and gives up on missing paths, so we don't emit them. +fn session_start_does_not_emit_watch_paths() { + // install-hooks never registers FileChanged, so asking Claude Code to + // watch marker files only made it watch them for nothing. let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(workdir.join(".docs").join("plans")).unwrap(); std::fs::write(workdir.join("CLAUDE.md"), "# Project rules").unwrap(); - // README.md intentionally absent — must NOT appear in watchPaths. - - let task_id = String::from_utf8( - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["create", "Watch paths test"]) - .assert() - .success() - .get_output() - .stdout - .clone(), - ) - .unwrap() - .trim() - .to_string(); - let _ = task_id; - - let body = String::from_utf8( - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["ingest-hook", "--kind", "SessionStart", "--text", ""]) - .assert() - .success() - .get_output() - .stdout - .clone(), - ) - .unwrap(); - let v: serde_json::Value = serde_json::from_str(body.trim()).unwrap(); - let watches = v["hookSpecificOutput"]["watchPaths"] - .as_array() - .expect("watchPaths must be present when at least one marker exists"); - let joined: String = watches - .iter() - .filter_map(|x| x.as_str()) - .collect::<Vec<_>>() - .join("|"); - assert!( - joined.contains("CLAUDE.md"), - "CLAUDE.md must be watched: {joined}" - ); - assert!( - joined.contains("plans"), - ".docs/plans must be watched: {joined}" - ); - assert!( - !joined.contains("README.md"), - "README.md does not exist, must NOT be watched: {joined}" - ); -} - -#[test] -fn session_start_omits_watch_paths_when_disabled_via_env() { - let dir = assert_fs::TempDir::new().unwrap(); - let workdir = dir.path().join("proj"); - std::fs::create_dir_all(&workdir).unwrap(); - std::fs::write(workdir.join("CLAUDE.md"), "# Project rules").unwrap(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "Watch paths env-disabled"]) + .args(["create", "Watch paths test"]) .assert() .success(); @@ -3675,7 +5533,6 @@ fn session_start_omits_watch_paths_when_disabled_via_env() { Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) - .env("TJ_WATCH_PATHS", "0") .current_dir(&workdir) .args(["ingest-hook", "--kind", "SessionStart", "--text", ""]) .assert() @@ -3688,19 +5545,18 @@ fn session_start_omits_watch_paths_when_disabled_via_env() { let v: serde_json::Value = serde_json::from_str(body.trim()).unwrap(); assert!( v["hookSpecificOutput"]["watchPaths"].is_null(), - "TJ_WATCH_PATHS=0 must suppress watchPaths emission" + "SessionStart must not emit watchPaths: {body}" ); assert!( v["hookSpecificOutput"]["additionalContext"].is_string(), - "additionalContext still emitted independently of watchPaths" + "resume context is still emitted: {body}" ); } #[test] -fn file_changed_hook_appends_evidence_to_active_task() { - // v0.10.2 X4: FileChanged hook handler should append an evidence - // event to the most-recent open task with the changed path - // (trimmed project-relative) and the change kind. +fn file_changed_payload_is_ignored() { + // No FileChanged handler any more: the payload carries no text, so the + // hook writes nothing even with an open task. let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); @@ -3710,7 +5566,7 @@ fn file_changed_hook_appends_evidence_to_active_task() { .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "FileChanged evidence test"]) + .args(["create", "Watched file ignored"]) .assert() .success() .get_output() @@ -3723,7 +5579,6 @@ fn file_changed_hook_appends_evidence_to_active_task() { let touched = workdir.join("CLAUDE.md"); std::fs::write(&touched, "# rules v2").unwrap(); - let stdin_payload = serde_json::json!({ "hook_event_name": "FileChanged", "file_path": touched.to_str().unwrap(), @@ -3734,8 +5589,9 @@ fn file_changed_hook_appends_evidence_to_active_task() { Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) + .args(["ingest-hook", "--backend", "heuristic"]) .write_stdin(stdin_payload) .assert() .success(); @@ -3747,8 +5603,7 @@ fn file_changed_hook_appends_evidence_to_active_task() { .args(["pack", &task_id, "--mode", "full"]) .assert() .success() - .stdout(contains("FileChanged (change)")) - .stdout(contains("CLAUDE.md")); + .stdout(contains("FileChanged").not()); } #[test] @@ -3788,7 +5643,7 @@ fn file_changed_hook_with_no_open_task_is_no_op() { } #[test] -fn asyncrewake_below_threshold_exits_zero() { +fn post_tool_use_backlog_never_exits_two() { let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); @@ -3797,13 +5652,13 @@ fn asyncrewake_below_threshold_exits_zero() { .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "Async wake test below threshold"]) + .args(["create", "Backlog test"]) .assert() .success(); - // Seed 5 entries — well under the 25 threshold. + // Seed 30 entries — over the old 25-entry wake threshold. let pending = dir.path().join("task-journal").join("pending"); - seed_pending_chunks(&pending, 5); + seed_pending_chunks(&pending, 30); Command::cargo_bin("task-journal") .unwrap() @@ -3811,51 +5666,56 @@ fn asyncrewake_below_threshold_exits_zero() { .env("TJ_ASYNC_REWAKE", "1") .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) + .args(["ingest-hook", "--backend", "heuristic"]) .write_stdin(posttooluse_payload()) .assert() - .success(); // exit 0, no wake + .success() + .stdout(contains("pending queue").not()); } #[test] -fn asyncrewake_overflow_exits_two_with_drain_hint() { +fn post_tool_use_queues_truncated_text() { let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": "Read", + "tool_input": {"file_path": "big.rs"}, + "tool_response": {"content": "ж".repeat(10_000)}, + }) + .to_string(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") .current_dir(&workdir) - .args(["create", "Async wake overflow test"]) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload) .assert() .success(); - // Seed 30 entries — over the 25 threshold. let pending = dir.path().join("task-journal").join("pending"); - seed_pending_chunks(&pending, 30); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .env("TJ_ASYNC_REWAKE", "1") - .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") - .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) - .write_stdin(posttooluse_payload()) - .assert() - .failure() - .code(2) - .stdout(contains("Task Journal pending queue")) - .stdout(contains("pending-gc")); + let entries: Vec<_> = std::fs::read_dir(&pending).unwrap().collect(); + assert_eq!(entries.len(), 1); + let body = std::fs::read_to_string(entries[0].as_ref().unwrap().path()).unwrap(); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + let chars = v["text"].as_str().unwrap().chars().count(); + assert!( + chars <= 2000, + "PostToolUse text must be capped, got {chars}" + ); + assert!( + chars > 100, + "the start of the tool call is kept, got {chars}" + ); } #[test] -fn asyncrewake_overflow_without_env_does_not_exit_two() { - // Sync hook safety: without TJ_ASYNC_REWAKE=1 we must NEVER exit 2 - // even on overflow, because exit 2 from a sync hook blocks the - // operation in Claude Code. CLI invocations and the PreCompact/Stop - // hooks (which stay sync) rely on this guarantee. +fn post_tool_use_output_wording_is_not_a_constraint() { + // "must be" / "requires" inside tool output used to become constraint + // events whose text was raw tool JSON. let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); @@ -3864,23 +5724,33 @@ fn asyncrewake_overflow_without_env_does_not_exit_two() { .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "Sync hook safety test"]) + .args(["create", "Tool output noise"]) .assert() .success(); - let pending = dir.path().join("task-journal").join("pending"); - seed_pending_chunks(&pending, 30); - + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": "Read", + "tool_input": {"file_path": "schema.json"}, + "tool_response": {"content": "the id field must be unique and requires an index"}, + }) + .to_string(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) - .env_remove("TJ_ASYNC_REWAKE") - .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .env("TJ_INGEST_SYNC", "1") .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) - .write_stdin(posttooluse_payload()) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload) .assert() - .success(); // exit 0, no wake — must not block sync hooks + .success(); + + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + let journal = project_events(dir.path(), &hash); + assert!( + !journal.contains("\"constraint\""), + "tool output must not become a constraint: {journal}" + ); } // --------------------------------------------------------------------------- @@ -3921,6 +5791,68 @@ fn search_does_not_crash_on_hyphenated_identifier() { .stdout(contains("tj-")); } +#[test] +fn search_with_an_empty_query_lists_tasks_newest_first() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()).current_dir(&workdir); + cmd + }; + let create = |title: &str| { + let out = tj().args(["create", title]).assert().success(); + String::from_utf8(out.get_output().stdout.clone()) + .unwrap() + .trim() + .to_string() + }; + let search = |args: &[&str]| { + let out = tj().arg("search").args(args).assert().success(); + String::from_utf8(out.get_output().stdout.clone()) + .unwrap() + .lines() + .map(str::to_string) + .collect::<Vec<_>>() + }; + + let older = create("Older task"); + let newer = create("Newer task"); + assert_eq!(search(&[""]), [newer.clone(), older.clone()]); + + tj().args(["event", &older, "--type", "decision", "--text", "Pick X"]) + .assert() + .success(); + assert_eq!(search(&[" "]), [older.clone(), newer.clone()]); + assert_eq!(search(&["", "--type", "decision"]), [older.as_str()]); + assert_eq!(search(&["", "--limit", "1"]), [older]); +} + +#[test] +fn search_in_a_project_without_a_journal_creates_no_state_db() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + + for query in ["", "anything"] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["search", query]) + .assert() + .success() + .stdout(""); + } + + let state = dir.path().join("task-journal").join("state"); + let created: Vec<_> = std::fs::read_dir(&state) + .map(|rd| rd.map(|e| e.unwrap().path()).collect()) + .unwrap_or_default(); + assert!(created.is_empty(), "{created:?}"); +} + #[test] fn search_does_not_crash_on_slash_or_colon() { // Same B1 family — paths and `ttl:30s`-style tokens used to crash. @@ -4444,6 +6376,109 @@ fn post_tool_use_emits_recall_additional_context() { ); } +/// One real (non-mock) PostToolUse hook run; `session: None` sends no +/// session id at all. Returns stdout. +fn post_tool_use(dir: &std::path::Path, session: Option<&str>, tool: &str, input: &str) -> String { + let mut payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": tool, + "tool_input": { "command": input }, + "tool_response": { "output": "" } + }); + if let Some(sid) = session { + payload["session_id"] = sid.into(); + } + + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +/// After every tool call the same hits used to come back, flooding the +/// agent's context. A session sees each hit once, on either recall path. +#[test] +fn push_recall_shows_each_hit_once_per_session() { + let (dir, _task_id) = seed_axum_rejection(); + let axum = "let's switch the server to axum"; + + let first = post_tool_use(dir.path(), Some("s-a"), "Bash", axum); + assert!(first.contains("⚠ recall"), "{first}"); + + let again = post_tool_use(dir.path(), Some("s-a"), "Bash", axum); + assert!(!again.contains("recall"), "{again}"); + let via_mcp = post_tool_use(dir.path(), Some("s-a"), "mcp__x__do", axum); + assert!(!via_mcp.contains("recall"), "{via_mcp}"); + + let other = post_tool_use(dir.path(), Some("s-b"), "Bash", axum); + assert!(other.contains("⚠ recall"), "{other}"); +} + +#[test] +fn push_recall_without_a_session_id_repeats_as_before() { + let (dir, _task_id) = seed_axum_rejection(); + let axum = "let's switch the server to axum"; + + for _ in 0..2 { + let out = post_tool_use(dir.path(), None, "Bash", axum); + assert!(out.contains("⚠ recall"), "{out}"); + } +} + +/// The agent just wrote that rejection on its own task in this session — +/// echoing it back is noise. Another session still gets it. +#[test] +fn push_recall_skips_what_this_session_wrote_on_its_task() { + let dir = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Own session host"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + let payload = serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": "s-own", + "prompt": "Tried switching the server to axum but it broke rmcp stdio." + }); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args([ + "ingest-hook", + "--mock-event-type", + "rejection", + "--mock-task-id", + &task_id, + ]) + .write_stdin(payload.to_string()) + .assert() + .success(); + let axum = "let's switch the server to axum"; + + let own = post_tool_use(dir.path(), Some("s-own"), "Bash", axum); + assert!(!own.contains("recall"), "{own}"); + + let other = post_tool_use(dir.path(), Some("s-other"), "Bash", axum); + assert!(other.contains("⚠ recall"), "{other}"); +} + #[test] fn post_tool_use_no_recall_when_no_match() { let (dir, task_id) = seed_axum_rejection(); @@ -5162,6 +7197,94 @@ fn export_memory_missing_task_exits_1() { .stderr(contains("task not found")); } +#[test] +fn export_memory_lists_the_same_decisions_as_the_pack_and_real_constraints() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + let (x, p) = (xdg.path(), proj.path()); + + let task = tj_stdout(x, p, &["create", "Memory selection", "--goal", "g"]); + let event = + |ty: &str, text: &str| tj_stdout(x, p, &["event", &task, "--type", ty, "--text", text]); + let ts = event("decision", "Adopt TypeScript"); + tj_stdout( + x, + p, + &[ + "event", + &task, + "--type", + "supersede", + "--text", + "TS replaced", + "--supersedes", + &ts, + ], + ); + for (ty, text) in [ + ("decision", "Wrong decision: use Mongo"), + ("constraint", "Wrong constraint: no disk"), + ] { + let id = event(ty, text); + tj_stdout( + x, + p, + &[ + "event-correct", + "--corrects", + &id, + "--task", + &task, + "--text", + "That was a mistake", + ], + ); + } + for (ty, text, kind) in [ + ( + tj_core::event::EventType::Decision, + "Conversation compacted at 2026-01-01T00:00:00Z", + "compaction_marker", + ), + ( + tj_core::event::EventType::Constraint, + "Model switched (auto): opus → haiku", + "model_switch", + ), + ] { + let mut e = tj_core::event::Event::new( + task.clone(), + ty, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text.to_string(), + ); + e.meta = serde_json::json!({ "kind": kind }); + append_jsonl_line(x, &serde_json::to_string(&e).unwrap()); + } + event("decision", "Adopt Rust"); + event("constraint", "Must run offline"); + + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", x) + .env("CLAUDE_CONFIG_DIR", claude.path()) + .current_dir(p) + .args(["export-memory", "--task", &task, "--dry-run"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); + for gone in ["TypeScript", "Wrong ", "compacted", "Model switched"] { + assert!(!out.contains(gone), "{gone} must be left out: {out}"); + } + assert!(out.contains("- Adopt Rust\n"), "{out}"); + assert!(out.contains("- Must run offline\n"), "{out}"); +} + #[test] fn embed_backfill_vectorises_events_then_idempotent() { // Pillar A / Phase 0: `embed --backfill` computes a vector per event using @@ -5922,6 +8045,48 @@ fn complete_command_runs_and_skips_cleanly_without_sessions() { .stdout(contains("session(s) to enrich")); } +#[test] +fn complete_enrich_without_claude_code_sessions_says_codex_is_not_read() { + // A Codex-only project has no `~/.claude/projects/<path>` dir: --enrich + // must say why it read nothing. `anthropic` without a key = no backend, + // so no model is ever called. + let dir = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Enrich me", "--goal", "g"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + for args in [ + vec!["complete", task_id.as_str(), "--enrich"], + vec!["complete", "--enrich", "--yes"], + ] { + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", dir.path()) + .env("CLAUDE_CONFIG_DIR", claude.path()) + .env_remove("ANTHROPIC_API_KEY") + .args(args) + .args(["--backend", "anthropic"]) + .assert() + .success() + .stderr(contains("Claude Code").and(contains("Codex"))); + } +} + #[test] fn complete_unknown_task_errors() { // A non-existent id is a hard error, not a silent no-op. @@ -6180,3 +8345,147 @@ fn artifact_add_renders_clickable_link_in_pack() { .success() .stdout(contains("[Design spec](https://example.com/spec.md) (doc)")); } + +#[test] +fn state_reports_the_sessions_own_task_as_json() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let state = |session: &str| -> serde_json::Value { + let out = tj() + .args(["state", "--session", session]) + .assert() + .success() + .get_output() + .stdout + .clone(); + serde_json::from_slice(&out).unwrap() + }; + + let empty = state("s1"); + assert_eq!(empty["schema"], "tj-state/1"); + assert!(empty["active"].is_null()); + assert!( + !dir.path().join("task-journal").join("state").exists(), + "no journal yet, so no state DB either" + ); + + let out = tj() + .args(["create", "Fix refresh", "--goal", "Stop dropping the token"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + tj().args([ + "event", + &task_id, + "--type", + "decision", + "--text", + "Use <= on expiry", + "--session", + "s1", + ]) + .assert() + .success(); + + let mine = state("s1"); + assert_eq!(mine["active"]["task_id"], task_id.as_str()); + assert_eq!(mine["active"]["goal"], "Stop dropping the token"); + assert_eq!(mine["active"]["counts"]["decision"], 1); + assert_eq!(mine["active"]["recent"][0]["text"], "Use <= on expiry"); + assert_eq!(mine["open_tasks"], 1); + + let other = state("s2"); + assert!( + other["active"].is_null(), + "another session must not inherit the task" + ); + assert_eq!(other["open_tasks"], 1); +} + +#[test] +fn event_can_record_a_suggested_entry_for_a_session() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let out = tj() + .args(["create", "Distill target"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + + tj().args([ + "event", + &task_id, + "--type", + "rejection", + "--text", + "Ruled out a cache", + "--suggested", + "--session", + "s9", + "--origin", + "mod-distill", + ]) + .assert() + .success(); + + let events_dir = dir.path().join("task-journal").join("events"); + let journal = std::fs::read_dir(&events_dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let last: serde_json::Value = serde_json::from_str( + std::fs::read_to_string(journal) + .unwrap() + .lines() + .last() + .unwrap(), + ) + .unwrap(); + assert_eq!(last["type"], "rejection"); + assert_eq!(last["status"], "suggested"); + assert_eq!(last["author"], "classifier"); + assert_eq!(last["meta"]["session_id"], "s9"); + assert_eq!(last["meta"]["origin"], "mod-distill"); +} + +#[test] +fn event_on_an_unknown_task_fails_and_writes_nothing() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + tj().args(["create", "Real task"]).assert().success(); + let events_dir = dir.path().join("task-journal").join("events"); + let journal = std::fs::read_dir(&events_dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let before = std::fs::read_to_string(&journal).unwrap(); + + tj().args(["event", "tj-typo", "--type", "decision", "--text", "x"]) + .assert() + .failure() + .stderr(contains("task not found: tj-typo")); + + assert_eq!(std::fs::read_to_string(&journal).unwrap(), before); +} diff --git a/crates/tj-cli/tests/dream.rs b/crates/tj-cli/tests/dream.rs new file mode 100644 index 0000000..0924829 --- /dev/null +++ b/crates/tj-cli/tests/dream.rs @@ -0,0 +1,253 @@ +//! End-to-end checks for the offline mining commands (`dream`, `backfill`) +//! against a sandboxed Claude Code projects dir and a mocked LLM backend. +//! Unix-only: the fixtures rely on the OS reporting a canonical cwd. +#![cfg(unix)] + +use assert_cmd::Command; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +struct Fixture { + xdg: assert_fs::TempDir, + proj: assert_fs::TempDir, + claude: assert_fs::TempDir, + /// `<claude>/projects/<encoded project path>` — where transcripts live. + sessions_dir: PathBuf, + events_path: PathBuf, + state_path: PathBuf, +} + +/// A project with an (empty) Claude Code sessions dir and events log dir. +fn fixture() -> Fixture { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + // The CLI resolves the project from its cwd, which the OS reports + // canonicalised (macOS `/var` → `/private/var`). + let path = proj.path().canonicalize().unwrap(); + let sessions_dir = + claude + .path() + .join("projects") + .join(tj_core::session::discovery::encode_project_path( + &path.to_string_lossy(), + )); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let hash = tj_core::project_hash::from_path(&path).unwrap(); + let events_path = xdg + .path() + .join("task-journal") + .join("events") + .join(format!("{hash}.jsonl")); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = xdg + .path() + .join("task-journal") + .join("state") + .join(format!("{hash}.sqlite")); + Fixture { + xdg, + proj, + claude, + sessions_dir, + events_path, + state_path, + } +} + +/// Write a two-turn transcript containing `marker`, aged `ago_secs`. +fn write_session(fx: &Fixture, id: &str, marker: &str, ago_secs: u64) { + let p = fx.sessions_dir.join(format!("{id}.jsonl")); + let body = format!( + "{{\"type\":\"user\",\"uuid\":\"u1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{{\"content\":\"{marker} first question\"}}}}\n\ + {{\"type\":\"assistant\",\"uuid\":\"a1\",\"timestamp\":\"2026-01-01T00:00:01Z\",\"message\":{{\"content\":[{{\"type\":\"text\",\"text\":\"an answer\"}}]}}}}\n\ + {{\"type\":\"user\",\"uuid\":\"u2\",\"timestamp\":\"2026-01-01T00:00:02Z\",\"message\":{{\"content\":\"second question\"}}}}\n" + ); + std::fs::write(&p, body).unwrap(); + let f = std::fs::File::options().write(true).open(&p).unwrap(); + f.set_modified(SystemTime::now() - Duration::from_secs(ago_secs)) + .unwrap(); +} + +/// Tie task `tj-dream` to each session via `meta.session_id`. +fn write_task_events(fx: &Fixture, session_ids: &[&str]) { + use tj_core::event::{Author, Event, EventType, Source}; + let mut w = tj_core::storage::JsonlWriter::open(&fx.events_path).unwrap(); + for sid in session_ids { + let mut e = Event::new( + "tj-dream", + EventType::Open, + Author::User, + Source::Cli, + "Dream task".into(), + ); + e.meta = serde_json::json!({ "session_id": sid }); + w.append(&e).unwrap(); + } + w.flush_durable().unwrap(); +} + +/// Mock OpenAI-compatible server: transcripts with OKSESSION succeed (no +/// events), transcripts with FAILME get a 500. +fn mock_llm() -> (mockito::ServerGuard, Vec<mockito::Mock>) { + let mut server = mockito::Server::new(); + let ok = server + .mock("POST", "/v1/chat/completions") + .match_body(mockito::Matcher::Regex("OKSESSION".into())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"choices":[{"message":{"content":"[]"}}]}"#) + .create(); + let fail = server + .mock("POST", "/v1/chat/completions") + .match_body(mockito::Matcher::Regex("FAILME".into())) + .with_status(500) + .create(); + (server, vec![ok, fail]) +} + +fn dream(fx: &Fixture, url: &str, extra: &[&str]) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .current_dir(fx.proj.path()) + .env("XDG_DATA_HOME", fx.xdg.path()) + .env("CLAUDE_CONFIG_DIR", fx.claude.path()) + .env("TJ_OLLAMA_URL", url) + .args(["dream", "--backend", "ollama"]) + .args(extra) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn dream_scoped_run_does_not_advance_the_watermark() { + let fx = fixture(); + write_session(&fx, "sess-old", "OKSESSION", 7200); + write_session(&fx, "sess-new", "OKSESSION", 3600); + write_task_events(&fx, &["sess-old", "sess-new"]); + let (server, _mocks) = mock_llm(); + + let scoped = dream(&fx, &server.url(), &["--limit", "1"]); + assert!(scoped.contains("1 session(s) processed"), "{scoped}"); + + // The --limit run skipped sess-old, so an unscoped run still sees both. + let full = dream(&fx, &server.url(), &[]); + assert!(full.contains("2 session(s) processed"), "{full}"); + + // Everything mined cleanly → watermark reached the newest session. + let again = dream(&fx, &server.url(), &[]); + assert!(again.contains("0 session(s) processed"), "{again}"); +} + +#[test] +fn dream_session_with_a_failed_chunk_is_mined_again() { + let fx = fixture(); + write_session(&fx, "sess-old", "OKSESSION", 7200); + write_session(&fx, "sess-new", "FAILME", 3600); + write_task_events(&fx, &["sess-old", "sess-new"]); + let (server, _mocks) = mock_llm(); + + let first = dream(&fx, &server.url(), &[]); + assert!(first.contains("2 session(s) processed"), "{first}"); + + // sess-old was clean so the watermark moved to it; sess-new failed and + // must stay in scope. + let second = dream(&fx, &server.url(), &[]); + assert!(second.contains("1 session(s) processed"), "{second}"); +} + +fn backfill(fx: &Fixture, extra: &[&str]) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .current_dir(fx.proj.path()) + .env("XDG_DATA_HOME", fx.xdg.path()) + .env("CLAUDE_CONFIG_DIR", fx.claude.path()) + .args(["backfill"]) + .args(extra) + .assert() + .success() + .get_output() + .stderr + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn backfill_imports_a_session_only_mentioned_in_unrelated_text() { + use tj_core::event::{Author, Event, EventType, Source}; + let fx = fixture(); + write_session(&fx, "sess-mention", "hello", 60); + // Another task's text names the session id; it was never imported. + let e = Event::new( + "tj-other", + EventType::Finding, + Author::User, + Source::Cli, + "see sess-mention for the repro".into(), + ); + let mut w = tj_core::storage::JsonlWriter::open(&fx.events_path).unwrap(); + w.append(&e).unwrap(); + w.flush_durable().unwrap(); + + let first = backfill(&fx, &[]); + assert!(first.contains("Imported 1 task(s)"), "{first}"); + + // Now it really was imported (its events carry meta.session_id). + let second = backfill(&fx, &[]); + assert!(second.contains("already imported"), "{second}"); + assert!(second.contains("Imported 0 task(s)"), "{second}"); +} + +#[test] +fn backfill_limit_counts_only_sessions_not_yet_imported() { + let fx = fixture(); + write_session(&fx, "sess-old", "hello", 3600); + write_session(&fx, "sess-new", "hello", 60); + + let first = backfill(&fx, &["--limit", "1"]); + assert!(first.contains("Imported 1 task(s)"), "{first}"); + + // sess-new is imported now; the next --limit 1 run must reach sess-old. + let second = backfill(&fx, &["--limit", "1"]); + assert!(second.contains("Imported 1 task(s)"), "{second}"); +} + +#[test] +fn backfill_indexes_imported_tasks_into_sqlite() { + let fx = fixture(); + write_session(&fx, "sess-index", "hello", 60); + + let out = backfill(&fx, &[]); + assert!(out.contains("Imported 1 task(s)"), "{out}"); + + // search / pack read SQLite, so the task must be there right away. + let conn = rusqlite::Connection::open(&fx.state_path).unwrap(); + let tasks: i64 = conn + .query_row("SELECT COUNT(*) FROM tasks", [], |r| r.get(0)) + .unwrap(); + assert_eq!(tasks, 1); +} + +#[test] +fn mining_without_claude_code_sessions_says_codex_is_not_read() { + // A Codex-only project has no `~/.claude/projects/<path>` dir: say why + // nothing was mined instead of a bare zero. + let fx = fixture(); + std::fs::remove_dir(&fx.sessions_dir).unwrap(); + + let out = dream(&fx, "http://127.0.0.1:9", &[]); + assert!( + out.contains("Claude Code") && out.contains("Codex"), + "{out}" + ); + + let err = backfill(&fx, &[]); + assert!( + err.contains("Claude Code") && err.contains("Codex"), + "{err}" + ); +} diff --git a/crates/tj-cli/tests/plugin_metadata.rs b/crates/tj-cli/tests/plugin_metadata.rs index 8c9ebb2..4532c79 100644 --- a/crates/tj-cli/tests/plugin_metadata.rs +++ b/crates/tj-cli/tests/plugin_metadata.rs @@ -42,6 +42,38 @@ fn plugin_manifests_carry_the_crate_version() { ); } +/// `cargo publish` uploads tj-cli and tj-mcp with their `task-journal-core` +/// requirement taken from the version literal, which Cargo cannot inherit +/// from `workspace.package.version`. A stale literal ships binaries built +/// against the previous core release. +#[test] +fn internal_dependency_pins_carry_the_crate_version() { + let want = format!("version = \"{}\"", env!("CARGO_PKG_VERSION")); + + for rel in [ + "Cargo.toml", + "crates/tj-cli/Cargo.toml", + "crates/tj-mcp/Cargo.toml", + ] { + let body = std::fs::read_to_string(repo_root().join(rel)) + .unwrap_or_else(|e| panic!("read {rel}: {e}")); + + let pins = body.lines().filter(|l| { + l.contains("package = \"task-journal-") || l.trim_start().starts_with("task-journal-") + }); + + for line in pins { + if line.contains("workspace = true") { + continue; + } + assert!( + line.contains(&want), + "{rel}: internal dependency pin is stale: {line}" + ); + } + } +} + /// The MCP server is declared once, in `plugin.json`. A second declaration in /// `plugin/.mcp.json` made Claude Code see the same stdio server twice. #[test] diff --git a/crates/tj-core/src/artifacts.rs b/crates/tj-core/src/artifacts.rs index d5bf53e..83cfa22 100644 --- a/crates/tj-core/src/artifacts.rs +++ b/crates/tj-core/src/artifacts.rs @@ -11,6 +11,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; +use std::sync::LazyLock; /// Structured artifacts collected from one or many events. All vectors /// are deduplicated (case-sensitive) by the `merge` constructor — the @@ -102,8 +103,10 @@ pub fn extract(text: &str) -> Artifacts { // Word boundary on \b avoids matching inside longer non-hex tokens // (e.g. ULIDs are base32, but adjacent digits + letters could // technically pass — the boundary keeps matches clean). + static COMMIT_HASH: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"\b[0-9a-f]{7,40}\b").unwrap()); static_re( - r"\b[0-9a-f]{7,40}\b", + &COMMIT_HASH, |m| { // Reject if all-digits (could be a year, an ID, a port). // A real abbreviated commit always has at least one letter. @@ -116,31 +119,28 @@ pub fn extract(text: &str) -> Artifacts { ); // GitHub / GitLab PR URLs. - static_re( - r"https?://[A-Za-z0-9.\-]+/[A-Za-z0-9_./\-]+/(?:pull|merge_requests)/\d+", - |m| a.pr_urls.push(m.to_string()), - text, - ); + static PR_URL: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"https?://[A-Za-z0-9.\-]+/[A-Za-z0-9_./\-]+/(?:pull|merge_requests)/\d+") + .unwrap() + }); + static_re(&PR_URL, |m| a.pr_urls.push(m.to_string()), text); // Short PR references: "PR #51", "PR#51", "pull request #51". Anchored to // the PR / "pull request" keyword so a bare "#3" in prose (step #3, issue // #3) is NOT captured. Normalised to "PR #<n>" so it dedupes cleanly and // renders next to full URLs under the same `PRs:` group. - if let Ok(re) = Regex::new(r"(?i)\b(?:PR|pull request)\s*#(\d+)\b") { - for cap in re.captures_iter(text) { - if let Some(m) = cap.get(1) { - a.pr_urls.push(format!("PR #{}", m.as_str())); - } + static PR_REF: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"(?i)\b(?:PR|pull request)\s*#(\d+)\b").unwrap()); + for cap in PR_REF.captures_iter(text) { + if let Some(m) = cap.get(1) { + a.pr_urls.push(format!("PR #{}", m.as_str())); } } // Ticket IDs: ABC-123. At least 2 letters to avoid matching version // strings like v1-2 and minimum 1 digit. - static_re( - r"\b[A-Z]{2,}-\d+\b", - |m| a.linked_issues.push(m.to_string()), - text, - ); + static TICKET: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\b[A-Z]{2,}-\d+\b").unwrap()); + static_re(&TICKET, |m| a.linked_issues.push(m.to_string()), text); // File paths — heuristic: path-like tokens with at least one slash // (and an extension) OR a leading ./ . Path segments allow a @@ -148,8 +148,11 @@ pub fn extract(text: &str) -> Artifacts { // etc are captured as artifacts. Tight enough to skip prose, loose // enough to catch the common cases (src/foo.rs, ./bar.ts, // crates/tj-core/src/db.rs). + static FILE_PATH: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"(?:\./|\.?[A-Za-z0-9_\-]+/)+[A-Za-z0-9_.\-]+\.[A-Za-z0-9]{1,8}\b").unwrap() + }); static_re( - r"(?:\./|\.?[A-Za-z0-9_\-]+/)+[A-Za-z0-9_.\-]+\.[A-Za-z0-9]{1,8}\b", + &FILE_PATH, |m| { // Reject version-dir lookalikes: a numeric-only "extension" // (foo/1.0.0, …/superpowers/5.1.0) is a version number, not a @@ -170,13 +173,12 @@ pub fn extract(text: &str) -> Artifacts { // branch. The bare-`branch <name>` form is intentionally dropped — // it caused too many false positives in journal events that // mention the word "branch" without naming one. - if let Ok(re) = - Regex::new(r"\bgit\s+(?:checkout\s+-b|switch\s+-c|branch)\s+([A-Za-z0-9._/\-]+)") - { - for cap in re.captures_iter(text) { - if let Some(m) = cap.get(1) { - a.branch_names.push(m.as_str().to_string()); - } + static BRANCH: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"\bgit\s+(?:checkout\s+-b|switch\s+-c|branch)\s+([A-Za-z0-9._/\-]+)").unwrap() + }); + for cap in BRANCH.captures_iter(text) { + if let Some(m) = cap.get(1) { + a.branch_names.push(m.as_str().to_string()); } } @@ -194,11 +196,9 @@ fn dedup(v: &mut Vec<String>) { v.retain(|x| seen.insert(x.clone())); } -fn static_re(pat: &str, mut f: impl FnMut(&str), text: &str) { - if let Ok(re) = Regex::new(pat) { - for m in re.find_iter(text) { - f(m.as_str()); - } +fn static_re(re: &Regex, mut f: impl FnMut(&str), text: &str) { + for m in re.find_iter(text) { + f(m.as_str()); } } diff --git a/crates/tj-core/src/classifier/agent_sdk.rs b/crates/tj-core/src/classifier/agent_sdk.rs index 777a54c..6ec870d 100644 --- a/crates/tj-core/src/classifier/agent_sdk.rs +++ b/crates/tj-core/src/classifier/agent_sdk.rs @@ -50,10 +50,10 @@ pub trait CommandRunner: Send + Sync { } } -/// Build the base `claude` invocation shared by both runners: print mode, the -/// pinned model, the JSON envelope, an isolated MCP config, and — critically — -/// the [`IN_CLASSIFIER_ENV`] recursion marker. The argv runner appends the -/// prompt as a positional arg; the stdin runner feeds it on stdin. Extracted so +/// Build the base `claude` invocation: print mode, the pinned model, the JSON +/// envelope, an isolated MCP config, and — critically — the +/// [`IN_CLASSIFIER_ENV`] recursion marker. The prompt goes on stdin (see +/// [`ClaudeBinaryStdinRunner`]), never as a positional arg. Extracted so /// a unit test can assert the marker is present without spawning `claude` (the /// missing marker is exactly what let the fork bomb through before). fn base_claude_command(model: &str) -> Command { @@ -79,12 +79,6 @@ fn base_claude_command(model: &str) -> Command { const DISABLED_TOOLS: &str = "Bash Read Edit Write Glob Grep Task WebFetch \ WebSearch NotebookEdit TodoWrite BashOutput KillBash"; -/// Production runner: invokes the local `claude` binary in print mode, pinned -/// to the given model, asking for the JSON envelope and an isolated MCP config -/// (`--strict-mcp-config` keeps the project's own MCP servers — including this -/// very journal — out of the classification subprocess). -pub struct ClaudeBinaryRunner; - /// Build the error for a non-zero `claude -p` exit. With `--output-format /// json` claude reports the real cause (invalid model, usage limit, auth) as /// JSON on **stdout**, not stderr — so surface both, capped, or the user just @@ -129,10 +123,12 @@ fn claude_timeout() -> std::time::Duration { /// Wait for `child` up to `timeout`, draining stdout/stderr concurrently so a /// full pipe can't deadlock the wait. On timeout the child is killed and an -/// error returned; otherwise the captured output is handed back. +/// error naming `what` (e.g. "claude -p") is returned; otherwise the captured +/// output is handed back. pub(crate) fn wait_with_timeout( mut child: std::process::Child, timeout: std::time::Duration, + what: &str, ) -> anyhow::Result<std::process::Output> { use std::io::Read; let mut out_pipe = child.stdout.take(); @@ -159,7 +155,7 @@ pub(crate) fn wait_with_timeout( if start.elapsed() >= timeout { let _ = child.kill(); let _ = child.wait(); - anyhow::bail!("`claude -p` timed out after {}s", timeout.as_secs()); + anyhow::bail!("`{what}` timed out after {}s", timeout.as_secs()); } std::thread::sleep(std::time::Duration::from_millis(150)); }; @@ -170,31 +166,15 @@ pub(crate) fn wait_with_timeout( }) } -impl CommandRunner for ClaudeBinaryRunner { - fn run(&self, model: &str, prompt: &str) -> anyhow::Result<String> { - let child = base_claude_command(model) - .arg(prompt) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn `claude` (is Claude Code installed and on PATH?)")?; - let output = wait_with_timeout(child, claude_timeout())?; - if !output.status.success() { - return Err(claude_exit_error( - output.status, - &output.stdout, - &output.stderr, - )); - } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } -} - -/// Like [`ClaudeBinaryRunner`] but feeds the prompt on **stdin** instead of as -/// an argv argument. Use for large prompts (e.g. a whole session transcript in -/// dream backfill) that would otherwise blow the per-argument size limit -/// (`E2BIG`, ~128 KiB on Linux). `claude -p` with no positional prompt reads -/// the prompt from stdin. +/// Production runner: invokes the local `claude` binary in print mode, pinned +/// to the given model, asking for the JSON envelope and an isolated MCP config +/// (`--strict-mcp-config` keeps the project's own MCP servers — including this +/// very journal — out of the classification subprocess). The prompt goes on +/// **stdin**, not as an argv argument: that sidesteps the `--disallowed-tools` +/// collision (see [`CommandRunner::feeds_prompt_on_stdin`]) and the +/// per-argument size limit (`E2BIG`, ~128 KiB on Linux) that a whole session +/// transcript in dream backfill would blow. `claude -p` with no positional +/// prompt reads the prompt from stdin. pub struct ClaudeBinaryStdinRunner; impl CommandRunner for ClaudeBinaryStdinRunner { @@ -219,7 +199,7 @@ impl CommandRunner for ClaudeBinaryStdinRunner { .context("claude stdin was not captured")? .write_all(prompt.as_bytes()) .context("failed to write prompt to claude stdin")?; - let output = wait_with_timeout(child, claude_timeout())?; + let output = wait_with_timeout(child, claude_timeout(), "claude -p")?; if !output.status.success() { return Err(claude_exit_error( output.status, @@ -397,6 +377,7 @@ mod tests { text: "We adopted Rust for the journal core.".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, } } @@ -511,9 +492,9 @@ mod tests { assert!(format!("{err}").contains("error"), "got: {err}"); } - /// Regression: the production classifier MUST feed the prompt on stdin. With - /// the argv runner the prompt lands right after `--disallowed-tools` and the - /// current `claude` CLI swallows it as bogus deny-rules, failing every + /// Regression: the production classifier MUST feed the prompt on stdin. As + /// an argv positional the prompt lands right after `--disallowed-tools` and + /// the current `claude` CLI swallows it as bogus deny-rules, failing every /// classification (the "139 pending" backlog). Lock the choice here. #[test] fn production_runner_feeds_prompt_on_stdin() { @@ -521,8 +502,6 @@ mod tests { default_runner().feeds_prompt_on_stdin(), "classifier prompt must go on stdin, not as an argv positional" ); - // the argv runner is the one that collides — keep the contrast explicit - assert!(!ClaudeBinaryRunner.feeds_prompt_on_stdin()); assert!(ClaudeBinaryStdinRunner.feeds_prompt_on_stdin()); } } diff --git a/crates/tj-core/src/classifier/heuristic.rs b/crates/tj-core/src/classifier/heuristic.rs index db99a70..6861f5e 100644 --- a/crates/tj-core/src/classifier/heuristic.rs +++ b/crates/tj-core/src/classifier/heuristic.rs @@ -31,6 +31,11 @@ pub fn try_heuristic(input: &ClassifyInput) -> Option<ClassifyOutput> { let task_id_guess = input.recent_tasks.first().map(|t| t.task_id.clone()); for (patterns, etype, confidence) in RULES { + // Tool output is JSON and logs full of "must be" / "requires" / + // "rejected": only a test result is a reliable signal there. + if input.tool_output && *etype != EventType::Evidence { + continue; + } for p in *patterns { if lower.contains(p) { return Some(ClassifyOutput { @@ -189,6 +194,7 @@ mod tests { last_events: vec![], constraints: vec![], }], + tool_output: false, } } @@ -255,6 +261,31 @@ mod tests { assert_eq!(out.event_type, EventType::Decision); } + /// Tool output is JSON and logs full of "must be" / "requires" / "rate + /// limit": only a test result is a reliable signal there. + #[test] + fn tool_output_only_matches_test_results() { + let tool = |text: &str| ClassifyInput { + tool_output: true, + ..input(text) + }; + + assert!(try_heuristic(&tool( + r#"Read: {"file_path":"a.rs"} → {"content":"the id must be unique; requires auth"}"# + )) + .is_none()); + assert!(try_heuristic(&tool( + r#"Bash: {"command":"x"} → {"stdout":"we'll use the cached build, rejected 2 files"}"# + )) + .is_none()); + + let out = try_heuristic(&tool( + r#"Bash: {"command":"cargo test"} → {"stdout":"all 12 tests pass"}"#, + )) + .unwrap(); + assert_eq!(out.event_type, EventType::Evidence); + } + #[test] fn suggested_text_is_truncated() { let long = "we'll use ".to_string() + &"x".repeat(500); diff --git a/crates/tj-core/src/classifier/http.rs b/crates/tj-core/src/classifier/http.rs index c828687..27d572b 100644 --- a/crates/tj-core/src/classifier/http.rs +++ b/crates/tj-core/src/classifier/http.rs @@ -130,6 +130,7 @@ mod tests { text: "We adopted Rust.".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }) .unwrap(); @@ -163,6 +164,7 @@ mod tests { text: "x".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }); let elapsed = start.elapsed(); diff --git a/crates/tj-core/src/classifier/hybrid.rs b/crates/tj-core/src/classifier/hybrid.rs index 3ae5e09..5a07de3 100644 --- a/crates/tj-core/src/classifier/hybrid.rs +++ b/crates/tj-core/src/classifier/hybrid.rs @@ -141,6 +141,7 @@ mod tests { last_events: vec![], constraints: vec![], }], + tool_output: false, } } @@ -173,7 +174,8 @@ mod tests { #[test] fn from_env_constructs_without_key() { - // SAFETY: tests in this crate do not concurrently read these env vars. + // SAFETY: the shared env lock serializes the crate's env tests. + let _env = crate::test_env_lock(); let prev_key = std::env::var("ANTHROPIC_API_KEY").ok(); // Force heuristic-only by disabling both LLM backends via an order that // names no real one, so this stays deterministic regardless of whether diff --git a/crates/tj-core/src/classifier/mock.rs b/crates/tj-core/src/classifier/mock.rs index 8086cb3..66bf9e4 100644 --- a/crates/tj-core/src/classifier/mock.rs +++ b/crates/tj-core/src/classifier/mock.rs @@ -34,6 +34,7 @@ mod tests { text: "ignored".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }) .unwrap(); assert_eq!(out.event_type, EventType::Decision); diff --git a/crates/tj-core/src/classifier/mod.rs b/crates/tj-core/src/classifier/mod.rs index 4748d17..bd22b1e 100644 --- a/crates/tj-core/src/classifier/mod.rs +++ b/crates/tj-core/src/classifier/mod.rs @@ -9,6 +9,11 @@ pub struct ClassifyInput { pub text: String, pub author_hint: String, pub recent_tasks: Vec<TaskContext>, + /// The chunk is a tool call's input + output (PostToolUse). The + /// heuristic stage then only trusts test results; the LLM prompt is + /// unchanged. + #[serde(skip)] + pub tool_output: bool, } #[derive(Debug, Clone, Serialize)] @@ -86,10 +91,11 @@ mod tests { /// other tests in this crate. #[test] fn tj_classifier_model_env_var_overrides_http_default() { + let _env = crate::test_env_lock(); let prev_model = std::env::var("TJ_CLASSIFIER_MODEL").ok(); let prev_key = std::env::var("ANTHROPIC_API_KEY").ok(); - // SAFETY: tests in this crate do not concurrently read these env vars. + // SAFETY: the shared env lock serializes the crate's env tests. unsafe { std::env::remove_var("TJ_CLASSIFIER_MODEL"); std::env::set_var("ANTHROPIC_API_KEY", "test-key-do-not-use"); @@ -133,6 +139,7 @@ mod tests { text: "Adopted Rust for the journal".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }; let s = serde_json::to_string(&i).unwrap(); assert!(s.contains("Adopted Rust")); diff --git a/crates/tj-core/src/classifier/prompt.rs b/crates/tj-core/src/classifier/prompt.rs index 5f0b75e..07fbda9 100644 --- a/crates/tj-core/src/classifier/prompt.rs +++ b/crates/tj-core/src/classifier/prompt.rs @@ -122,6 +122,7 @@ mod tests { last_events: vec!["[hypothesis] PKCE vs implicit".into()], constraints: vec![], }], + tool_output: false, }; let p = build(&input); assert!(p.contains("We adopted PKCE.")); @@ -145,6 +146,7 @@ mod tests { constraints: vec![], }) .collect(), + tool_output: false, }; let p = build(&input); assert!( @@ -160,6 +162,7 @@ mod tests { text: "anything".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }; let p = build(&input); assert!(p.contains("## Examples"), "Examples section missing"); @@ -188,6 +191,7 @@ mod tests { last_events: vec!["[decision] use ureq".into()], constraints: vec!["must stay sync (no tokio)".into()], }], + tool_output: false, }; let p = build(&input); assert!(p.contains("Known constraints for tj-9:")); @@ -207,6 +211,7 @@ mod tests { last_events: vec!["[finding] y".into()], constraints: vec![], }], + tool_output: false, }; let p = build(&input); assert!(!p.contains("Known constraints")); @@ -219,6 +224,7 @@ mod tests { text: "Hello".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }; let p = build(&input); assert!(p.contains("(no active tasks)")); diff --git a/crates/tj-core/src/completeness.rs b/crates/tj-core/src/completeness.rs index dc327d8..ce1d0a6 100644 --- a/crates/tj-core/src/completeness.rs +++ b/crates/tj-core/src/completeness.rs @@ -109,7 +109,12 @@ pub fn assess( let mut evidence = 0usize; let mut suggested = 0usize; { - let mut stmt = conn.prepare("SELECT type, status FROM events_index WHERE task_id = ?1")?; + // Bookkeeping (compaction markers) is not a decision to verify, and a + // corrected event no longer needs confirming. + let mut stmt = conn.prepare( + "SELECT type, status FROM events_index + WHERE task_id = ?1 AND bookkeeping = 0 AND corrected_by IS NULL", + )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) })?; @@ -439,6 +444,25 @@ mod tests { .any(|g| g.kind == GapKind::DecisionNoEvidence)); } + #[test] + fn compaction_marker_alone_is_not_an_unverified_decision() { + let (_d, c) = conn(); + open_task(&c, "t3m"); + c.execute("UPDATE tasks SET goal='g' WHERE task_id='t3m'", []) + .unwrap(); + let marker = Event::new( + "t3m", + EventType::Decision, + Author::Classifier, + Source::Hook, + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events…".into(), + ); + crate::db::index_event(&c, &marker).unwrap(); + + let r = assess(&c, "t3m", 0).unwrap(); + assert!(r.is_complete(), "{:?}", r.gaps); + } + #[test] fn suggested_unconfirmed_counts() { use crate::event::EventStatus; @@ -457,6 +481,37 @@ mod tests { assert!(g.detail.contains('2')); } + #[test] + fn corrected_suggested_event_is_not_unconfirmed() { + use crate::event::EventStatus; + let (_d, c) = conn(); + open_task(&c, "t4c"); + c.execute("UPDATE tasks SET goal='g' WHERE task_id='t4c'", []) + .unwrap(); + let mut wrong = Event::new( + "t4c", + EventType::Finding, + Author::Agent, + Source::Hook, + "x".into(), + ); + wrong.status = EventStatus::Suggested; + let mut corr = Event::new( + "t4c", + EventType::Correction, + Author::User, + Source::Cli, + "y".into(), + ); + corr.status = EventStatus::Confirmed; + corr.corrects = Some(wrong.event_id.clone()); + crate::db::index_event(&c, &wrong).unwrap(); + crate::db::index_event(&c, &corr).unwrap(); + + let r = assess(&c, "t4c", 0).unwrap(); + assert!(r.is_complete(), "{:?}", r.gaps); + } + #[test] fn pending_leak_fires_when_count_positive() { let (_d, c) = conn(); diff --git a/crates/tj-core/src/consolidate.rs b/crates/tj-core/src/consolidate.rs index ffbe61c..646302d 100644 --- a/crates/tj-core/src/consolidate.rs +++ b/crates/tj-core/src/consolidate.rs @@ -202,6 +202,7 @@ mod tests { #[test] fn summarize_skips_when_backend_forced_none() { + let _env = crate::test_env_lock(); std::env::set_var("TJ_CONSOLIDATE_BACKEND", "none"); let r = summarize(&["chose ledger".into()], 5, None).unwrap(); std::env::remove_var("TJ_CONSOLIDATE_BACKEND"); diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index e116ce0..04de66a 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -150,6 +150,56 @@ CREATE INDEX IF NOT EXISTS idx_emb_project_tier ON embeddings(project_hash, tier ALTER TABLE events_index ADD COLUMN memory_tier TEXT NOT NULL DEFAULT 'episodic'; "#; +/// v0.30.0 per-session queries — `session_id` projects an event's +/// `meta.session_id`. Clearing `index_state` makes the next +/// `ingest_new_events` replay the whole log (every projection is an +/// idempotent upsert), which fills the column for events indexed before. +const MIGRATION_009: &str = r#" +ALTER TABLE events_index ADD COLUMN session_id TEXT; +CREATE INDEX IF NOT EXISTS idx_events_session_time ON events_index(session_id, timestamp); +DELETE FROM index_state; +"#; + +/// v0.30.0 corrections — `corrected_by` holds the `event_id` of the +/// `correction` event whose `corrects` points at this event, so packs leave +/// the corrected event out without re-reading the log. Clearing +/// `index_state` replays the log once to fill it for existing events. +const MIGRATION_010: &str = r#" +ALTER TABLE events_index ADD COLUMN corrected_by TEXT; +DELETE FROM index_state; +"#; + +/// v0.30.0 bookkeeping — `bookkeeping` flags machine-written events that are +/// not reasoning (see [`is_bookkeeping`]) so active decisions, export-pr, +/// recall and the global memory can skip them. Clearing `index_state` +/// replays the log once to flag existing events. +const MIGRATION_011: &str = r#" +ALTER TABLE events_index ADD COLUMN bookkeeping INTEGER NOT NULL DEFAULT 0; +DELETE FROM index_state; +"#; + +/// v0.30.0 pack cache holds only the stable body; the header and the gaps +/// are rendered on every call. Rows cached as the whole pack text go. +const MIGRATION_012: &str = r#" +DELETE FROM task_pack_cache; +"#; + +/// v0.30.0 projection marker — `projection_state` holds the `index_state` row +/// as this version last wrote it. An older binary still running (or a +/// downgrade) ingests or rebuilds without the 0.30 columns and moves +/// `index_state`; the mismatch makes the next ingest replay the log to +/// re-derive them. It starts empty, so the first ingest after the upgrade +/// replays the log once — which also fills `author`, the event's writer, so +/// only deliberate writes bind a session to a task. +const MIGRATION_013: &str = r#" +CREATE TABLE IF NOT EXISTS projection_state ( + project_hash TEXT PRIMARY KEY, + last_indexed_event_id TEXT NOT NULL, + updated_at TEXT NOT NULL +); +ALTER TABLE events_index ADD COLUMN author TEXT; +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -185,6 +235,26 @@ const MIGRATIONS: &[Migration] = &[ version: 8, sql: MIGRATION_008, }, + Migration { + version: 9, + sql: MIGRATION_009, + }, + Migration { + version: 10, + sql: MIGRATION_010, + }, + Migration { + version: 11, + sql: MIGRATION_011, + }, + Migration { + version: 12, + sql: MIGRATION_012, + }, + Migration { + version: 13, + sql: MIGRATION_013, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -211,9 +281,27 @@ fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { if applied.contains(&migration.version) { continue; } - conn.execute_batch(migration.sql) + + // One IMMEDIATE transaction per migration: it takes the write lock up + // front, so a second process opening the same fresh DB waits here and + // then sees the version as applied instead of re-running its ALTERs. + // The migration and its row commit together — a failure halfway rolls + // the whole migration back instead of leaving a partial schema. + let tx = + rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate) + .with_context(|| format!("begin schema migration v{:03}", migration.version))?; + let already_applied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = ?1)", + rusqlite::params![migration.version], + |r| r.get(0), + )?; + if already_applied { + continue; + } + + tx.execute_batch(migration.sql) .with_context(|| format!("apply schema migration v{:03}", migration.version))?; - conn.execute( + tx.execute( "INSERT INTO schema_migrations(version, applied_at) VALUES (?1, ?2)", rusqlite::params![ migration.version, @@ -226,6 +314,8 @@ fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { migration.version ) })?; + tx.commit() + .with_context(|| format!("commit schema migration v{:03}", migration.version))?; } Ok(()) } @@ -258,6 +348,27 @@ pub fn upsert_task_from_event( ON CONFLICT(task_id) DO UPDATE SET last_event_at = ?4", rusqlite::params![event.task_id, title, project_hash, event.timestamp, parent_id], )?; + + // Goal and external refs given at creation ride in the open + // event's meta so a rebuild from the JSONL restores them. A replay + // over an existing row keeps a goal changed since. + if let Some(goal) = event.meta.get("goal").and_then(|v| v.as_str()) { + conn.execute( + "UPDATE tasks SET goal = COALESCE(goal, ?2) WHERE task_id = ?1", + rusqlite::params![event.task_id, goal], + )?; + } + for reference in meta_strings(&event.meta, "external") { + add_task_external(conn, &event.task_id, reference)?; + } + } + EventType::Amend => { + if let Some(goal) = event.meta.get("goal").and_then(|v| v.as_str()) { + set_task_goal(conn, &event.task_id, goal)?; + } + for reference in meta_strings(&event.meta, "external_add") { + add_task_external(conn, &event.task_id, reference)?; + } } EventType::Close => { conn.execute( @@ -304,6 +415,15 @@ pub fn upsert_task_from_event( Ok(()) } +/// The string items of the JSON array at `meta[key]`; empty when absent. +fn meta_strings<'a>(meta: &'a serde_json::Value, key: &str) -> impl Iterator<Item = &'a str> { + meta.get(key) + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + .filter_map(|v| v.as_str()) +} + use std::io::BufRead; pub fn list_all_projects(state_dir: impl AsRef<Path>) -> anyhow::Result<Vec<String>> { @@ -324,55 +444,44 @@ pub fn list_all_projects(state_dir: impl AsRef<Path>) -> anyhow::Result<Vec<Stri Ok(out) } +/// Replay the whole log from scratch: `search_fts` is cleared once, then every +/// event is indexed again. Commits in chunks like [`ingest_new_events`]. pub fn rebuild_state( conn: &Connection, jsonl_path: impl AsRef<Path>, project_hash: &str, ) -> anyhow::Result<usize> { - let f = match std::fs::File::open(&jsonl_path) { - Ok(f) => f, - // A fresh project has no events log on disk yet — there is simply nothing - // to read, which is not an error (this is what crashed task_create the - // first time the journal was touched in a new worktree). - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(e) => { - return Err(anyhow::Error::new(e).context(format!("open {:?}", jsonl_path.as_ref()))) - } - }; - let reader = std::io::BufReader::new(f); + replay(conn, jsonl_path.as_ref(), project_hash, true) +} - let tx = conn.unchecked_transaction()?; - let mut count = 0; - let mut last_event_id: Option<String> = None; - for (i, line) in reader.lines().enumerate() { - let line = line.with_context(|| format!("read line {i}"))?; - if line.trim().is_empty() { - continue; - } - // Malformed JSONL lines are skipped with a warning so that one bad - // event cannot abort an otherwise-recoverable rebuild. SQL errors - // still propagate — those indicate schema/integrity problems. - let event: Event = match serde_json::from_str(&line) { - Ok(e) => e, - Err(err) => { - tracing::warn!( - line_number = i + 1, - error = %err, - "skipping malformed JSONL line in rebuild_state" - ); - continue; +/// The events of a JSONL log in order. Malformed lines are skipped with a +/// warning so that one bad event cannot abort an otherwise-recoverable replay; +/// read errors still propagate. +fn log_events(f: std::fs::File) -> impl Iterator<Item = anyhow::Result<Event>> { + std::io::BufReader::new(f) + .lines() + .enumerate() + .filter_map(|(i, line)| { + let line = match line.with_context(|| format!("read line {i}")) { + Ok(line) => line, + Err(e) => return Some(Err(e)), + }; + if line.trim().is_empty() { + return None; } - }; - upsert_task_from_event(&tx, &event, project_hash)?; - index_event(&tx, &event)?; - last_event_id = Some(event.event_id.clone()); - count += 1; - } - if let Some(eid) = last_event_id.as_deref() { - record_last_indexed(&tx, project_hash, eid)?; - } - tx.commit()?; - Ok(count) + + match serde_json::from_str(&line) { + Ok(event) => Some(Ok(event)), + Err(err) => { + tracing::warn!( + line_number = i + 1, + error = %err, + "skipping malformed JSONL line" + ); + None + } + } + }) } /// Returns whether a task with this id has been recorded in the derived @@ -437,17 +546,30 @@ pub fn set_task_outcome( } /// Append an external reference to `tasks.external`. The column is -/// stored as a comma-separated list — small, append-mostly, no -/// uniqueness constraint. Acceptable shapes (loose, not enforced): -/// `beads:claude-memory-rsw`, `github:#42`, `jira:PROJ-1234`. +/// stored as a comma-separated list — small, append-mostly. A reference +/// already in the list is not added again, so replaying the events that +/// carry it is idempotent; an unknown task is a no-op. Acceptable shapes +/// (loose, not enforced): `beads:claude-memory-rsw`, `github:#42`, +/// `jira:PROJ-1234`. pub fn add_task_external(conn: &Connection, task_id: &str, reference: &str) -> anyhow::Result<()> { - let current: Option<String> = conn + let current: Option<Option<String>> = conn .query_row( "SELECT external FROM tasks WHERE task_id = ?1", rusqlite::params![task_id], |r| r.get::<_, Option<String>>(0), ) + .optional() .with_context(|| format!("read external for {task_id}"))?; + let Some(current) = current else { + return Ok(()); + }; + if current + .as_deref() + .is_some_and(|s| s.split(',').any(|r| r == reference)) + { + return Ok(()); + } + let next = match current { Some(s) if !s.is_empty() => format!("{s},{reference}"), _ => reference.to_string(), @@ -479,6 +601,32 @@ pub fn task_id_by_external(conn: &Connection, reference: &str) -> anyhow::Result Ok(id) } +/// The open task of `project_hash` whose most recent event carries +/// `meta.session_id == session_id` — the task a live agent session is +/// working on. `None` when the session has no event on an open task. +/// +/// Only deliberate writes count: hooks and the classifier fall back to the +/// newest open task and stamp the current session, so their events (and +/// bookkeeping) would bind a fresh session to another session's task. +pub fn active_task_for_session( + conn: &Connection, + project_hash: &str, + session_id: &str, +) -> anyhow::Result<Option<String>> { + let id: Option<String> = conn + .query_row( + "SELECT ei.task_id FROM events_index ei + JOIN tasks t ON t.task_id = ei.task_id + WHERE ei.session_id = ?2 AND t.project_hash = ?1 AND t.status = 'open' + AND ei.author IN ('user', 'agent') AND ei.bookkeeping = 0 + ORDER BY ei.timestamp DESC LIMIT 1", + rusqlite::params![project_hash, session_id], + |r| r.get::<_, String>(0), + ) + .optional()?; + Ok(id) +} + /// Read-only metadata bundle used by pack rendering (and TUI list /// teasers in v0.4.0+). Returns `None` for unknown tasks. #[derive(Debug, Clone, Default)] @@ -725,14 +873,15 @@ pub fn reclassify_task_artifacts(conn: &Connection, task_id: &str) -> anyhow::Re /// branches) across every event of a task, deduplicated. Reads the /// per-event JSON payload that `ingest_new_events` populated. Skips /// events whose `artifacts` column is NULL or unparseable rather than -/// failing the pack render. +/// failing the pack render, and corrected events, so a wrong hash or path +/// stops raising a gap once a correction retires it. pub fn task_artifacts( conn: &Connection, task_id: &str, ) -> anyhow::Result<crate::artifacts::Artifacts> { let mut stmt = conn.prepare( "SELECT artifacts FROM events_index - WHERE task_id = ?1 AND artifacts IS NOT NULL + WHERE task_id = ?1 AND artifacts IS NOT NULL AND corrected_by IS NULL ORDER BY timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| r.get::<_, String>(0))?; @@ -746,47 +895,68 @@ pub fn task_artifacts( Ok(acc) } -/// Look up the most recent `event_id` we've ingested for this project. -/// Returns `None` when the project has never been indexed (first call, -/// or migration v002 just landed on an existing 0.1.x DB). -fn last_indexed_event_id(conn: &Connection, project_hash: &str) -> anyhow::Result<Option<String>> { - let mut stmt = - conn.prepare("SELECT last_indexed_event_id FROM index_state WHERE project_hash = ?1")?; - let mut rows = stmt.query(rusqlite::params![project_hash])?; - if let Some(row) = rows.next()? { - Ok(Some(row.get::<_, String>(0)?)) - } else { - Ok(None) - } +/// How far the log is indexed for a project: the `index_state` row as +/// `(last_indexed_event_id, updated_at)`. `None` when the project has never +/// been indexed (first call, or a migration cleared the marker). +type IndexMark = (String, String); + +/// The mark stored in `table`: `index_state`, which every version writes, or +/// `projection_state`, the copy only this version writes. +fn read_mark( + conn: &Connection, + table: &str, + project_hash: &str, +) -> anyhow::Result<Option<IndexMark>> { + Ok(conn + .query_row( + &format!( + "SELECT last_indexed_event_id, updated_at FROM {table} WHERE project_hash = ?1" + ), + rusqlite::params![project_hash], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?) } fn record_last_indexed( conn: &Connection, project_hash: &str, event_id: &str, -) -> anyhow::Result<()> { - conn.execute( - "INSERT INTO index_state(project_hash, last_indexed_event_id, updated_at) - VALUES (?1, ?2, ?3) - ON CONFLICT(project_hash) DO UPDATE SET - last_indexed_event_id = excluded.last_indexed_event_id, - updated_at = excluded.updated_at", - rusqlite::params![ - project_hash, - event_id, - chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) - ], - )?; - Ok(()) +) -> anyhow::Result<IndexMark> { + let mark = ( + event_id.to_string(), + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + ); + for table in ["index_state", "projection_state"] { + conn.execute( + &format!( + "INSERT INTO {table}(project_hash, last_indexed_event_id, updated_at) + VALUES (?1, ?2, ?3) + ON CONFLICT(project_hash) DO UPDATE SET + last_indexed_event_id = excluded.last_indexed_event_id, + updated_at = excluded.updated_at" + ), + rusqlite::params![project_hash, mark.0, mark.1], + )?; + } + + Ok(mark) } +/// Events a replay indexes per transaction. Each commit moves the marker, so +/// a replay killed partway (a hook timeout) resumes after its last chunk +/// instead of starting over, and other writers wait one chunk at most. +const REPLAY_CHUNK: usize = 500; + /// Read only the tail of the JSONL log since the last call. The cheap path /// for hot loops (every MCP tool invocation): scan to the marker, ingest /// the rest, update the marker. /// -/// Falls back to a full [`rebuild_state`] in two cases: -/// - No marker yet for this project (first call after migration v002 or -/// on a brand-new install). +/// Falls back to a full replay (see [`rebuild_state`]) in three cases: +/// - No marker yet for this project (first call after a migration cleared +/// it, or a brand-new install). +/// - The marker is not the one this version last wrote: an older binary +/// ingested or rebuilt without the columns it does not know. /// - The stored marker is not present in the JSONL (corrupted / truncated /// file). A `tracing::warn!` is emitted so the operator notices. pub fn ingest_new_events( @@ -794,77 +964,151 @@ pub fn ingest_new_events( jsonl_path: impl AsRef<Path>, project_hash: &str, ) -> anyhow::Result<usize> { - let marker = match last_indexed_event_id(conn, project_hash)? { - Some(id) => id, - None => return rebuild_state(conn, jsonl_path, project_hash), - }; - - let f = match std::fs::File::open(&jsonl_path) { - Ok(f) => f, - // A fresh project has no events log on disk yet — there is simply nothing - // to read, which is not an error (this is what crashed task_create the - // first time the journal was touched in a new worktree). - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(e) => { - return Err(anyhow::Error::new(e).context(format!("open {:?}", jsonl_path.as_ref()))) - } - }; - let reader = std::io::BufReader::new(f); + replay(conn, jsonl_path.as_ref(), project_hash, false) +} - // First pass: confirm the marker still exists in the file. If it does - // not, the JSONL has been rewritten under us — we can't trust the - // marker, so we fall back to a full rebuild. - let tx = conn.unchecked_transaction()?; - let mut found_marker = false; +/// Index the log after the marker — or all of it when `from_scratch` or there +/// is no usable marker, clearing `search_fts` first — in chunks of +/// [`REPLAY_CHUNK`] events. Returns how many events it indexed. +fn replay( + conn: &Connection, + jsonl_path: &Path, + project_hash: &str, + mut from_scratch: bool, +) -> anyhow::Result<usize> { let mut count = 0; - let mut last_event_id: Option<String> = None; - for (i, line) in reader.lines().enumerate() { - let line = line.with_context(|| format!("read line {i}"))?; - if line.trim().is_empty() { - continue; - } - let event: Event = match serde_json::from_str(&line) { - Ok(e) => e, - Err(err) => { - tracing::warn!( - line_number = i + 1, - error = %err, - "skipping malformed JSONL line in ingest_new_events" - ); - continue; - } + + 'pass: loop { + let f = match std::fs::File::open(jsonl_path) { + Ok(f) => f, + // A fresh project has no events log on disk yet — there is simply nothing + // to read, which is not an error (this is what crashed task_create the + // first time the journal was touched in a new worktree). + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(count), + Err(e) => return Err(anyhow::Error::new(e).context(format!("open {jsonl_path:?}"))), }; - if !found_marker { - if event.event_id == marker { - found_marker = true; + let mut events = log_events(f); + let mut committed: Option<IndexMark> = None; + + loop { + // IMMEDIATE: the marker is read under the write lock, so two + // processes never index the same chunk. + let tx = rusqlite::Transaction::new_unchecked( + conn, + rusqlite::TransactionBehavior::Immediate, + )?; + let mark = read_mark(&tx, "index_state", project_hash)?; + + if committed.is_none() { + let ours = mark == read_mark(&tx, "projection_state", project_hash)?; + if mark.is_some() && !ours { + tracing::warn!( + project_hash = project_hash, + "index_state was written by another version — replaying the log" + ); + } + + match mark.filter(|_| !from_scratch && ours) { + None => clear_search_fts(&tx, project_hash)?, + Some((marker, _)) => { + let mut found = false; + for event in events.by_ref() { + if event?.event_id == marker { + found = true; + break; + } + } + if !found { + tracing::warn!( + project_hash = project_hash, + marker = marker.as_str(), + "last_indexed_event_id not found in JSONL — falling back to full rebuild" + ); + from_scratch = true; + continue 'pass; + } + } + } + } else if mark != committed { + // Another process indexed past our last chunk: go on from its marker. + from_scratch = false; + continue 'pass; + } + + let mut in_chunk = 0; + let mut last_event_id = None; + for event in events.by_ref().take(REPLAY_CHUNK) { + let event = event?; + upsert_task_from_event(&tx, &event, project_hash)?; + index_event(&tx, &event)?; + last_event_id = Some(event.event_id); + in_chunk += 1; + } + + if let Some(eid) = last_event_id { + committed = Some(record_last_indexed(&tx, project_hash, &eid)?); + } + tx.commit()?; + count += in_chunk; + + if in_chunk < REPLAY_CHUNK { + return Ok(count); } - continue; } - upsert_task_from_event(&tx, &event, project_hash)?; - index_event(&tx, &event)?; - last_event_id = Some(event.event_id.clone()); - count += 1; - } - - if !found_marker { - // Discard the (empty) tx and rebuild from scratch. - drop(tx); - tracing::warn!( - project_hash = project_hash, - marker = marker.as_str(), - "last_indexed_event_id not found in JSONL — falling back to full rebuild" - ); - return rebuild_state(conn, jsonl_path, project_hash); } +} + +/// Drop this project's rows from `search_fts` before a replay re-inserts +/// them: one scan, where deleting per event scanned the table every time. +fn clear_search_fts(conn: &Connection, project_hash: &str) -> anyhow::Result<()> { + conn.execute( + "DELETE FROM search_fts + WHERE task_id NOT IN (SELECT task_id FROM tasks WHERE project_hash <> ?1)", + rusqlite::params![project_hash], + )?; - if let Some(eid) = last_event_id.as_deref() { - record_last_indexed(&tx, project_hash, eid)?; + Ok(()) +} + +/// The `search_fts` rowid of an event: a stable FNV-1a hash of its id. +/// `event_id` is UNINDEXED, so finding an event's row by it scans the whole +/// table; by rowid, re-indexing an event replaces its row in O(log n). +// ponytail: two ids hashing alike (odds ~n²/2^64) would share one search row; +// a mapping table removes that if it ever matters. +fn fts_rowid(event_id: &str) -> i64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in event_id.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + + (hash >> 1) as i64 +} + +/// Machine-written bookkeeping, not reasoning: the PreCompact boundary marker +/// (a `decision`) and the model-switch note (a `constraint`). Known by +/// `meta.kind`; events written before the writers set it, by text prefix. +pub fn is_bookkeeping(event: &Event) -> bool { + match event.meta.get("kind").and_then(|v| v.as_str()) { + Some(kind) => matches!(kind, "compaction_marker" | "model_switch"), + None => match event.event_type { + EventType::Decision => event.text.starts_with("Conversation compacted at"), + EventType::Constraint => event + .text + .starts_with(crate::reminder::MODEL_SWITCH_TEXT_PREFIX), + _ => false, + }, } - tx.commit()?; - Ok(count) } pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { + // An amend only changes task metadata, which upsert_task_from_event + // applies. It is not reasoning: keeping it out of events_index and + // search_fts keeps it out of packs, search, recall and memory sync. + if event.event_type == EventType::Amend { + return invalidate_pack_cascade(conn, &event.task_id); + } + let type_str = serde_json::to_value(event.event_type)? .as_str() .unwrap() @@ -895,22 +1139,32 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { } else { Some(serde_json::to_string(&artifacts)?) }; + let session_id = event.meta.get("session_id").and_then(|v| v.as_str()); + let author_str = serde_json::to_value(event.author)? + .as_str() + .unwrap() + .to_string(); conn.execute( - "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id, bookkeeping, author) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", rusqlite::params![ event.event_id, event.task_id, type_str, - event.timestamp, event.confidence, status_str, artifacts_json + event.timestamp, event.confidence, status_str, artifacts_json, session_id, + is_bookkeeping(event), author_str ], )?; - // search_fts has no PK; clear then insert to keep idempotent across rebuild_state replays. + // search_fts has no PK; replacing by the event's own rowid keeps it + // idempotent across replays. conn.execute( - "DELETE FROM search_fts WHERE event_id=?1", - rusqlite::params![event.event_id], - )?; - conn.execute( - "INSERT INTO search_fts(task_id, event_id, text, type) VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![event.task_id, event.event_id, event.text, type_str], + "INSERT OR REPLACE INTO search_fts(rowid, task_id, event_id, text, type) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + fts_rowid(&event.event_id), + event.task_id, + event.event_id, + event.text, + type_str + ], )?; if event.event_type == EventType::Decision { @@ -937,6 +1191,27 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { } } + // A correction retires the event it corrects from packs and export-pr. + // The target may sit in another task, whose cached pack is now stale too. + if event.event_type == EventType::Correction { + if let Some(target) = &event.corrects { + conn.execute( + "UPDATE events_index SET corrected_by=?1 WHERE event_id=?2", + rusqlite::params![event.event_id, target], + )?; + let target_task: Option<String> = conn + .query_row( + "SELECT task_id FROM events_index WHERE event_id=?1", + rusqlite::params![target], + |r| r.get(0), + ) + .optional()?; + if let Some(t) = target_task.filter(|t| *t != event.task_id) { + invalidate_pack_cascade(conn, &t)?; + } + } + } + if event.event_type == EventType::Evidence { let strength_str = event .evidence_strength @@ -962,13 +1237,33 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { Ok(()) } +/// Switch the DB to WAL. On a fresh file that needs an exclusive lock, and +/// SQLite answers SQLITE_BUSY at once — no busy handler, to avoid a deadlock — +/// when another connection is switching the same file: two processes opening +/// a fresh DB together. Retry briefly; once the file is in WAL it is a no-op. +fn enable_wal(conn: &Connection) -> anyhow::Result<()> { + let mut attempts = 0; + loop { + match conn.execute_batch("PRAGMA journal_mode=WAL;") { + Err(rusqlite::Error::SqliteFailure(e, _)) + if e.code == rusqlite::ErrorCode::DatabaseBusy && attempts < 100 => + { + attempts += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + } + res => return res.context("set WAL journal mode"), + } + } +} + pub fn open(path: impl AsRef<Path>) -> anyhow::Result<Connection> { if let Some(parent) = path.as_ref().parent() { std::fs::create_dir_all(parent).with_context(|| format!("create dir {parent:?}"))?; } let conn = Connection::open(&path).with_context(|| format!("open SQLite at {:?}", path.as_ref()))?; - conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?; + enable_wal(&conn)?; + conn.execute_batch("PRAGMA foreign_keys=ON;")?; apply_migrations(&conn).context("apply schema migrations")?; Ok(conn) } @@ -1489,6 +1784,65 @@ mod tests { ); } + #[test] + fn a_migration_failing_halfway_leaves_no_partial_schema() { + let d = TempDir::new().unwrap(); + let conn = Connection::open(d.path().join("state.sqlite")).unwrap(); + conn.execute_batch( + "CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL); + INSERT INTO schema_migrations VALUES (1, 'x'), (2, 'x');", + ) + .unwrap(); + conn.execute_batch(MIGRATION_001).unwrap(); + conn.execute_batch(MIGRATION_002).unwrap(); + // v003 adds goal, then outcome: make its second ALTER fail. + conn.execute_batch("ALTER TABLE tasks ADD COLUMN outcome TEXT;") + .unwrap(); + + assert!(apply_migrations(&conn).is_err()); + + let goal_cols: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('tasks') WHERE name = 'goal'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(goal_cols, 0, "v003's first ALTER must roll back"); + let v3: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 3", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(v3, 0); + } + + #[test] + fn concurrent_opens_of_a_fresh_db_both_succeed() { + let d = TempDir::new().unwrap(); + + for round in 0..20 { + let path = d.path().join(format!("state-{round}.sqlite")); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let (path, barrier) = (path.clone(), barrier.clone()); + std::thread::spawn(move || { + barrier.wait(); + open(&path).map(|_| ()) + }) + }) + .collect(); + + for h in handles { + let res = h.join().unwrap(); + assert!(res.is_ok(), "round {round}: {:#}", res.unwrap_err()); + } + } + } + fn make_text_event(text: &str) -> crate::event::Event { crate::event::Event::new( "tj-x", @@ -1530,6 +1884,24 @@ mod tests { ); } + #[test] + fn task_artifacts_leave_out_corrected_events() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let wrong = make_text_event("fixed in commit dead00beef, see src/gone.rs"); + let kept = make_text_event("also touched src/kept.rs"); + let mut corr = make_text_event("the hash and path above were wrong"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + for e in [&wrong, &kept, &corr] { + index_event(&conn, e).unwrap(); + } + + let arts = task_artifacts(&conn, "tj-x").unwrap(); + assert!(arts.commit_hashes.is_empty(), "{:?}", arts.commit_hashes); + assert_eq!(arts.files, vec!["src/kept.rs".to_string()]); + } + #[test] fn embed_pending_embeds_all_then_is_idempotent() { let d = TempDir::new().unwrap(); @@ -1989,6 +2361,132 @@ mod tests { } } + /// A log of `n` events: an open every 30 events, findings in between. + fn write_synthetic_log(path: &Path, n: usize) -> Vec<crate::event::Event> { + let mut events = Vec::with_capacity(n); + let mut task = String::new(); + for i in 0..n { + if i % 30 == 0 { + task = format!("tj-syn{i}"); + events.push(make_open_event(&task, &task)); + } else { + let mut e = make_text_event(&format!( + "step {i}: replay chunk marker in crates/tj-core/src/db.rs, commit {i:07x}ab" + )); + e.task_id = task.clone(); + events.push(e); + } + } + + let mut f = std::fs::File::create(path).unwrap(); + for e in &events { + write_event_line(&mut f, e); + } + + events + } + + fn count_rows(conn: &Connection, table: &str) -> i64 { + conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + /// Seconds a replay of an already indexed `n`-event log takes once + /// `index_state` is cleared, as the v009–v011 upgrade migrations do. + fn upgrade_replay_seconds(n: usize) -> f64 { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + write_synthetic_log(&jsonl, n); + let conn = open(d.path().join("s.sqlite")).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch("DELETE FROM index_state").unwrap(); + + let started = std::time::Instant::now(); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), n); + let secs = started.elapsed().as_secs_f64(); + + assert_eq!(count_rows(&conn, "search_fts"), n as i64); + secs + } + + #[test] + fn a_full_replay_grows_linearly_with_the_log() { + // Best of two runs each, to keep scheduler noise out of the ratio. + let small = upgrade_replay_seconds(1500).min(upgrade_replay_seconds(1500)); + let large = upgrade_replay_seconds(3000).min(upgrade_replay_seconds(3000)); + + // Linear work doubles; the old per-event FTS scan quadrupled it. + assert!( + large < small * 3.0, + "replay of 2x events took {:.2}x as long ({small:.3}s -> {large:.3}s)", + large / small + ); + } + + #[test] + fn a_replay_killed_partway_resumes_after_its_last_committed_chunk() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + let events = write_synthetic_log(&jsonl, 1200); + let conn = open(d.path().join("s.sqlite")).unwrap(); + + // The process dies while indexing an event of the second chunk. + conn.execute_batch(&format!( + "CREATE TRIGGER kill BEFORE INSERT ON events_index WHEN NEW.event_id = '{}' + BEGIN SELECT RAISE(ABORT, 'killed'); END;", + events[700].event_id + )) + .unwrap(); + assert!(ingest_new_events(&conn, &jsonl, ph).is_err()); + conn.execute_batch("DROP TRIGGER kill").unwrap(); + + let committed = count_rows(&conn, "events_index"); + assert!( + committed > 0 && committed <= 700, + "the chunks before the kill stay committed, got {committed}" + ); + + let resumed = ingest_new_events(&conn, &jsonl, ph).unwrap(); + assert_eq!( + resumed as i64, + 1200 - committed, + "the resumed replay must not redo committed chunks" + ); + assert_eq!(count_rows(&conn, "events_index"), 1200); + assert_eq!(count_rows(&conn, "search_fts"), 1200); + } + + #[test] + fn concurrent_replays_share_the_work_without_locking_errors_or_duplicates() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + write_synthetic_log(&jsonl, 2000); + drop(open(&db).unwrap()); + + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let (jsonl, db, barrier) = (jsonl.clone(), db.clone(), barrier.clone()); + std::thread::spawn(move || { + let conn = open(&db).unwrap(); + barrier.wait(); + ingest_new_events(&conn, &jsonl, "feedfacefeedface").map(|_| ()) + }) + }) + .collect(); + for h in handles { + let res = h.join().unwrap(); + assert!(res.is_ok(), "{:#}", res.unwrap_err()); + } + + let conn = open(&db).unwrap(); + assert_eq!(count_rows(&conn, "events_index"), 2000); + assert_eq!(count_rows(&conn, "search_fts"), 2000); + } + #[test] fn rebuild_state_skips_malformed_jsonl_lines() { use std::io::Write; @@ -2215,6 +2713,447 @@ mod tests { assert_eq!(task_id_by_external(&conn, "loom:t-xy").unwrap(), None); } + /// An event of `task_id` stamped with `session` at a fixed `timestamp`. + fn session_event(task_id: &str, session: &str, timestamp: &str) -> crate::event::Event { + let mut e = make_text_event("work"); + e.task_id = task_id.into(); + e.timestamp = timestamp.into(); + e.meta = serde_json::json!({"session_id": session}); + e + } + + fn indexed_session_id(conn: &Connection, event_id: &str) -> Option<String> { + conn.query_row( + "SELECT session_id FROM events_index WHERE event_id = ?1", + rusqlite::params![event_id], + |r| r.get(0), + ) + .unwrap() + } + + #[test] + fn index_event_stores_the_session_id_from_meta() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + + let stamped = session_event("tj-s", "sess-1", "2026-01-01T00:00:00.000Z"); + let plain = make_text_event("no session"); + index_event(&conn, &stamped).unwrap(); + index_event(&conn, &plain).unwrap(); + + assert_eq!( + indexed_session_id(&conn, &stamped.event_id).as_deref(), + Some("sess-1") + ); + assert_eq!(indexed_session_id(&conn, &plain.event_id), None); + } + + #[test] + fn active_task_for_session_picks_the_open_task_with_its_latest_event() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let ph = "feedfacefeedface"; + + for id in ["tj-a", "tj-b", "tj-c"] { + upsert_task_from_event(&conn, &make_open_event(id, id), ph).unwrap(); + } + upsert_task_from_event(&conn, &make_open_event("tj-other", "o"), "otherproject0000") + .unwrap(); + let events = [ + session_event("tj-a", "s1", "2026-01-01T00:00:01.000Z"), + session_event("tj-b", "s1", "2026-01-01T00:00:02.000Z"), + session_event("tj-a", "s2", "2026-01-01T00:00:03.000Z"), + session_event("tj-c", "s1", "2026-01-01T00:00:04.000Z"), + session_event("tj-other", "s1", "2026-01-01T00:00:05.000Z"), + ]; + for e in &events { + upsert_task_from_event(&conn, e, ph).unwrap(); + index_event(&conn, e).unwrap(); + } + let mut close = make_text_event("done"); + close.task_id = "tj-c".into(); + close.event_type = crate::event::EventType::Close; + upsert_task_from_event(&conn, &close, ph).unwrap(); + + // tj-c has s1's latest event but is closed; tj-other is another project. + assert_eq!( + active_task_for_session(&conn, ph, "s1").unwrap().as_deref(), + Some("tj-b") + ); + assert_eq!( + active_task_for_session(&conn, ph, "s2").unwrap().as_deref(), + Some("tj-a") + ); + assert_eq!(active_task_for_session(&conn, ph, "s3").unwrap(), None); + } + + #[test] + fn hook_and_classifier_writes_never_bind_a_session_to_a_task() { + use crate::event::{Author, EventType}; + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let ph = "feedfacefeedface"; + upsert_task_from_event(&conn, &make_open_event("tj-t1", "T1"), ph).unwrap(); + + // Session S1 works on T1 through the agent. + let mut work = session_event("tj-t1", "s1", "2026-01-01T00:00:01.000Z"); + work.author = Author::Agent; + // A fresh session S2: its PostModelSwitch note and a Stop catch-up + // classifier event fall back to the newest open task, T1. + let mut switch = session_event("tj-t1", "s2", "2026-01-01T00:00:02.000Z"); + switch.author = Author::Classifier; + switch.event_type = EventType::Constraint; + switch.meta = serde_json::json!({"session_id": "s2", "kind": "model_switch"}); + let mut caught_up = session_event("tj-t1", "s2", "2026-01-01T00:00:03.000Z"); + caught_up.author = Author::Classifier; + for e in [&work, &switch, &caught_up] { + upsert_task_from_event(&conn, e, ph).unwrap(); + index_event(&conn, e).unwrap(); + } + + assert_eq!(active_task_for_session(&conn, ph, "s2").unwrap(), None); + assert_eq!( + active_task_for_session(&conn, ph, "s1").unwrap().as_deref(), + Some("tj-t1") + ); + } + + #[test] + fn upgrading_an_existing_db_backfills_session_ids_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-up", "Upgrade"); + let stamped = session_event("tj-up", "sess-old", "2026-01-01T00:00:01.000Z"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + write_event_line(&mut f, &stamped); + drop(f); + + // A pre-session_id database that has already indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + // Written straight to SQLite by older versions: the replay must keep them. + set_task_goal(&conn, "tj-up", "legacy goal").unwrap(); + add_task_external(&conn, "tj-up", "loom:t-legacy").unwrap(); + conn.execute_batch( + "DROP INDEX IF EXISTS idx_events_session_time; + ALTER TABLE events_index DROP COLUMN session_id; + DELETE FROM schema_migrations WHERE version = 9;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + assert_eq!( + indexed_session_id(&conn, &stamped.event_id).as_deref(), + Some("sess-old") + ); + assert_eq!( + active_task_for_session(&conn, ph, "sess-old") + .unwrap() + .as_deref(), + Some("tj-up") + ); + let meta = task_metadata(&conn, "tj-up").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("legacy goal")); + assert_eq!(meta.external.as_deref(), Some("loom:t-legacy")); + } + + #[test] + fn upgrading_an_existing_db_links_corrections_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-x", "Upgrade"); + let wrong = make_text_event("Migration done (wrong)"); + let mut corr = make_text_event("Migration NOT done"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &wrong, &corr] { + write_event_line(&mut f, e); + } + drop(f); + + // A pre-corrected_by database that has already indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "ALTER TABLE events_index DROP COLUMN corrected_by; + DELETE FROM schema_migrations WHERE version = 10;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + let corrected_by: Option<String> = conn + .query_row( + "SELECT corrected_by FROM events_index WHERE event_id = ?1", + rusqlite::params![wrong.event_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); + } + + #[test] + fn upgrading_wipes_packs_cached_as_whole_text() { + let d = TempDir::new().unwrap(); + let db = d.path().join("s.sqlite"); + let conn = open(&db).unwrap(); + conn.execute_batch( + "INSERT INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) + VALUES ('tj-x', 'compact', '# Old whole pack', '', 1); + DELETE FROM schema_migrations WHERE version = 12;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + let cached: i64 = conn + .query_row("SELECT COUNT(*) FROM task_pack_cache", [], |r| r.get(0)) + .unwrap(); + assert_eq!(cached, 0); + } + + #[test] + fn upgrading_an_existing_db_flags_old_bookkeeping_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + // Written before the writers set meta.kind: known by text alone. + let open_ev = make_open_event("tj-x", "Upgrade"); + let mut marker = make_text_event("Conversation compacted at 2026-01-01T00:00:00Z; …"); + marker.event_type = crate::event::EventType::Decision; + let mut switch = make_text_event("Model switched (user): opus → sonnet"); + switch.event_type = crate::event::EventType::Constraint; + let real = make_text_event("Use SQLite"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &marker, &switch, &real] { + write_event_line(&mut f, e); + } + drop(f); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "ALTER TABLE events_index DROP COLUMN bookkeeping; + DELETE FROM schema_migrations WHERE version = 11;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + let flagged: Vec<String> = conn + .prepare("SELECT event_id FROM events_index WHERE bookkeeping = 1 ORDER BY event_id") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .collect::<Result<_, _>>() + .unwrap(); + let mut expected = vec![marker.event_id.clone(), switch.event_id.clone()]; + expected.sort(); + assert_eq!(flagged, expected); + } + + #[test] + fn a_rebuild_by_an_older_binary_is_repaired_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-old", "Old binary"); + let mut wrong = session_event("tj-old", "sess-1", "2026-01-01T00:00:01.000Z"); + wrong.event_type = crate::event::EventType::Decision; + let mut corr = make_text_event("Not this"); + corr.task_id = "tj-old".into(); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + let amend = amend_event("tj-old", serde_json::json!({"goal": "g"})); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &wrong, &corr, &amend] { + write_event_line(&mut f, e); + } + drop(f); + + let conn = open(d.path().join("s.sqlite")).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + // A 0.29 server cannot parse the trailing amend, so it misses the + // marker and rebuilds: its rows lack the 0.30 columns, and it records + // the last event it could read. + conn.execute_batch( + "UPDATE events_index SET session_id = NULL, corrected_by = NULL, bookkeeping = 0;", + ) + .unwrap(); + conn.execute( + "UPDATE index_state SET last_indexed_event_id = ?1, updated_at = '2026-01-02T00:00:00.000Z'", + rusqlite::params![corr.event_id], + ) + .unwrap(); + + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + assert_eq!( + active_task_for_session(&conn, ph, "sess-1") + .unwrap() + .as_deref(), + Some("tj-old") + ); + let corrected_by: Option<String> = conn + .query_row( + "SELECT corrected_by FROM events_index WHERE event_id = ?1", + rusqlite::params![wrong.event_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); + assert_eq!( + ingest_new_events(&conn, &jsonl, ph).unwrap(), + 0, + "once repaired, the projection is in sync again" + ); + } + + #[test] + fn upgrading_from_0_29_replays_the_log_exactly_once() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + let events = write_synthetic_log(&jsonl, 40); + + // A 0.29 database (schema v008) that has indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "DROP INDEX idx_events_session_time; + ALTER TABLE events_index DROP COLUMN session_id; + ALTER TABLE events_index DROP COLUMN corrected_by; + ALTER TABLE events_index DROP COLUMN bookkeeping; + ALTER TABLE events_index DROP COLUMN author; + DROP TABLE projection_state; + DELETE FROM schema_migrations WHERE version >= 9;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), 40); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), 0); + + let authored: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events_index WHERE author = 'user'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(authored, events.len() as i64); + } + + #[test] + fn open_meta_goal_and_external_are_restored_by_a_rebuild() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let mut open_ev = make_open_event("tj-g", "Goal task"); + open_ev.meta = serde_json::json!({ + "title": "Goal task", + "goal": "Ship PKCE", + "external": ["loom:t-42", "github:#7"], + }); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + drop(f); + + let conn = open(d.path().join("fresh.sqlite")).unwrap(); + rebuild_state(&conn, &jsonl, ph).unwrap(); + + let meta = task_metadata(&conn, "tj-g").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("Ship PKCE")); + assert_eq!(meta.external.as_deref(), Some("loom:t-42,github:#7")); + assert_eq!( + task_id_by_external(&conn, "loom:t-42").unwrap().as_deref(), + Some("tj-g") + ); + } + + fn amend_event(task_id: &str, meta: serde_json::Value) -> crate::event::Event { + let mut e = crate::event::Event::new( + task_id, + crate::event::EventType::Amend, + crate::event::Author::User, + crate::event::Source::Cli, + "amend".into(), + ); + e.meta = meta; + e + } + + #[test] + fn amend_events_are_replayed_idempotently_and_stay_out_of_the_index() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let mut open_ev = make_open_event("tj-am", "Amended"); + open_ev.meta = serde_json::json!({"title": "Amended", "goal": "first goal"}); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + write_event_line( + &mut f, + &amend_event("tj-am", serde_json::json!({"goal": "second goal"})), + ); + for _ in 0..2 { + write_event_line( + &mut f, + &amend_event("tj-am", serde_json::json!({"external_add": ["beads:x"]})), + ); + } + // An amend for a task that does not exist must not break ingest. + write_event_line( + &mut f, + &amend_event("tj-ghost", serde_json::json!({"external_add": ["beads:y"]})), + ); + drop(f); + + let conn = open(d.path().join("fresh.sqlite")).unwrap(); + rebuild_state(&conn, &jsonl, ph).unwrap(); + // Replaying over an existing DB (e.g. after a migration) changes nothing. + rebuild_state(&conn, &jsonl, ph).unwrap(); + + let meta = task_metadata(&conn, "tj-am").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("second goal")); + assert_eq!(meta.external.as_deref(), Some("beads:x")); + + let indexed: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events_index WHERE type = 'amend'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(indexed, 0, "amend is metadata, not a pack/search event"); + let pack = crate::pack::assemble(&conn, "tj-am", crate::pack::PackMode::Full).unwrap(); + assert!(pack.text.contains("**Goal**: second goal"), "{}", pack.text); + assert!(!pack.text.contains("[amend]"), "{}", pack.text); + } + #[test] fn open_event_meta_parent_id_is_persisted() { let d = tempfile::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/dream/agent_sdk.rs b/crates/tj-core/src/dream/agent_sdk.rs deleted file mode 100644 index 9c37806..0000000 --- a/crates/tj-core/src/dream/agent_sdk.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Claude CLI ("agent SDK") implementation of [`DreamBackend`]. -//! -//! Runs the local, already-authenticated `claude` binary in print mode, -//! **pinned to Haiku**, so offline dream backfill works on the Claude -//! subscription with no `ANTHROPIC_API_KEY`. Mirrors -//! [`crate::dream::http::AnthropicDreamBackend`] but reuses the classifier's -//! [`crate::classifier::agent_sdk`] command plumbing. -//! -//! Unlike the realtime classifier, dream feeds a whole session transcript, so -//! the prompt can be large. It therefore uses [`ClaudeBinaryStdinRunner`] -//! (prompt on stdin) to avoid the per-argument size limit (`E2BIG`). -//! -//! Cost note: as with the classifier agent-sdk backend, since 2026-06-15 a -//! headless `claude -p` run draws from the separate Agent SDK monthly credit -//! pool. Haiku keeps each backfill call cheap. - -use crate::classifier::agent_sdk::{ - claude_on_path, run_claude_json, ClaudeBinaryStdinRunner, CommandRunner, -}; -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; -use crate::dream::prompt::build_prompt; -use anyhow::Context; - -/// Default dream model: Haiku — cheap and subscription-friendly. Override with -/// `TJ_DREAM_MODEL` (shared with the API backend's override). -pub const DEFAULT_MODEL: &str = "claude-haiku-4-5"; - -pub struct ClaudeCliDreamBackend { - model: String, - runner: Box<dyn CommandRunner>, -} - -impl ClaudeCliDreamBackend { - /// Build from environment. Returns `None` unless a `claude` binary is on - /// PATH, so the caller can fall back to the API backend. Model comes from - /// `TJ_DREAM_MODEL`, else Haiku. - pub fn from_env() -> Option<Self> { - if !claude_on_path() { - return None; - } - let model = std::env::var("TJ_DREAM_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); - Some(Self { - model, - runner: Box::new(ClaudeBinaryStdinRunner), - }) - } - - /// Test/dev constructor: inject a fake runner and explicit model so the - /// parse path is exercised without a live `claude` login. - pub fn with_runner(model: impl Into<String>, runner: Box<dyn CommandRunner>) -> Self { - Self { - model: model.into(), - runner, - } - } -} - -impl DreamBackend for ClaudeCliDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result<Vec<BackfillEvent>> { - let prompt = build_prompt(input); - let verdict = run_claude_json(self.runner.as_ref(), &self.model, &prompt)?; - // The model returns a JSON array of events, sometimes fenced. - let json_str = verdict - .trim() - .trim_start_matches("```json") - .trim_start_matches("```") - .trim_end_matches("```") - .trim(); - serde_json::from_str(json_str) - .with_context(|| format!("dream JSON parse failed; got: {json_str}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dream::backend::BackfillInput; - use crate::event::EventType; - - /// Fake runner returning a canned `--output-format json` envelope whose - /// `result` is the model's reply text. - struct FakeRunner { - result_text: String, - } - - impl CommandRunner for FakeRunner { - fn run(&self, _model: &str, _prompt: &str) -> anyhow::Result<String> { - Ok(serde_json::json!({ - "type": "result", - "is_error": false, - "result": self.result_text, - }) - .to_string()) - } - } - - fn input() -> BackfillInput { - BackfillInput { - tasks: vec![], - transcript: "user: решили взять Postgres вместо Mongo\nassistant: ок".into(), - } - } - - #[test] - fn parses_event_array_from_envelope() { - let runner = FakeRunner { - result_text: r#"[{"event_type":"decision","task_id":"tj-2","text":"Взяли Postgres вместо Mongo.","timestamp":"2026-06-08T10:00:00Z"}]"# - .into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].event_type, EventType::Decision); - assert_eq!(out[0].task_id, "tj-2"); - assert_eq!(out[0].text, "Взяли Postgres вместо Mongo."); - } - - #[test] - fn tolerates_code_fence_wrapped_array() { - let runner = FakeRunner { - result_text: "```json\n[{\"event_type\":\"finding\",\"task_id\":\"tj-1\",\"text\":\"x\",\"timestamp\":\"2026-06-08T10:00:00Z\"}]\n```" - .into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].event_type, EventType::Finding); - } - - #[test] - fn empty_array_yields_no_events() { - let runner = FakeRunner { - result_text: "[]".into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert!(out.is_empty()); - } -} diff --git a/crates/tj-core/src/dream/backend.rs b/crates/tj-core/src/dream/backend.rs index 69d4ad5..464fa78 100644 --- a/crates/tj-core/src/dream/backend.rs +++ b/crates/tj-core/src/dream/backend.rs @@ -33,19 +33,34 @@ pub struct BackfillTaskContext { pub existing_events: Vec<String>, } +/// What one session's backfill call produced. +#[derive(Debug, Default)] +pub struct BackfillOutput { + pub events: Vec<BackfillEvent>, + /// Transcript chunks whose call failed or whose reply was unusable. + /// Non-zero means the session was only partly mined. + pub failed_chunks: usize, +} + pub trait DreamBackend { /// Return the events the realtime classifier missed for this session. - fn backfill(&self, input: &BackfillInput) -> anyhow::Result<Vec<BackfillEvent>>; + fn backfill(&self, input: &BackfillInput) -> anyhow::Result<BackfillOutput>; } /// Test backend that returns a canned list, ignoring the input. +#[cfg(test)] pub struct MockDreamBackend { pub events: Vec<BackfillEvent>, + pub failed_chunks: usize, } +#[cfg(test)] impl DreamBackend for MockDreamBackend { - fn backfill(&self, _input: &BackfillInput) -> anyhow::Result<Vec<BackfillEvent>> { - Ok(self.events.clone()) + fn backfill(&self, _input: &BackfillInput) -> anyhow::Result<BackfillOutput> { + Ok(BackfillOutput { + events: self.events.clone(), + failed_chunks: self.failed_chunks, + }) } } @@ -62,14 +77,16 @@ mod tests { text: "Chose A over B.".into(), timestamp: "2026-06-08T10:00:00Z".into(), }], + failed_chunks: 0, }; let input = BackfillInput { tasks: vec![], transcript: "x".into(), }; let out = be.backfill(&input).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].task_id, "tj-1"); - assert_eq!(out[0].event_type, EventType::Decision); + assert_eq!(out.events.len(), 1); + assert_eq!(out.events[0].task_id, "tj-1"); + assert_eq!(out.events[0].event_type, EventType::Decision); + assert_eq!(out.failed_chunks, 0); } } diff --git a/crates/tj-core/src/dream/backfill.rs b/crates/tj-core/src/dream/backfill.rs index 952db1a..a249313 100644 --- a/crates/tj-core/src/dream/backfill.rs +++ b/crates/tj-core/src/dream/backfill.rs @@ -32,16 +32,20 @@ pub fn similarity(a: &str, b: &str) -> f64 { } } -/// Drop proposed events that are near-duplicates of `existing` texts. +/// Drop proposed events that are near-duplicates of `existing` texts or of +/// an earlier proposal in the same batch (two chunks restating one event). pub fn dedup_guard(proposed: Vec<BackfillEvent>, existing: &[String]) -> Vec<BackfillEvent> { - proposed - .into_iter() - .filter(|p| { - !existing - .iter() - .any(|e| similarity(&p.text, e) >= DUP_THRESHOLD) - }) - .collect() + let mut kept: Vec<BackfillEvent> = Vec::new(); + for p in proposed { + let dup = existing + .iter() + .chain(kept.iter().map(|k| &k.text)) + .any(|e| similarity(&p.text, e) >= DUP_THRESHOLD); + if !dup { + kept.push(p); + } + } + kept } /// Build a journal Event from a proposed backfill event, stamping dream @@ -90,6 +94,20 @@ mod tests { assert!(kept[0].text.contains("TTL")); } + #[test] + fn drops_near_duplicates_among_proposals() { + // Two chunks of one session restating the same decision. + let proposed = vec![ + ev("We decided to use SQLite instead of Postgres"), + ev("Decided to use SQLite instead of Postgres"), + ev("The cache layer needs a TTL of 60 seconds"), + ]; + let kept = dedup_guard(proposed, &[]); + assert_eq!(kept.len(), 2); + assert!(kept[0].text.starts_with("We decided")); + assert!(kept[1].text.contains("TTL")); + } + #[test] fn identical_text_is_similarity_one() { assert!((similarity("hello world", "hello world") - 1.0).abs() < f64::EPSILON); diff --git a/crates/tj-core/src/dream/http.rs b/crates/tj-core/src/dream/http.rs deleted file mode 100644 index a1af1c2..0000000 --- a/crates/tj-core/src/dream/http.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Anthropic API HTTP client implementing DreamBackend. Mirrors -//! classifier::http but returns a list of missed events. - -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; -use crate::dream::prompt::build_prompt; -use anyhow::{anyhow, Context}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); -/// Stronger than the classifier default — dream sees the whole transcript. -pub const DEFAULT_MODEL: &str = "claude-sonnet-4-6"; -pub const DEFAULT_MAX_TOKENS: u32 = 2048; - -pub struct AnthropicDreamBackend { - pub api_key: String, - pub model: String, - pub base_url: String, - pub max_tokens: u32, - pub timeout: Duration, -} - -impl AnthropicDreamBackend { - pub fn from_env() -> anyhow::Result<Self> { - let api_key = - std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY env var not set")?; - let model = std::env::var("TJ_DREAM_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); - let max_tokens = std::env::var("TJ_DREAM_MAX_TOKENS") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(DEFAULT_MAX_TOKENS); - Ok(Self { - api_key, - model, - base_url: "https://api.anthropic.com".into(), - max_tokens, - timeout: DEFAULT_TIMEOUT, - }) - } -} - -#[derive(Serialize)] -struct MessagesRequest<'a> { - model: &'a str, - max_tokens: u32, - messages: Vec<MessageIn<'a>>, -} -#[derive(Serialize)] -struct MessageIn<'a> { - role: &'a str, - content: &'a str, -} -#[derive(Deserialize)] -struct MessagesResponse { - content: Vec<ContentBlock>, -} -#[derive(Deserialize)] -struct ContentBlock { - #[serde(rename = "type")] - kind: String, - #[serde(default)] - text: String, -} - -impl DreamBackend for AnthropicDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result<Vec<BackfillEvent>> { - let prompt = build_prompt(input); - let body = MessagesRequest { - model: &self.model, - max_tokens: self.max_tokens, - messages: vec![MessageIn { - role: "user", - content: &prompt, - }], - }; - let url = format!("{}/v1/messages", self.base_url); - let resp: MessagesResponse = ureq::post(&url) - .timeout(self.timeout) - .set("x-api-key", &self.api_key) - .set("anthropic-version", "2023-06-01") - .set("content-type", "application/json") - .send_json(serde_json::to_value(&body)?) - .context("Anthropic API request failed")? - .into_json() - .context("decode Anthropic response")?; - - let text = resp - .content - .iter() - .find(|b| b.kind == "text") - .map(|b| b.text.clone()) - .ok_or_else(|| anyhow!("no text content in response"))?; - - let json_str = text - .trim() - .trim_start_matches("```json") - .trim_start_matches("```") - .trim_end_matches("```") - .trim(); - let out: Vec<BackfillEvent> = serde_json::from_str(json_str) - .with_context(|| format!("dream JSON parse failed; got: {json_str}"))?; - Ok(out) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dream::backend::BackfillInput; - - #[test] - fn backend_parses_event_array() { - let mut server = mockito::Server::new(); - let url = server.url(); - let body = serde_json::json!({ - "content": [ - { "type": "text", "text": "[{\"event_type\":\"finding\",\"task_id\":\"tj-2\",\"text\":\"Found the bug.\",\"timestamp\":\"2026-06-08T10:00:00Z\"}]" } - ] - }); - let _m = server - .mock("POST", "/v1/messages") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(body.to_string()) - .create(); - - let be = AnthropicDreamBackend { - api_key: "k".into(), - model: "m".into(), - base_url: url, - max_tokens: 256, - timeout: Duration::from_secs(5), - }; - let input = BackfillInput { - tasks: vec![], - transcript: "t".into(), - }; - let out = be.backfill(&input).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].text, "Found the bug."); - assert_eq!(out[0].task_id, "tj-2"); - } -} diff --git a/crates/tj-core/src/dream/llm_backend.rs b/crates/tj-core/src/dream/llm_backend.rs index 777ab4c..7b6e110 100644 --- a/crates/tj-core/src/dream/llm_backend.rs +++ b/crates/tj-core/src/dream/llm_backend.rs @@ -5,7 +5,7 @@ use anyhow::Context; -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; +use crate::dream::backend::{BackfillEvent, BackfillInput, BackfillOutput, DreamBackend}; use crate::llm::LlmBackend; /// Adapts any [`LlmBackend`] into a [`DreamBackend`]: build the dream prompt, @@ -44,8 +44,8 @@ impl LlmDreamBackend { const TRANSCRIPT_CHAR_BUDGET: usize = 150_000; impl DreamBackend for LlmDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result<Vec<BackfillEvent>> { - let mut out = Vec::new(); + fn backfill(&self, input: &BackfillInput) -> anyhow::Result<BackfillOutput> { + let mut out = BackfillOutput::default(); for chunk in chunk_transcript(&input.transcript, TRANSCRIPT_CHAR_BUDGET) { let chunk_input = BackfillInput { tasks: input.tasks.clone(), @@ -57,17 +57,22 @@ impl DreamBackend for LlmDreamBackend { // (model continued the transcript dialogue) — is skipped, never // aborting the finalize. A genuinely broken backend still surfaces // at the judge step, which has its own (small, always-sized) call. + // Skips are counted so `dream` keeps the session in scope. match self.llm.complete_usage(&prompt, 1024) { Ok((text, usage)) => { self.usage.lock().unwrap().add(usage); match parse_backfill_json(&text) { - Ok(evs) => out.extend(evs), + Ok(evs) => out.events.extend(evs), Err(e) => { + out.failed_chunks += 1; tracing::warn!(error = %e, "dream backfill: skipping unparseable chunk") } } } - Err(e) => tracing::warn!(error = %e, "dream backfill: skipping failed chunk"), + Err(e) => { + out.failed_chunks += 1; + tracing::warn!(error = %e, "dream backfill: skipping failed chunk") + } } } Ok(out) @@ -176,8 +181,11 @@ mod tests { tasks: vec![], transcript: "user: hi\nassistant: hello".into(), }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + // Reported, not silently swallowed, so the caller won't advance the + // dream watermark past this session. + assert_eq!(out.failed_chunks, 1); } #[test] @@ -200,8 +208,9 @@ mod tests { tasks: vec![], transcript: "user: hi\nassistant: hello".into(), }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + assert_eq!(out.failed_chunks, 1); } #[test] @@ -251,8 +260,9 @@ mod tests { tasks: vec![], transcript, }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + assert_eq!(out.failed_chunks, 0); } #[test] @@ -275,7 +285,7 @@ mod tests { tasks: vec![], transcript: "x".into(), }; - let evs = b.backfill(&input).unwrap(); + let evs = b.backfill(&input).unwrap().events; assert_eq!(evs.len(), 1); assert_eq!(evs[0].text, "found it"); assert_eq!(b.backend_name(), "fake"); diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index 3924953..ba9e292 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -4,10 +4,8 @@ //! significant typed events the realtime classifier missed. Additive — //! the JSONL source of truth is never mutated. -pub mod agent_sdk; pub mod backend; pub mod backfill; -pub mod http; pub mod llm_backend; pub mod prompt; pub mod scope; @@ -25,6 +23,12 @@ pub struct DreamOptions { pub struct DreamReport { pub sessions_processed: usize, pub events_backfilled: usize, + /// Proposals dropped because their `task_id` was not one of the + /// session's candidate tasks (the model invented or misspelled it). + pub events_dropped_unknown_task: usize, + /// Sessions with at least one failed transcript chunk — only partly + /// mined, so the watermark must not move past them. + pub failed_sessions: Vec<String>, } /// Run one dream Pass A over the given sessions, using the supplied @@ -39,17 +43,32 @@ pub fn run_dream( run_id: &str, ) -> anyhow::Result<DreamReport> { let mut report = DreamReport::default(); + // Texts appended earlier in this run: a later session of the same task + // must not re-add them (its `existing_events` predate the run). + let mut appended: Vec<String> = Vec::new(); for (session_id, input) in sessions { report.sessions_processed += 1; if opts.dry_run { continue; } - let proposed = backend.backfill(&input)?; + let out = backend.backfill(&input)?; + if out.failed_chunks > 0 { + report.failed_sessions.push(session_id.clone()); + } + let mut proposed = out.events; + + // Only the session's candidate tasks may receive events; anything + // else would be an orphan event for a nonexistent task. + let before = proposed.len(); + proposed.retain(|p| input.tasks.iter().any(|t| t.task_id == p.task_id)); + report.events_dropped_unknown_task += before - proposed.len(); + // Flatten existing texts across candidate tasks for the guard. let existing: Vec<String> = input .tasks .iter() .flat_map(|t| t.existing_events.clone()) + .chain(appended.iter().cloned()) .collect(); let kept = crate::dream::backfill::dedup_guard(proposed, &existing); let mut writer = crate::storage::JsonlWriter::open(events_path)?; @@ -58,6 +77,7 @@ pub fn run_dream( writer.append(&e)?; crate::db::upsert_task_from_event(conn, &e, &opts.project_hash)?; crate::db::index_event(conn, &e)?; + appended.push(b.text.clone()); report.events_backfilled += 1; } writer.flush_durable()?; @@ -117,6 +137,7 @@ mod tests { timestamp: "2026-06-08T10:01:00Z".into(), }, ], + failed_chunks: 0, }; let opts = DreamOptions { project_hash: "ph".into(), @@ -140,12 +161,122 @@ mod tests { assert!(!body.contains("\"text\":\"Already known fact.\",\"refs\"")); } + #[test] + fn run_dream_drops_events_for_tasks_outside_the_candidates() { + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![ + BackfillEvent { + event_type: EventType::Finding, + task_id: "tj-ghost".into(), // not a candidate → orphan + text: "Invented task id.".into(), + timestamp: "2026-06-08T10:00:00Z".into(), + }, + BackfillEvent { + event_type: EventType::Finding, + task_id: "tj-1".into(), + text: "A real finding.".into(), + timestamp: "2026-06-08T10:01:00Z".into(), + }, + ], + failed_chunks: 0, + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input()], + "run-1", + ) + .unwrap(); + + assert_eq!(report.events_backfilled, 1); + assert_eq!(report.events_dropped_unknown_task, 1); + let body = std::fs::read_to_string(&events_path).unwrap(); + assert!(!body.contains("tj-ghost")); + } + + #[test] + fn run_dream_does_not_repeat_an_event_across_sessions() { + // Two sessions of one task both surface the same decision: the + // second must not append it again. + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![BackfillEvent { + event_type: EventType::Decision, + task_id: "tj-1".into(), + text: "Chose SQLite over Postgres.".into(), + timestamp: "2026-06-08T10:00:00Z".into(), + }], + failed_chunks: 0, + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + let mut second = task_input(); + second.0 = "sess-2".into(); + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input(), second], + "run-1", + ) + .unwrap(); + + assert_eq!(report.sessions_processed, 2); + assert_eq!(report.events_backfilled, 1); + } + + #[test] + fn run_dream_reports_sessions_with_failed_chunks() { + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![], + failed_chunks: 1, + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input()], + "run-1", + ) + .unwrap(); + + assert_eq!(report.failed_sessions, vec!["sess-1".to_string()]); + } + #[test] fn dry_run_writes_nothing_and_skips_backend() { let d = TempDir::new().unwrap(); let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); let events_path = d.path().join("events.jsonl"); - let backend = MockDreamBackend { events: vec![] }; + let backend = MockDreamBackend { + events: vec![], + failed_chunks: 0, + }; let opts = DreamOptions { project_hash: "ph".into(), dry_run: true, diff --git a/crates/tj-core/src/dream/scope.rs b/crates/tj-core/src/dream/scope.rs index b758116..adf56fb 100644 --- a/crates/tj-core/src/dream/scope.rs +++ b/crates/tj-core/src/dream/scope.rs @@ -31,6 +31,25 @@ pub fn in_scope( out } +/// Where an unscoped run may move the watermark, given every in-scope +/// session as `(mtime, mined_cleanly)`: the newest clean mtime older than +/// every failed or skipped session, so the next run (which keeps only +/// mtimes strictly after the watermark) still sees those. `None` = leave +/// the watermark where it is. +pub fn next_watermark(sessions: &[(SystemTime, bool)]) -> Option<SystemTime> { + let first_bad = sessions + .iter() + .filter(|(_, clean)| !clean) + .map(|(t, _)| *t) + .min(); + + sessions + .iter() + .filter(|(t, clean)| *clean && first_bad.is_none_or(|bad| *t < bad)) + .map(|(t, _)| *t) + .max() +} + #[cfg(test)] mod tests { use super::*; @@ -64,6 +83,27 @@ mod tests { ); } + #[test] + fn watermark_advances_to_newest_when_all_clean() { + let s = [(at(300), true), (at(100), true), (at(200), true)]; + assert_eq!(next_watermark(&s), Some(at(300))); + } + + #[test] + fn watermark_never_jumps_over_a_failed_session() { + // 200 failed → stop at 100, so 200 and 300 are mined again. + let s = [(at(100), true), (at(200), false), (at(300), true)]; + assert_eq!(next_watermark(&s), Some(at(100))); + // Oldest failed → nothing to advance to. + let s = [(at(100), false), (at(200), true)]; + assert_eq!(next_watermark(&s), None); + // A clean session sharing the failed one's mtime can't be the + // watermark either (`in_scope` keeps only mtime > watermark). + let s = [(at(100), true), (at(200), true), (at(200), false)]; + assert_eq!(next_watermark(&s), Some(at(100))); + assert_eq!(next_watermark(&[]), None); + } + #[test] fn none_since_keeps_all_limit_caps() { let s = vec![ diff --git a/crates/tj-core/src/event.rs b/crates/tj-core/src/event.rs index de603a1..6dae98c 100644 --- a/crates/tj-core/src/event.rs +++ b/crates/tj-core/src/event.rs @@ -17,6 +17,11 @@ pub enum EventType { Close, Redirect, Rename, + /// Changes task metadata after creation: `meta.goal` replaces the goal, + /// `meta.external_add` (array of strings) adds external references. + /// Not reasoning — kept out of packs, search and recall. Written by the + /// `goal` / `external` commands, never accepted as a user event type. + Amend, } impl EventType { @@ -34,6 +39,7 @@ impl EventType { Self::Close, Self::Redirect, Self::Rename, + Self::Amend, ]; } diff --git a/crates/tj-core/src/fts.rs b/crates/tj-core/src/fts.rs index 5e34640..d96e639 100644 --- a/crates/tj-core/src/fts.rs +++ b/crates/tj-core/src/fts.rs @@ -1,30 +1,24 @@ //! FTS5 query sanitization. //! -//! SQLite FTS5 treats `-`, `:`, `*`, `(`, `)`, `"`, and `/` as syntax. -//! A raw query like `OPS-306` parses as column-prefix `OPS-` followed by -//! reference `306`, raising `no such column: 306` at runtime. +//! SQLite FTS5 treats almost any punctuation (`-`, `:`, `*`, `.`, `,`, `'`, +//! `+`, `^`, `{`, `?`, …) and the bare words `AND`/`OR`/`NOT`/`NEAR` as +//! syntax. A raw query like `OPS-306` parses as column-prefix `OPS-` +//! followed by reference `306`, raising `no such column: 306` at runtime. //! -//! [`sanitize_query`] wraps queries containing any metacharacter in -//! double quotes, turning them into FTS5 phrase queries that match the -//! sequence of tokens as the unicode61 tokenizer split them. Internal -//! `"` characters are doubled per FTS5 escape rules. - -/// Wrap a query in FTS5 phrase quotes if it contains any character the -/// FTS5 parser treats as syntax. Strings that only contain word -/// characters / whitespace pass through untouched so multi-term -/// queries keep their default AND semantics. +//! [`sanitize_query`] splits the query on whitespace and wraps every token +//! in double quotes, so each one is an FTS5 string matched as the unicode61 +//! tokenizer splits it, and the tokens are implicitly AND-ed. Internal `"` +//! characters are doubled per FTS5 escape rules. + +/// Quote every whitespace-separated token as its own FTS5 string. Any +/// punctuation inside a token is then literal, operator words lose their +/// meaning, and a multi-token query keeps its default AND semantics. pub fn sanitize_query(query: &str) -> String { - if query.is_empty() { - return String::new(); - } - let needs_quote = query - .chars() - .any(|c| matches!(c, '-' | '"' | '*' | ':' | '(' | ')' | '/')); - if !needs_quote { - return query.to_string(); - } - let escaped = query.replace('"', "\"\""); - format!("\"{escaped}\"") + query + .split_whitespace() + .map(|tok| format!("\"{}\"", tok.replace('"', "\"\""))) + .collect::<Vec<_>>() + .join(" ") } /// Build a `LIKE` pattern equivalent to a free-text search — used as a @@ -40,19 +34,71 @@ pub fn like_pattern(query: &str) -> String { mod tests { use super::{like_pattern, sanitize_query}; + /// Count rows of a throwaway FTS5 table matching `sanitize_query(query)`. + /// `Err` means FTS5 rejected the sanitized query as syntax. + fn fts_hits(rows: &[&str], query: &str) -> rusqlite::Result<i64> { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE VIRTUAL TABLE t USING fts5(text);") + .unwrap(); + for row in rows { + conn.execute("INSERT INTO t(text) VALUES (?1)", [row]) + .unwrap(); + } + + conn.query_row( + "SELECT COUNT(*) FROM t WHERE t MATCH ?1", + [sanitize_query(query)], + |r| r.get(0), + ) + } + + #[test] + fn plain_word_is_quoted_as_one_token() { + assert_eq!(sanitize_query("hello"), "\"hello\""); + } + + #[test] + fn multi_word_quotes_each_token_for_default_and_search() { + assert_eq!(sanitize_query("bulk repack"), "\"bulk\" \"repack\""); + } + #[test] - fn plain_word_passes_through() { - assert_eq!(sanitize_query("hello"), "hello"); + fn cyrillic_tokens_are_quoted() { + assert_eq!(sanitize_query("слим модели"), "\"слим\" \"модели\""); } #[test] - fn multi_word_passes_through_for_default_and_search() { - assert_eq!(sanitize_query("bulk repack"), "bulk repack"); + fn punctuation_and_operator_words_never_raise_fts5_syntax_errors() { + let rows = ["release v1.2, don't c++ AND or NOT near"]; + for q in [ + "v1.2", + "a, b", + "don't", + "c++", + "^start", + "{x}", + "why?", + "AND", + "OR", + "NOT", + "NEAR", + "this AND", + "NEAR(a b)", + "a + b", + ] { + assert!( + fts_hits(&rows, q).is_ok(), + "FTS5 rejected sanitized query {q:?}: {:?}", + fts_hits(&rows, q) + ); + } } #[test] - fn cyrillic_passes_through() { - assert_eq!(sanitize_query("слим модели"), "слим модели"); + fn multi_word_query_with_hyphen_is_and_of_tokens_not_a_phrase() { + let rows = ["OPS-306 needs a quick fix"]; + assert_eq!(fts_hits(&rows, "fix OPS-306").unwrap(), 1); + assert_eq!(fts_hits(&rows, "fix OPS-307").unwrap(), 0); } #[test] @@ -82,12 +128,13 @@ mod tests { #[test] fn embedded_quote_is_doubled() { - assert_eq!(sanitize_query("say \"hi\""), "\"say \"\"hi\"\"\""); + assert_eq!(sanitize_query("say \"hi\""), "\"say\" \"\"\"hi\"\"\""); } #[test] fn empty_query_stays_empty() { assert_eq!(sanitize_query(""), ""); + assert_eq!(sanitize_query(" "), ""); } #[test] diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index df98c4a..e66599a 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -19,6 +19,16 @@ pub fn new_task_id() -> String { ) } +/// One lock for every tj-core test that sets env vars or asserts on +/// something read from them: std env is process-global, so tests running in +/// parallel see each other's values. Poisoning is ignored so one failed test +/// does not fail every other env test after it. +#[cfg(test)] +pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + #[cfg(test)] mod task_id_tests { use super::new_task_id; @@ -68,6 +78,7 @@ pub mod recall; pub mod reminder; pub mod session; pub mod session_id; +pub mod session_state; pub mod storage; pub mod title; diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs index c6f49a6..498bef9 100644 --- a/crates/tj-core/src/llm.rs +++ b/crates/tj-core/src/llm.rs @@ -217,7 +217,10 @@ impl LlmBackend for CodexCliBackend { .args(["--sandbox", "read-only"]) .args(["--color", "never"]) .arg("-o") - .arg(&out_path); + .arg(&out_path) + // `codex exec` runs the user's ~/.codex hooks; the marker makes our + // own `ingest-hook` no-op inside this child instead of recursing. + .env(crate::classifier::agent_sdk::IN_CLASSIFIER_ENV, "1"); if let Some(model) = &self.model { cmd.args(["-m", model]); } @@ -225,22 +228,27 @@ impl LlmBackend for CodexCliBackend { // transcript-sized prompt away from the argv size limit. cmd.arg("-"); - let mut child = cmd - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn `codex` (is the Codex CLI installed and on PATH?)")?; - child - .stdin - .take() - .context("codex stdin was not captured")? - .write_all(prompt.as_bytes()) - .context("failed to write prompt to codex stdin")?; - - let output = crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout())?; + // Run inside a closure so the output file is removed on every exit + // path — a failed stdin write and a timeout included. + let mut run = || -> anyhow::Result<std::process::Output> { + let mut child = cmd + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("failed to spawn `codex` (is the Codex CLI installed and on PATH?)")?; + child + .stdin + .take() + .context("codex stdin was not captured")? + .write_all(prompt.as_bytes()) + .context("failed to write prompt to codex stdin")?; + crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout(), "codex exec") + }; + let output = run(); let answer = std::fs::read_to_string(&out_path).unwrap_or_default(); let _ = std::fs::remove_file(&out_path); + let output = output?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -491,15 +499,12 @@ mod tests { } } - // Serialise env-touching tests (process-global env). - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - /// `codex` used to resolve to the OpenAI API backend, so an OPENAI_API_KEY /// in the environment must no longer pull the wrong backend in. Whether the /// Codex CLI is installed decides between `Some(codex-exec)` and `None`. #[test] fn codex_backend_is_the_codex_cli_not_the_openai_api() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _key = EnvGuard::set("OPENAI_API_KEY", "sk-should-not-be-used"); match backend_from_env(Some("codex")).unwrap() { Some(b) => assert_eq!(b.name(), "codex-exec"), @@ -510,22 +515,102 @@ mod tests { } } + /// A stand-in `codex` that writes `TJ_IN_CLASSIFIER` into its `-o` file, + /// then sleeps `FAKE_CODEX_SLEEP` seconds when that is set. + #[cfg(unix)] + fn fake_codex_dir() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + let script = dir.path().join("codex"); + std::fs::write( + &script, + "#!/bin/sh\nout=\"\"\nwhile [ $# -gt 0 ]; do\n if [ \"$1\" = \"-o\" ]; then out=\"$2\"; shift; fi\n shift\ndone\ncat > /dev/null\nprintf 'in_classifier=%s' \"$TJ_IN_CLASSIFIER\" > \"$out\"\nif [ -n \"$FAKE_CODEX_SLEEP\" ]; then exec sleep \"$FAKE_CODEX_SLEEP\"; fi\n", + ) + .unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + dir + } + + #[cfg(unix)] + fn path_with(dir: &std::path::Path) -> EnvGuard { + let old = std::env::var("PATH").unwrap_or_default(); + EnvGuard::set("PATH", &format!("{}:{old}", dir.display())) + } + + /// `codex exec` runs the user's ~/.codex hooks; without the marker a + /// `dream`/`complete` run re-enters `ingest-hook` from inside itself. + #[cfg(unix)] + #[test] + fn codex_child_runs_with_the_recursion_marker() { + let _l = crate::test_env_lock(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _marker = EnvGuard::unset("TJ_IN_CLASSIFIER"); + let _sleep = EnvGuard::unset("FAKE_CODEX_SLEEP"); + + let (answer, _) = CodexCliBackend { model: None } + .complete_usage("classify me", 64) + .unwrap(); + + assert_eq!(answer, "in_classifier=1"); + } + + #[cfg(unix)] + #[test] + fn codex_output_file_is_removed_when_the_call_times_out() { + let _l = crate::test_env_lock(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); + let _sleep = EnvGuard::set("FAKE_CODEX_SLEEP", "5"); + + let res = CodexCliBackend { model: None }.complete_usage("classify me", 64); + + assert!(res.is_err(), "a wedged codex must time out"); + let prefix = format!("task-journal-codex-{}-", std::process::id()); + let left: Vec<_> = std::fs::read_dir(std::env::temp_dir()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix)) + .collect(); + assert!(left.is_empty(), "codex temp file leaked: {left:?}"); + } + + /// The timeout error names the binary that hung, not always `claude -p`. + #[cfg(unix)] + #[test] + fn codex_timeout_error_names_codex() { + let _l = crate::test_env_lock(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); + let _sleep = EnvGuard::set("FAKE_CODEX_SLEEP", "5"); + + let err = CodexCliBackend { model: None } + .complete_usage("classify me", 64) + .unwrap_err() + .to_string(); + + assert!(err.contains("`codex exec` timed out"), "{err}"); + } + #[test] fn unknown_backend_errors() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); assert!(backend_from_env(Some("nonsense")).is_err()); } #[test] fn anthropic_unavailable_without_key_is_none() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _g = EnvGuard::unset("ANTHROPIC_API_KEY"); assert!(backend_from_env(Some("anthropic")).unwrap().is_none()); } #[test] fn anthropic_with_key_resolves() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _g = EnvGuard::set("ANTHROPIC_API_KEY", "k"); let b = backend_from_env(Some("anthropic")).unwrap().unwrap(); assert_eq!(b.name(), "anthropic"); @@ -533,7 +618,7 @@ mod tests { #[test] fn ollama_always_resolves_no_key() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let b = backend_from_env(Some("ollama")).unwrap().unwrap(); assert_eq!(b.name(), "ollama"); } diff --git a/crates/tj-core/src/memory.rs b/crates/tj-core/src/memory.rs index 395831f..e4caa10 100644 --- a/crates/tj-core/src/memory.rs +++ b/crates/tj-core/src/memory.rs @@ -85,10 +85,12 @@ pub fn sync_from_project( .join(","); let sql = format!( "SELECT e.event_id, e.task_id, f.type, e.tier, f.text, e.model, e.dim, e.vec, e.created_at, - CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END, + COALESCE(ei.bookkeeping, 0) OR ei.corrected_by IS NOT NULL OR ei.status = 'suggested' FROM embeddings e JOIN search_fts f ON f.event_id = e.event_id LEFT JOIN decisions d ON d.decision_id = e.event_id + LEFT JOIN events_index ei ON ei.event_id = e.event_id WHERE f.type IN ({placeholders})" ); let mut stmt = project.prepare(&sql)?; @@ -104,12 +106,27 @@ pub fn sync_from_project( r.get::<_, Vec<u8>>(7)?, // vec r.get::<_, String>(8)?, // created_at r.get::<_, i64>(9)?, // superseded + r.get::<_, bool>(10)?, // bookkeeping or corrected )) })?; let mut n = 0usize; for row in rows { - let (event_id, task_id, ty, tier, text, model, dim, vec, created_at, superseded) = row?; + let (event_id, task_id, ty, tier, text, model, dim, vec, created_at, superseded, retired) = + row?; + // Compaction markers / model switches are not reasoning to recall, and a + // corrected event is no longer true; also drop any copy synced earlier. + if retired { + global.execute( + "DELETE FROM global_memory WHERE event_id = ?1", + rusqlite::params![event_id], + )?; + global.execute( + "DELETE FROM global_fts WHERE event_id = ?1", + rusqlite::params![event_id], + )?; + continue; + } global.execute( "INSERT OR REPLACE INTO global_memory (event_id, project_hash, task_id, type, tier, text, model, dim, vec, created_at, superseded) @@ -346,6 +363,94 @@ mod tests { .is_empty()); } + #[test] + fn sync_leaves_out_and_drops_bookkeeping_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + // An old marker, recognised by its text alone. + let marker = finding( + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events should be treated as a single reasoning unit.", + ); + let mut switch = finding("Model switched (user): opus → sonnet"); + switch.event_type = crate::event::EventType::Constraint; + switch.meta = serde_json::json!({ "kind": "model_switch" }); + for e in [ + &marker, + &switch, + &finding("chose the idempotent payment ledger"), + ] { + crate::db::index_event(&proj, e).unwrap(); + } + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + // A marker an older version already copied into the global index. + global + .execute( + "INSERT INTO global_memory + (event_id, project_hash, task_id, type, text, model, dim, vec, created_at) + VALUES (?1, 'ph', 'tj-x', 'decision', ?2, 't', 0, x'', '')", + rusqlite::params![marker.event_id, marker.text], + ) + .unwrap(); + global + .execute( + "INSERT INTO global_fts(event_id, text) VALUES (?1, ?2)", + rusqlite::params![marker.event_id, marker.text], + ) + .unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 1); + assert_eq!(count(&global).unwrap(), 1); + assert!(keyword_search(&global, "compacted reasoning switched", 5) + .unwrap() + .is_empty()); + } + + #[test] + fn sync_leaves_out_and_drops_corrected_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + let wrong = finding("chose the kafka audit pipeline"); + crate::db::index_event(&proj, &wrong).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 1); + + // Corrected after the first sync: the next sync must drop the copy. + let mut corr = finding("that kafka choice was a misread of the brief"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + crate::db::index_event(&proj, &corr).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 0); + assert_eq!(count(&global).unwrap(), 0); + assert!(keyword_search(&global, "kafka audit pipeline", 5) + .unwrap() + .is_empty()); + } + + #[test] + fn sync_leaves_out_suggested_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + let mut guess = finding("the classifier guessed a redis cache"); + guess.status = crate::event::EventStatus::Suggested; + crate::db::index_event(&proj, &guess).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 0); + assert_eq!(count(&global).unwrap(), 0); + } + #[test] fn preferences_store_dedup_and_list_in_order() { let d = tempfile::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index b4dab00..4759c76 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -47,7 +47,8 @@ fn render_recent_events(conn: &Connection, task_id: &str, limit: usize) -> anyho let mut stmt = conn.prepare( "SELECT ei.timestamp, ei.type, ei.status, sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id=?1 ORDER BY ei.timestamp DESC LIMIT ?2", + WHERE ei.task_id=?1 AND ei.corrected_by IS NULL + ORDER BY ei.timestamp DESC LIMIT ?2", )?; let rows = stmt.query_map(rusqlite::params![task_id, limit as i64], |r| { let ts: String = r.get(0)?; @@ -80,7 +81,10 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result<String> { // v0.10.3: newest evidence first (ULID DESC). Matches the // decision-ordering fix so truncation prefers older rows. let mut stmt = conn.prepare( - "SELECT text, strength FROM evidence WHERE task_id=?1 ORDER BY evidence_id DESC", + "SELECT e.text, e.strength FROM evidence e + JOIN events_index ei ON ei.event_id = e.evidence_id + WHERE e.task_id=?1 AND ei.corrected_by IS NULL + ORDER BY e.evidence_id DESC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { let t: String = r.get(0)?; @@ -88,8 +92,13 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result<String> { Ok((t, s)) })?; let mut count = 0; + let mut seen: HashSet<String> = HashSet::new(); for row in rows { let (t, s) = row?; + // Same noise / repeat filter as Active decisions and Rejected. + if is_noise(&t) || !seen.insert(t.trim().to_string()) { + continue; + } out.push_str(&format!("- {t} ({s})\n")); count += 1; } @@ -100,30 +109,95 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result<String> { Ok(out) } -fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { - let mut out = String::from("## Rejected\n"); +/// A decision or rejection as the pack lists it. +pub struct Choice { + pub text: String, + /// A decision's `meta.alternatives` JSON; always `None` for a rejection. + pub alternatives: Option<String>, + /// A classifier's guess (`status = suggested`). + pub unconfirmed: bool, +} + +impl Choice { + /// A guess must not read like a choice the agent made: it stays listed + /// but carries this trailing marker. + pub fn marker(&self) -> &'static str { + if self.unconfirmed { + " _(unconfirmed)_" + } else { + "" + } + } +} + +/// Keep the first of each text, minus machine noise (compaction markers), so +/// a section reads as crisp choices, not repeated essays. +fn dedupe(choices: impl IntoIterator<Item = Choice>) -> Vec<Choice> { + let mut seen: HashSet<String> = HashSet::new(); + choices + .into_iter() + .filter(|c| !is_noise(&c.text) && seen.insert(c.text.trim().to_string())) + .collect() +} + +/// The pack's Active decisions, newest first: neither superseded, corrected +/// nor bookkeeping. export-pr lists the same ones. +pub fn active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result<Vec<Choice>> { + // v0.10.3: newest decision first. `decision_id` is a ULID so DESC + // gives reverse-chronological order. The summary/final-decision + // event the agent records just before close is now the FIRST line + // of this section, surviving end-of-pack truncation. + let mut stmt = conn.prepare( + "SELECT d.text, d.alternatives, ei.status FROM decisions d + JOIN events_index ei ON ei.event_id = d.decision_id + WHERE d.task_id=?1 AND d.status='active' AND ei.corrected_by IS NULL + AND ei.bookkeeping = 0 + ORDER BY d.decision_id DESC", + )?; + let rows = stmt + .query_map(rusqlite::params![task_id], |r| { + Ok(Choice { + text: r.get(0)?, + alternatives: r.get(1)?, + unconfirmed: r.get::<_, String>(2)? == "suggested", + }) + })? + .collect::<Result<Vec<_>, _>>()?; + Ok(dedupe(rows)) +} + +/// The pack's Rejected list, newest first, without corrected rejections. +/// export-pr lists the same ones. +pub fn rejections(conn: &Connection, task_id: &str) -> anyhow::Result<Vec<Choice>> { // v0.10.3: newest-first so end-of-pack truncation drops the // OLDEST rejections, not the latest decision the agent recorded. let mut id_stmt = conn.prepare( - "SELECT event_id FROM events_index - WHERE task_id=?1 AND type='rejection' + "SELECT event_id, status FROM events_index + WHERE task_id=?1 AND type='rejection' AND corrected_by IS NULL ORDER BY timestamp DESC", )?; let mut text_stmt = conn.prepare("SELECT text FROM search_fts WHERE event_id=?1 LIMIT 1")?; - let event_ids: Vec<String> = id_stmt - .query_map(rusqlite::params![task_id], |r| r.get::<_, String>(0))? + let event_ids: Vec<(String, String)> = id_stmt + .query_map(rusqlite::params![task_id], |r| Ok((r.get(0)?, r.get(1)?)))? .collect::<Result<_, _>>()?; - let mut count = 0; - let mut seen: HashSet<String> = HashSet::new(); - for eid in event_ids { - let text: String = text_stmt.query_row(rusqlite::params![eid], |r| r.get(0))?; - if is_noise(&text) || !seen.insert(text.trim().to_string()) { - continue; - } - out.push_str(&format!("- {text}\n")); - count += 1; + let mut rows = Vec::new(); + for (eid, status) in event_ids { + rows.push(Choice { + text: text_stmt.query_row(rusqlite::params![eid], |r| r.get(0))?, + alternatives: None, + unconfirmed: status == "suggested", + }); } - if count == 0 { + Ok(dedupe(rows)) +} + +fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { + let mut out = String::from("## Rejected\n"); + let rejected = rejections(conn, task_id)?; + for r in &rejected { + out.push_str(&format!("- {}{}\n", r.text, r.marker())); + } + if rejected.is_empty() { out.push_str("- (none)\n"); } out.push('\n'); @@ -132,35 +206,17 @@ fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { fn render_active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result<String> { let mut out = String::from("## Active decisions\n"); - // v0.10.3: newest decision first. `decision_id` is a ULID so DESC - // gives reverse-chronological order. The summary/final-decision - // event the agent records just before close is now the FIRST line - // of this section, surviving end-of-pack truncation. - let mut stmt = conn.prepare( - "SELECT text, alternatives FROM decisions WHERE task_id=?1 AND status='active' ORDER BY decision_id DESC", - )?; - let rows = stmt.query_map(rusqlite::params![task_id], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?)) - })?; - let mut count = 0; - let mut seen: HashSet<String> = HashSet::new(); - for row in rows { - let (text, alternatives) = row?; - // Skip machine noise (compaction markers) and exact duplicates so the - // section reads as crisp decisions, not repeated essays. - if is_noise(&text) || !seen.insert(text.trim().to_string()) { - continue; - } - out.push_str(&format!("- {text}\n")); + let decisions = active_decisions(conn, task_id)?; + for d in &decisions { + out.push_str(&format!("- {}{}\n", d.text, d.marker())); // v0.12.0: structured alternatives render under the decision so the // pack shows "considered A/B/C, chose X" without reconstructing it // from the hypothesis+rejection chain. - if let Some(block) = render_alternatives(alternatives.as_deref()) { + if let Some(block) = render_alternatives(d.alternatives.as_deref()) { out.push_str(&block); } - count += 1; } - if count == 0 { + if decisions.is_empty() { out.push_str("- (none)\n"); } out.push('\n'); @@ -176,12 +232,13 @@ fn render_alternatives(raw: Option<&str>) -> Option<String> { let raw = raw?; let parsed: serde_json::Value = serde_json::from_str(raw).ok()?; let arr = parsed.as_array()?; - if arr.is_empty() { - return None; - } let mut block = String::from(" - considered:\n"); + let header_len = block.len(); for entry in arr { - let option = entry.get("option").and_then(|v| v.as_str())?; + // Skip only the entry without an option, not the whole block. + let Some(option) = entry.get("option").and_then(|v| v.as_str()) else { + continue; + }; let chosen = entry .get("chosen") .and_then(|v| v.as_bool()) @@ -193,7 +250,8 @@ fn render_alternatives(raw: Option<&str>) -> Option<String> { None => block.push_str(&format!(" - {marker} {option}\n")), } } - Some(block) + + (block.len() > header_len).then_some(block) } fn render_lifecycle(conn: &Connection, task_id: &str) -> anyhow::Result<String> { @@ -247,6 +305,29 @@ fn render_subtasks(conn: &Connection, task_id: &str) -> anyhow::Result<Option<St Ok(Some(s)) } +/// Compact-mode Completeness: the heading plus ONE line with the score and +/// every gap, capped so it always fits the 2 KB budget beside the reasoning. +fn render_gap_line(report: &crate::completeness::CompletenessReport) -> Option<String> { + const GAP_LINE_MAX: usize = 400; + + if report.gaps.is_empty() { + return None; + } + let details = report + .gaps + .iter() + .map(|g| g.detail.as_str()) + .collect::<Vec<_>>() + .join("; "); + let mut line = format!("- ⚠ honesty score: {}/100 — {details}", report.score()); + truncate_to_budget(&mut line, GAP_LINE_MAX, "…"); + + Some(format!( + "\n## Completeness ({})\n{line}\n", + report.gaps.len() + )) +} + fn truncate_to_budget(text: &mut String, budget: usize, marker: &str) { if text.len() <= budget { return; @@ -265,32 +346,7 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res PackMode::Compact => "compact", PackMode::Full => "full", }; - - // Read-through cache: if we have a stored pack with the same mode, return it. - let cached: Option<(String, String, i64)> = conn - .query_row( - "SELECT text, generated_at, source_event_count FROM task_pack_cache - WHERE task_id=?1 AND mode=?2", - rusqlite::params![task_id, mode_str], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - ) - .ok(); - if let Some((cached_text, cached_at, cached_count)) = cached { - // Detect truncation by re-checking for the marker. - let was_truncated = cached_text.contains("_(truncated to fit pack budget)_"); - return Ok(TaskPack { - task_id: task_id.to_string(), - mode, - schema_version: crate::SCHEMA_VERSION.into(), - text: cached_text, - metadata: PackMetadata { - generated_at: cached_at, - source_event_count: cached_count as usize, - cache_hit: true, - truncated: was_truncated, - }, - }); - } + let generated_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); let (title, status, goal, outcome, outcome_tag, external): ( String, @@ -316,12 +372,6 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res ) .with_context(|| format!("task not found: {task_id}"))?; - let event_count: usize = conn.query_row( - "SELECT COUNT(*) FROM events_index WHERE task_id=?1", - rusqlite::params![task_id], - |r| r.get::<_, i64>(0).map(|n| n as usize), - )?; - let mut text = format!("# {title} [status: {status}]\n\n"); // v0.4.0 task-as-goal block. Goal renders even when empty so the @@ -412,19 +462,36 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res } text.push('\n'); - if matches!(mode, PackMode::Full) { - text.push_str(&render_lifecycle(conn, task_id)?); - } - text.push_str(&render_active_decisions(conn, task_id)?); - if matches!(mode, PackMode::Full) { - text.push_str(&render_rejected(conn, task_id)?); - text.push_str(&render_evidence(conn, task_id)?); - } - let recent_limit = match mode { - PackMode::Compact => 3, - PackMode::Full => 10, + // Read-through cache of the body. It changes only with this task's own + // events (or a child's), and index_event clears it then. The header above + // (live status of linked tasks) and the gaps below (pending queue, cwd git + // state) change without such an event, so they are rendered every call. + let cached: Option<(String, i64)> = conn + .query_row( + "SELECT text, source_event_count FROM task_pack_cache WHERE task_id=?1 AND mode=?2", + rusqlite::params![task_id, mode_str], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .ok(); + let cache_hit = cached.is_some(); + let (body, event_count) = match cached { + Some((body, count)) => (body, count as usize), + None => { + let body = render_body(conn, task_id, mode)?; + let event_count: usize = conn.query_row( + "SELECT COUNT(*) FROM events_index WHERE task_id=?1", + rusqlite::params![task_id], + |r| r.get::<_, i64>(0).map(|n| n as usize), + )?; + conn.execute( + "INSERT OR REPLACE INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![task_id, mode_str, body, generated_at, event_count as i64], + )?; + (body, event_count) + } }; - text.push_str(&render_recent_events(conn, task_id, recent_limit)?); + text.push_str(&body); let mut report = crate::completeness::assess(conn, task_id, crate::completeness::pending_count())?; @@ -433,16 +500,13 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res report .gaps .extend(crate::completeness::artifact_gaps_for_cwd(&arts)); - if let Some(section) = crate::completeness::render_section(&report) { - text.push_str(§ion); - } - - // One-level roll-up of direct children (parents only). Appended before - // truncation so it shares the pack budget. Task 5 busts the parent cache - // when a child changes, so the next assemble regenerates fresh. - if let Some(subtasks) = render_subtasks(conn, task_id)? { - text.push_str(&subtasks); + // The gaps tell the next agent what is missing, so they go last with + // their room reserved: truncation cuts the reasoning above, never them. + let gaps = match mode { + PackMode::Full => crate::completeness::render_section(&report), + PackMode::Compact => render_gap_line(&report), } + .unwrap_or_default(); // Token-budget truncation: cap pack size so it always fits an LLM context window. // v0.10.3: full bumped 10K → 24K → 32K. Real tasks accumulate 50-100 events @@ -456,19 +520,11 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res PackMode::Full => FULL_BUDGET, PackMode::Compact => COMPACT_BUDGET, }; - let truncated = text.len() > budget; + let truncated = text.len() + gaps.len() > budget; if truncated { - truncate_to_budget(&mut text, budget, TRUNC_MARKER); + truncate_to_budget(&mut text, budget.saturating_sub(gaps.len()), TRUNC_MARKER); } - - let generated_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); - - // Write-through cache. - conn.execute( - "INSERT OR REPLACE INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) - VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![task_id, mode_str, text, generated_at, event_count as i64], - )?; + text.push_str(&gaps); Ok(TaskPack { task_id: task_id.to_string(), @@ -478,12 +534,40 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res metadata: PackMetadata { generated_at, source_event_count: event_count, - cache_hit: false, + cache_hit, truncated, }, }) } +/// The cacheable part of a pack: every section built from this task's own +/// events and its children's, in pack order. +fn render_body(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Result<String> { + let mut text = String::new(); + if matches!(mode, PackMode::Full) { + text.push_str(&render_lifecycle(conn, task_id)?); + } + text.push_str(&render_active_decisions(conn, task_id)?); + if matches!(mode, PackMode::Full) { + text.push_str(&render_rejected(conn, task_id)?); + text.push_str(&render_evidence(conn, task_id)?); + } + let recent_limit = match mode { + PackMode::Compact => 3, + PackMode::Full => 10, + }; + text.push_str(&render_recent_events(conn, task_id, recent_limit)?); + + // One-level roll-up of direct children (parents only). Appended before + // truncation so it shares the pack budget. Task 5 busts the parent cache + // when a child changes, so the next assemble regenerates fresh. + if let Some(subtasks) = render_subtasks(conn, task_id)? { + text.push_str(&subtasks); + } + + Ok(text) +} + #[cfg(test)] mod tests { use super::*; @@ -592,6 +676,7 @@ mod tests { // pending_count() resolves `<data_dir>/pending`. Point the data dir at // the isolated tempdir (no pending/ child) so the PendingLeak rule // stays silent regardless of the real dev environment. + let _env = crate::test_env_lock(); std::env::set_var("TASK_JOURNAL_DATA_DIR", d.path()); let pack = assemble(&conn, "g2", PackMode::Compact).unwrap(); @@ -1207,4 +1292,278 @@ mod tests { assert_eq!(pack.metadata.source_event_count, 1); assert!(!pack.metadata.cache_hit); } + + /// Upsert + index one event, the way the tests above do inline. + fn put(conn: &Connection, e: &crate::event::Event) { + crate::db::upsert_task_from_event(conn, e, "feedface").unwrap(); + crate::db::index_event(conn, e).unwrap(); + } + + fn ev(task: &str, ty: crate::event::EventType, text: &str) -> crate::event::Event { + crate::event::Event::new( + task, + ty, + crate::event::Author::Agent, + crate::event::Source::Chat, + text.into(), + ) + } + + fn open_task(conn: &Connection, task: &str, title: &str) { + let mut e = ev(task, crate::event::EventType::Open, "x"); + e.meta = serde_json::json!({ "title": title }); + put(conn, &e); + } + + #[test] + fn corrected_events_drop_out_of_every_pack_section() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-cx", "Corrected"); + + let wrong = [ + ev("tj-cx", EventType::Decision, "Wrong decision: use Mongo"), + ev( + "tj-cx", + EventType::Rejection, + "Wrong rejection: SQLite too slow", + ), + ev("tj-cx", EventType::Evidence, "Wrong evidence: bench 5ms"), + ev("tj-cx", EventType::Finding, "Wrong finding: migration done"), + ]; + for w in &wrong { + put(&conn, w); + let mut c = ev("tj-cx", EventType::Correction, "That was a mistake"); + c.corrects = Some(w.event_id.clone()); + put(&conn, &c); + } + put(&conn, &ev("tj-cx", EventType::Decision, "Use SQLite")); + + let pack = assemble(&conn, "tj-cx", PackMode::Full).unwrap(); + assert!(!pack.text.contains("Wrong "), "{}", pack.text); + assert!(pack.text.contains("Use SQLite"), "{}", pack.text); + assert_eq!( + pack.text.matches("[correction] That was a mistake").count(), + 4, + "{}", + pack.text + ); + } + + #[test] + fn correcting_another_tasks_event_refreshes_that_tasks_cached_pack() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-a", "A"); + open_task(&conn, "tj-b", "B"); + let wrong = ev("tj-a", EventType::Decision, "Wrong decision in A"); + put(&conn, &wrong); + assert!(assemble(&conn, "tj-a", PackMode::Full) + .unwrap() + .text + .contains("Wrong decision in A")); + + let mut c = ev("tj-b", EventType::Correction, "A's decision was wrong"); + c.corrects = Some(wrong.event_id.clone()); + put(&conn, &c); + + let pack = assemble(&conn, "tj-a", PackMode::Full).unwrap(); + assert!(!pack.text.contains("Wrong decision in A"), "{}", pack.text); + } + + /// The body of the `## {heading}` section, up to the next `## ` heading. + fn section<'a>(text: &'a str, heading: &str) -> &'a str { + let start = text + .find(&format!("## {heading}")) + .unwrap_or_else(|| panic!("no {heading} section in:\n{text}")); + let end = text[start + 3..] + .find("\n## ") + .map_or(text.len(), |i| start + 3 + i); + &text[start..end] + } + + #[test] + fn suggested_decisions_and_rejections_are_marked_unconfirmed() { + use crate::event::{EventStatus, EventType}; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-un", "Unconfirmed"); + let mut guess = ev("tj-un", EventType::Decision, "Maybe adopt Rust"); + guess.status = EventStatus::Suggested; + put(&conn, &guess); + let mut guess_rej = ev("tj-un", EventType::Rejection, "Maybe drop TypeScript"); + guess_rej.status = EventStatus::Suggested; + put(&conn, &guess_rej); + put(&conn, &ev("tj-un", EventType::Decision, "Use SQLite")); + put( + &conn, + &ev("tj-un", EventType::Rejection, "Postgres: too heavy"), + ); + + let pack = assemble(&conn, "tj-un", PackMode::Full).unwrap(); + let active = section(&pack.text, "Active decisions"); + assert!( + active.contains("- Maybe adopt Rust _(unconfirmed)_\n"), + "{active}" + ); + assert!(active.contains("- Use SQLite\n"), "{active}"); + let rejected = section(&pack.text, "Rejected"); + assert!( + rejected.contains("- Maybe drop TypeScript _(unconfirmed)_\n"), + "{rejected}" + ); + assert!(rejected.contains("- Postgres: too heavy\n"), "{rejected}"); + } + + #[test] + fn compaction_marker_kind_stays_out_of_active_decisions() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-bk", "Bookkeeping"); + // Identified by meta.kind, whatever the marker text says. + let mut marker = ev("tj-bk", EventType::Decision, "Context boundary here"); + marker.meta = serde_json::json!({ "kind": "compaction_marker" }); + put(&conn, &marker); + put(&conn, &ev("tj-bk", EventType::Decision, "Use SQLite")); + + let pack = assemble(&conn, "tj-bk", PackMode::Full).unwrap(); + let active = section(&pack.text, "Active decisions"); + assert!(!active.contains("Context boundary"), "{active}"); + assert!(active.contains("Use SQLite"), "{active}"); + } + + #[test] + fn truncated_packs_keep_the_gap_summary() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-gap", "Gaps"); // no goal → a gap + for i in 0..60 { + let long = format!("Decision #{i}: {}", "lorem ipsum ".repeat(60)); + put(&conn, &ev("tj-gap", EventType::Decision, &long)); + } + + for mode in [PackMode::Compact, PackMode::Full] { + let pack = assemble(&conn, "tj-gap", mode).unwrap(); + assert!(pack.metadata.truncated, "{mode:?}"); + let gaps = section(&pack.text, "Completeness"); + assert!(gaps.contains("no goal recorded"), "{mode:?}: {gaps}"); + } + + let compact = assemble(&conn, "tj-gap", PackMode::Compact).unwrap(); + assert!( + compact.text.len() <= 2 * 1024 + 64, + "{}", + compact.text.len() + ); + let gaps = section(&compact.text, "Completeness"); + assert_eq!( + gaps.trim_end().lines().count(), + 2, + "one summary line: {gaps}" + ); + } + + #[test] + fn compact_gap_line_stays_short_with_many_gaps() { + use crate::completeness::{CompletenessReport, Gap, GapKind}; + + let report = CompletenessReport { + gaps: (0..50) + .map(|i| Gap { + kind: GapKind::MissingFile, + detail: format!("referenced file no longer exists: src/deep/path/file_{i}.rs"), + }) + .collect(), + }; + let line = render_gap_line(&report).unwrap(); + assert!(line.len() <= 512, "{} bytes: {line}", line.len()); + assert!(line.contains("## Completeness (50)"), "{line}"); + assert!(line.contains("honesty score: 0/100"), "{line}"); + } + + #[test] + fn cached_pack_shows_live_status_of_linked_tasks() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-a", "A"); + open_task(&conn, "tj-b", "B"); + crate::db::add_task_external(&conn, "tj-a", "linked:tj-b").unwrap(); + assert!(assemble(&conn, "tj-a", PackMode::Compact) + .unwrap() + .text + .contains("- tj-b [open]")); + + // Closing B touches only B's events, so A's cached body stays valid. + put(&conn, &ev("tj-b", EventType::Close, "done")); + + let pack = assemble(&conn, "tj-a", PackMode::Compact).unwrap(); + assert!(pack.metadata.cache_hit); + assert!(pack.text.contains("- tj-b [closed]"), "{}", pack.text); + } + + #[test] + fn cached_pack_shows_live_gaps() { + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-g", "G"); + crate::db::set_task_goal(&conn, "tj-g", "g").unwrap(); + let _env = crate::test_env_lock(); + std::env::set_var("TASK_JOURNAL_DATA_DIR", d.path()); + let first = assemble(&conn, "tj-g", PackMode::Compact).unwrap(); + + // A pending entry appears without any new event on the task. + std::fs::create_dir_all(d.path().join("pending")).unwrap(); + std::fs::write(d.path().join("pending/x.json"), "{}").unwrap(); + let second = assemble(&conn, "tj-g", PackMode::Compact).unwrap(); + std::env::remove_var("TASK_JOURNAL_DATA_DIR"); + + assert!(!first.text.contains("## Completeness"), "{}", first.text); + assert!(second.metadata.cache_hit); + assert!(second.text.contains("1 pending entry"), "{}", second.text); + } + + #[test] + fn one_bad_alternative_does_not_hide_the_others() { + let raw = r#"[ + {"chosen": true, "rationale": "no option field"}, + {"option": "SQLite", "chosen": true}, + {"option": "Postgres", "rationale": "too heavy"} + ]"#; + let block = render_alternatives(Some(raw)).expect("good entries still render"); + assert!(block.contains("✓ chose SQLite"), "{block}"); + assert!(block.contains("✗ Postgres — too heavy"), "{block}"); + assert!(!block.contains("no option field"), "{block}"); + } + + #[test] + fn evidence_drops_noise_and_repeats() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-ev2", "Evidence"); + for text in [ + "Tests green: 142 passed", + "Tests green: 142 passed", + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events…", + ] { + put(&conn, &ev("tj-ev2", EventType::Evidence, text)); + } + + let pack = assemble(&conn, "tj-ev2", PackMode::Full).unwrap(); + let evidence = section(&pack.text, "Evidence"); + assert_eq!(evidence.matches("Tests green").count(), 1, "{evidence}"); + assert!(!evidence.contains("Conversation compacted"), "{evidence}"); + } } diff --git a/crates/tj-core/src/paths.rs b/crates/tj-core/src/paths.rs index d854209..cda0a12 100644 --- a/crates/tj-core/src/paths.rs +++ b/crates/tj-core/src/paths.rs @@ -54,6 +54,7 @@ mod tests { #[test] fn data_dir_returns_a_path_containing_task_journal() { + let _env = crate::test_env_lock(); let p = data_dir().expect("data_dir"); let s = p.to_string_lossy(); assert!(s.contains("task-journal"), "got: {s}"); @@ -67,14 +68,15 @@ mod tests { /// Regression: on macOS/Windows the `directories` crate ignores XDG_DATA_HOME, but our /// tests (and power users) need a portable override. data_dir() must respect XDG_DATA_HOME - /// and TASK_JOURNAL_DATA_DIR on every OS. Using a thread-isolated env block since std env - /// is process-global; one test exercises both vars by serially restoring state. + /// and TASK_JOURNAL_DATA_DIR on every OS. std env is process-global, so the test holds + /// the shared env lock; one test exercises both vars by serially restoring state. #[test] #[cfg_attr( not(unix), ignore = "env semantics differ on Windows test runners; covered by integration tests" )] fn env_overrides_take_precedence() { + let _env = crate::test_env_lock(); // Snapshot existing values (best-effort cleanup). let prev_tjdd = std::env::var("TASK_JOURNAL_DATA_DIR").ok(); let prev_xdg = std::env::var("XDG_DATA_HOME").ok(); diff --git a/crates/tj-core/src/project_hash.rs b/crates/tj-core/src/project_hash.rs index bd687b7..80aae36 100644 --- a/crates/tj-core/src/project_hash.rs +++ b/crates/tj-core/src/project_hash.rs @@ -14,14 +14,17 @@ use std::path::{Path, PathBuf}; /// 2. `.git` (file or directory) — covers normal checkouts and /// worktrees alike (a worktree's root holds a `.git` *file* /// pointing at the real gitdir, but its presence still marks the -/// boundary). +/// boundary). Like git itself, a `.git` directory counts only when it +/// holds `HEAD`: Codex's sandbox (bubblewrap) briefly mounts an empty +/// `.git` into a writable root such as `/tmp` while a command runs. /// /// Falls back to `start` if no marker is found, preserving prior /// behaviour for non-git scratch directories. pub fn project_root(start: &Path) -> PathBuf { let mut cur = start; loop { - if cur.join(".task-journal").is_dir() || cur.join(".git").exists() { + let git = cur.join(".git"); + if cur.join(".task-journal").is_dir() || git.is_file() || git.join("HEAD").exists() { return cur.to_path_buf(); } match cur.parent() { @@ -103,6 +106,7 @@ mod tests { // repo/ with .git inside; repo/src/foo/ should normalise to repo/. let repo = TempDir::new().unwrap(); std::fs::create_dir(repo.path().join(".git")).unwrap(); + std::fs::write(repo.path().join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); let sub = repo.path().join("src").join("foo"); std::fs::create_dir_all(&sub).unwrap(); @@ -120,6 +124,7 @@ mod tests { // (explicit opt-out of the parent's journal). let repo = TempDir::new().unwrap(); std::fs::create_dir(repo.path().join(".git")).unwrap(); + std::fs::write(repo.path().join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); let sub = repo.path().join("sub"); std::fs::create_dir(&sub).unwrap(); std::fs::create_dir(sub.join(".task-journal")).unwrap(); @@ -148,4 +153,18 @@ mod tests { "worktree subdir must normalise to worktree root via .git file" ); } + + #[test] + fn empty_dot_git_dir_is_not_a_boundary() { + // Codex's bubblewrap sandbox briefly mounts an empty `.git` into a + // writable root such as `/tmp` while a command runs. Git does not take + // it for a repository, and the project hash must not flip meanwhile. + let base = TempDir::new().unwrap(); + let proj = base.path().join("proj"); + std::fs::create_dir(&proj).unwrap(); + let before = from_path(&proj).unwrap(); + + std::fs::create_dir(base.path().join(".git")).unwrap(); + assert_eq!(from_path(&proj).unwrap(), before); + } } diff --git a/crates/tj-core/src/recall.rs b/crates/tj-core/src/recall.rs index 68179d2..c8b96ea 100644 --- a/crates/tj-core/src/recall.rs +++ b/crates/tj-core/src/recall.rs @@ -8,6 +8,7 @@ use crate::event::EventType; /// One recalled high-signal event that matched the current context. #[derive(Debug, Clone, PartialEq)] pub struct RecallHit { + pub event_id: String, pub task_id: String, pub event_type: EventType, // Rejection | Decision pub text: String, @@ -86,6 +87,8 @@ pub fn relevant_recall( JOIN search_fts sf ON sf.event_id = ei.event_id WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') + AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND search_fts MATCH ?1" } else { "SELECT ei.event_id, ei.task_id, ei.type, sf.text @@ -93,6 +96,8 @@ pub fn relevant_recall( JOIN search_fts sf ON sf.event_id = ei.event_id WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') + AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND sf.text LIKE ?1" }; let bind = if let Some(or_query) = fts_or { @@ -136,6 +141,8 @@ pub fn relevant_recall( JOIN search_fts sf ON sf.event_id = ei.event_id WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') + AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND ei.artifacts LIKE ?1", ) { let rows = stmt.query_map(rusqlite::params![pattern], |r| { @@ -164,6 +171,7 @@ pub fn relevant_recall( .filter_map(|(eid, score)| { meta.remove(&eid) .map(|(task_id, event_type, text)| RecallHit { + event_id: eid, task_id, event_type, text, @@ -328,4 +336,48 @@ mod tests { .unwrap() .is_empty()); } + + #[test] + fn compaction_markers_are_never_recalled() { + // An old marker (recognised by its text) and a new one (by meta.kind). + let old = ev( + "tj-1", + EventType::Decision, + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events should be treated as a single reasoning unit.", + EventStatus::Confirmed, + ); + let mut new = ev( + "tj-1", + EventType::Decision, + "Compaction boundary: reasoning unit closed.", + EventStatus::Confirmed, + ); + new.meta = serde_json::json!({ "kind": "compaction_marker" }); + let (_d, conn) = seeded(&[old, new]); + + let hits = relevant_recall(&conn, "reasoning unit", DEFAULT_MAX_HITS).unwrap(); + assert!(hits.is_empty(), "got: {hits:?}"); + } + + #[test] + fn corrected_events_are_never_recalled() { + let rej = ev( + "tj-1", + EventType::Rejection, + "Rejected axum for the server in src/server.rs.", + EventStatus::Confirmed, + ); + let mut corr = ev( + "tj-1", + EventType::Correction, + "That was wrong: the stdio issue was unrelated.", + EventStatus::Confirmed, + ); + corr.corrects = Some(rej.event_id.clone()); + let (_d, conn) = seeded(&[rej, corr]); + + // Both the text and the artifact path must skip the corrected event. + let hits = relevant_recall(&conn, "axum src/server.rs", DEFAULT_MAX_HITS).unwrap(); + assert!(hits.is_empty(), "got: {hits:?}"); + } } diff --git a/crates/tj-core/src/reminder.rs b/crates/tj-core/src/reminder.rs index 19e4d6c..da2bd41 100644 --- a/crates/tj-core/src/reminder.rs +++ b/crates/tj-core/src/reminder.rs @@ -5,6 +5,12 @@ use rusqlite::Connection; pub const MAX_CONSTRAINTS: usize = 3; +/// Text prefix of the `constraint` events ingest-hook writes on a model +/// switch (they also carry `meta.kind = "model_switch"`). SQLite does not +/// index `meta`, so constraint lists filter on this prefix to keep a run of +/// switches from crowding out the task's real constraints. +pub const MODEL_SWITCH_TEXT_PREFIX: &str = "Model switched ("; + /// Most-recent OPEN task → "title + goal + up to MAX_CONSTRAINTS newest /// constraint texts". `None` when there is no open task. Read-only. pub fn active_task_reminder(conn: &Connection, label: &str) -> anyhow::Result<Option<String>> { @@ -30,12 +36,15 @@ pub fn active_task_reminder(conn: &Connection, label: &str) -> anyhow::Result<Op "SELECT sf.text FROM events_index ei \ LEFT JOIN search_fts sf ON sf.event_id = ei.event_id \ WHERE ei.task_id = ?1 AND ei.type = 'constraint' \ + AND COALESCE(sf.text, '') NOT LIKE ?3 \ ORDER BY ei.timestamp DESC LIMIT ?2", )?; + let model_switch = format!("{MODEL_SWITCH_TEXT_PREFIX}%"); let constraints: Vec<String> = stmt - .query_map(rusqlite::params![task_id, MAX_CONSTRAINTS as i64], |r| { - r.get::<_, Option<String>>(0) - })? + .query_map( + rusqlite::params![task_id, MAX_CONSTRAINTS as i64, model_switch], + |r| r.get::<_, Option<String>>(0), + )? .filter_map(|r| r.ok().flatten()) .filter(|t| !t.trim().is_empty()) .collect(); @@ -126,6 +135,35 @@ mod tests { assert!(!r.contains("OLDEST"), "oldest constraint leaked: {r}"); } + #[test] + fn reminder_skips_model_switch_constraints() { + // Model switches are recorded as constraints, but a run of them must + // not push the task's real constraints out of the 3 slots. + let events = vec![ + open_event("tj-1", "Build the widget"), + constraint_event("tj-1", "API key rotates daily", "2026-06-01T00:00:00Z"), + constraint_event("tj-1", "Must support offline mode", "2026-06-02T00:00:00Z"), + constraint_event("tj-1", "Ship before Friday", "2026-06-03T00:00:00Z"), + constraint_event( + "tj-1", + "Model switched (auto): opus → haiku", + "2026-06-04T00:00:00Z", + ), + constraint_event( + "tj-1", + "Model switched (user): haiku → opus", + "2026-06-05T00:00:00Z", + ), + ]; + let (_d, conn) = seed(&events); + + let r = active_task_reminder(&conn, "Active task").unwrap().unwrap(); + assert!(r.contains("API key rotates daily"), "got: {r}"); + assert!(r.contains("Must support offline mode"), "got: {r}"); + assert!(r.contains("Ship before Friday"), "got: {r}"); + assert!(!r.contains("Model switched"), "got: {r}"); + } + #[test] fn reminder_none_when_no_open_task() { let (_d, conn) = seed(&[]); diff --git a/crates/tj-core/src/session/discovery.rs b/crates/tj-core/src/session/discovery.rs index 669896c..ea3140c 100644 --- a/crates/tj-core/src/session/discovery.rs +++ b/crates/tj-core/src/session/discovery.rs @@ -37,7 +37,8 @@ pub fn encode_project_path(path: &str) -> String { } /// Find the project directory for a given filesystem path. -/// Tries exact match first, then prefix match for worktree variants. +/// Tries an exact match first, then a case-insensitive one (WSL paths can +/// differ in case). Worktrees are not matched to their main checkout. pub fn find_project_dir(project_path: &Path) -> anyhow::Result<Option<PathBuf>> { let projects = projects_dir()?; if !projects.exists() { @@ -98,35 +99,6 @@ pub fn list_sessions(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> { Ok(sessions.into_iter().map(|(p, _)| p).collect()) } -/// List all project directories in Claude Code config. -pub fn list_all_projects() -> anyhow::Result<Vec<(String, PathBuf)>> { - let projects = projects_dir()?; - if !projects.exists() { - return Ok(vec![]); - } - - let mut result = Vec::new(); - for entry in std::fs::read_dir(&projects)? { - let entry = entry?; - if entry.path().is_dir() { - let name = entry.file_name().to_string_lossy().to_string(); - // Decode the project name back to a readable path. - let decoded = decode_project_path(&name); - result.push((decoded, entry.path())); - } - } - result.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(result) -} - -/// Decode an encoded project directory name back to a readable path. -/// This is approximate — we can't distinguish `-` from original `/`. -fn decode_project_path(encoded: &str) -> String { - // Common pattern: leading `--` means the path started with a path separator. - // Replace double dashes carefully. - encoded.to_string() -} - fn dirs_home() -> anyhow::Result<PathBuf> { directories::BaseDirs::new() .map(|d| d.home_dir().to_path_buf()) @@ -136,16 +108,6 @@ fn dirs_home() -> anyhow::Result<PathBuf> { #[cfg(test)] mod tests { use super::*; - use std::sync::Mutex; - - /// Serialize every test that touches `CLAUDE_CONFIG_DIR`. Cargo runs - /// unit tests in parallel by default; two tests mutating the same - /// process env race (set in A, observed in B) and flaked Windows CI - /// (saw "C:\Users\runneradmin\.claude" when expecting the override). - /// Tests that touch the env take this lock before the first set_var. - /// `lock().unwrap_or_else(|p| p.into_inner())` swallows poisoning - /// from a panicking sibling test — env is restored regardless. - static ENV_LOCK: Mutex<()> = Mutex::new(()); #[test] fn encode_path_replaces_separators() { @@ -195,15 +157,19 @@ mod tests { fn list_sessions_sorted_by_mtime_newest_first() { let dir = tempfile::tempdir().unwrap(); - // Create files with different modification times. - let older = dir.path().join("older.jsonl"); - std::fs::write(&older, "{}").unwrap(); - - // Sleep briefly to ensure different mtime. - std::thread::sleep(std::time::Duration::from_millis(50)); - - let newer = dir.path().join("newer.jsonl"); - std::fs::write(&newer, "{}").unwrap(); + // Set the mtimes explicitly: a sleep between two writes is not enough + // when the wall clock steps back (seen on WSL), and it slows the test. + let base = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); + for (name, secs) in [("older.jsonl", 0), ("newer.jsonl", 60)] { + let path = dir.path().join(name); + std::fs::write(&path, "{}").unwrap(); + std::fs::File::options() + .write(true) + .open(&path) + .unwrap() + .set_modified(base + std::time::Duration::from_secs(secs)) + .unwrap(); + } let sessions = list_sessions(dir.path()).unwrap(); assert_eq!(sessions.len(), 2); @@ -229,45 +195,11 @@ mod tests { assert!(result.is_err()); } - // --- list_all_projects() --- - - #[test] - fn list_all_projects_with_temp_dir() { - let dir = tempfile::tempdir().unwrap(); - // Override CLAUDE_CONFIG_DIR for this test. - let config_dir = dir.path(); - let projects = config_dir.join("projects"); - std::fs::create_dir_all(&projects).unwrap(); - - // Create project directories. - std::fs::create_dir(projects.join("-home-user-project-alpha")).unwrap(); - std::fs::create_dir(projects.join("-home-user-project-beta")).unwrap(); - // Create a file (should be skipped — not a directory). - std::fs::write(projects.join("not-a-dir.txt"), "").unwrap(); - - // We can't easily test list_all_projects() because it uses projects_dir() - // which reads CLAUDE_CONFIG_DIR. Instead, test the directory listing logic directly. - let mut result = Vec::new(); - for entry in std::fs::read_dir(&projects).unwrap() { - let entry = entry.unwrap(); - if entry.path().is_dir() { - let name = entry.file_name().to_string_lossy().to_string(); - let decoded = decode_project_path(&name); - result.push((decoded, entry.path())); - } - } - result.sort_by(|a, b| a.0.cmp(&b.0)); - - assert_eq!(result.len(), 2); - assert!(result[0].0.contains("alpha")); - assert!(result[1].0.contains("beta")); - } - // --- find_project_dir() with CLAUDE_CONFIG_DIR env override --- #[test] fn find_project_dir_with_env_override() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); let projects = dir.path().join("projects"); std::fs::create_dir_all(&projects).unwrap(); @@ -292,7 +224,7 @@ mod tests { #[test] fn find_project_dir_returns_none_when_no_match() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); let projects = dir.path().join("projects"); std::fs::create_dir_all(&projects).unwrap(); @@ -308,7 +240,7 @@ mod tests { #[test] fn find_project_dir_returns_none_when_projects_dir_missing() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); // Don't create a "projects" subdir — it doesn't exist. @@ -321,15 +253,6 @@ mod tests { assert!(result.unwrap().is_none()); } - // --- decode_project_path --- - - #[test] - fn decode_project_path_returns_same_string() { - // Current implementation is identity — just verify it doesn't panic. - let decoded = decode_project_path("-home-user-project"); - assert_eq!(decoded, "-home-user-project"); - } - // --- claude_config_dir --- /// Both `CLAUDE_CONFIG_DIR` cases are combined into one test so the @@ -341,9 +264,9 @@ mod tests { /// than the hardcoded "/tmp/..." that doesn't exist on Windows. #[test] fn claude_config_dir_handles_env_var() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); - // SAFETY: the ENV_LOCK above serializes us against the other - // CLAUDE_CONFIG_DIR tests in this module; prev → restore at + let _g = crate::test_env_lock(); + // SAFETY: the shared env lock serializes us against the other + // env-touching tests; prev → restore at // the end gives a clean exit regardless of panic. let prev = std::env::var_os("CLAUDE_CONFIG_DIR"); diff --git a/crates/tj-core/src/session/extractor.rs b/crates/tj-core/src/session/extractor.rs index 4544bdd..d940cdd 100644 --- a/crates/tj-core/src/session/extractor.rs +++ b/crates/tj-core/src/session/extractor.rs @@ -94,7 +94,7 @@ pub fn extract_from_session(session: &ParsedSession) -> Option<ExtractedTask> { // Git commit → evidence. if tool_name == "Bash" { if let Some(cmd) = input.get("command").and_then(|v| v.as_str()) { - if cmd.contains("git commit") && !cmd.contains("git commit --amend") { + if is_git_commit(cmd) { let mut ev = Event::new( &task_id, EventType::Evidence, @@ -195,17 +195,15 @@ fn derive_title(session: &ParsedSession) -> String { .unwrap_or(&clean); let trimmed = first_line.trim(); // Skip empty or very short titles (likely slash commands). - if trimmed.len() > 5 { + if trimmed.chars().count() > 5 { return truncate(trimmed, 120); } } } } - format!( - "Session {}", - &session.session_id[..8.min(session.session_id.len())] - ) + let head: String = session.session_id.chars().take(8).collect(); + format!("Session {head}") } /// Strip XML/HTML-like tags from text (e.g. <command-message>, <command-name>). @@ -230,7 +228,8 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option let lower = text.to_lowercase(); // Skip very short texts (< 50 chars) — usually just confirmations. - if text.len() < 50 { + let char_count = text.chars().count(); + if char_count < 50 { return None; } @@ -302,7 +301,7 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option "must be", ]; for pattern in &constraint_patterns { - if lower.contains(pattern) && text.len() < 500 { + if lower.contains(pattern) && char_count < 500 { let mut ev = Event::new( task_id, EventType::Constraint, @@ -321,17 +320,60 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option None } -/// Check if a bash command is a test command. +const TEST_RUNNERS: [&str; 8] = [ + "cargo test", + "npm test", + "pytest", + "phpunit", + "jest", + "vitest", + "go test", + "make test", +]; + +/// True when a bash command actually runs a test runner: some `&&`/`;` step +/// must be it (after `VAR=value` assignments, an `npx` / `python -m` +/// launcher or a `vendor/bin/` path), so an `echo`/`grep` that merely +/// mentions "cargo test" doesn't count. fn is_test_command(cmd: &str) -> bool { let lower = cmd.to_lowercase(); - lower.contains("cargo test") - || lower.contains("npm test") - || lower.contains("pytest") - || lower.contains("phpunit") - || lower.contains("jest") - || lower.contains("vitest") - || lower.contains("go test") - || lower.contains("make test") + let is_env_assignment = |word: &&str| { + word.split_once('=').is_some_and(|(name, _)| { + !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + }) + }; + + lower.split("&&").flat_map(|s| s.split(';')).any(|step| { + let words: Vec<&str> = step + .split_whitespace() + .skip_while(is_env_assignment) + .collect(); + let words = match words.as_slice() { + ["npx", rest @ ..] => rest, + [python, "-m", rest @ ..] if python.starts_with("python") => rest, + all => all, + }; + let Some((bin, args)) = words.split_first() else { + return false; + }; + let bin = bin.rsplit('/').next().unwrap_or(bin); + + TEST_RUNNERS + .iter() + .any(|runner| match runner.split_once(' ') { + Some((runner_bin, sub)) => bin == runner_bin && args.first() == Some(&sub), + None => bin == *runner, + }) + }) +} + +/// True when a bash command actually runs `git commit` (not `--amend`): some +/// `&&`-chained step must start with it, so an `echo`/`grep` that merely +/// mentions "git commit" doesn't count. +fn is_git_commit(cmd: &str) -> bool { + cmd.split("&&").map(str::trim).any(|step| { + (step == "git commit" || step.starts_with("git commit ")) && !step.contains("--amend") + }) } /// Shorten a file path for display — keep last 2 components. @@ -344,17 +386,11 @@ fn shorten_path(path: &str) -> String { } } -/// Truncate text to max_len, adding "…" if truncated. +/// Truncate text to max_len chars, adding "…" if truncated. fn truncate(text: &str, max_len: usize) -> String { - if text.len() <= max_len { - text.to_string() - } else { - let mut end = max_len; - // Don't cut in the middle of a UTF-8 char. - while end > 0 && !text.is_char_boundary(end) { - end -= 1; - } - format!("{}…", &text[..end]) + match text.char_indices().nth(max_len) { + Some((end, _)) => format!("{}…", &text[..end]), + None => text.to_string(), } } @@ -626,6 +662,49 @@ mod tests { ); } + fn git_commit_evidence_count(cmd: &str) -> usize { + let session = ParsedSession { + session_id: "git-cmd-session".into(), + file_path: "/tmp/gcmd.jsonl".into(), + entries: vec![ + make_user_entry("u1", "2026-01-01T00:00:00Z", "Do the thing"), + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::ToolUse { + name: "Bash".into(), + input: serde_json::json!({ "command": cmd }), + }], + ), + make_user_entry("u2", "2026-01-01T00:00:02Z", "Thanks"), + ], + first_timestamp: Some("2026-01-01T00:00:00Z".into()), + last_timestamp: Some("2026-01-01T00:00:02Z".into()), + }; + extract_from_session(&session) + .unwrap() + .events + .iter() + .filter(|e| e.text.starts_with("Git commit")) + .count() + } + + #[test] + fn git_commit_evidence_requires_a_real_commit_command() { + assert_eq!(git_commit_evidence_count(" git commit -m 'fix'"), 1); + assert_eq!(git_commit_evidence_count("cd repo && git commit -m 'x'"), 1); + assert_eq!( + git_commit_evidence_count("git add -A && git commit -m 'x'"), + 1 + ); + assert_eq!( + git_commit_evidence_count("echo 'remember to git commit'"), + 0 + ); + assert_eq!(git_commit_evidence_count("grep -rn \"git commit\" ."), 0); + assert_eq!(git_commit_evidence_count("git commit --amend --no-edit"), 0); + } + // --- strip_xml_tags() --- #[test] @@ -735,6 +814,35 @@ mod tests { assert!(title.contains("abcdefgh")); } + #[test] + fn derive_title_counts_chars_not_bytes() { + // 5 chars (9 bytes): as short as "/init", so it is skipped. + let session = ParsedSession { + session_id: "abcdefghij".into(), + file_path: "/tmp/s.jsonl".into(), + entries: vec![ + make_user_entry("u1", "t", "Да да"), + make_user_entry("u2", "t", "Implement the feature for user profiles"), + ], + first_timestamp: None, + last_timestamp: None, + }; + assert!(derive_title(&session).contains("Implement the feature")); + } + + #[test] + fn derive_title_fallback_cuts_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let session = ParsedSession { + session_id: "aééééé".into(), + file_path: "/tmp/s.jsonl".into(), + entries: vec![make_user_entry("u1", "t", "hi")], + first_timestamp: None, + last_timestamp: None, + }; + assert_eq!(derive_title(&session), "Session aééééé"); + } + #[test] fn derive_title_strips_xml_from_summary() { let session = ParsedSession { @@ -764,6 +872,20 @@ mod tests { assert_eq!(ev.unwrap().event_type, EventType::Constraint); } + #[test] + fn classify_thresholds_count_chars_not_bytes() { + // 29 Cyrillic chars but 50+ bytes: still a short confirmation. + let short = "Я выбрал вариант Б, он проще."; + assert!(short.chars().count() < 50 && short.len() >= 50); + assert!(classify_text_heuristic("tj-test", short, "2026-01-01T00:00:00Z").is_none()); + + // Under 500 chars but over 500 bytes: still a constraint. + let long = format!("Есть ограничение: {}", "а".repeat(300)); + assert!(long.chars().count() < 500 && long.len() >= 500); + let ev = classify_text_heuristic("tj-test", &long, "2026-01-01T00:00:00Z"); + assert_eq!(ev.unwrap().event_type, EventType::Constraint); + } + #[test] fn test_classify_no_match_returns_none() { let ev = classify_text_heuristic( @@ -783,10 +905,21 @@ mod tests { assert!(is_test_command("go test ./...")); assert!(is_test_command("make test")); assert!(is_test_command("phpunit tests/Unit")); - assert!(is_test_command("echo 'cargo test'")); // matches because it contains "cargo test" assert!(!is_test_command("ls -la")); } + #[test] + fn is_test_command_requires_the_runner_to_be_the_command() { + assert!(is_test_command("cd crates && cargo test")); + assert!(is_test_command("cargo fmt; cargo test --workspace")); + assert!(is_test_command("RUST_LOG=debug cargo test")); + assert!(is_test_command("npx jest src/")); + assert!(is_test_command("./vendor/bin/phpunit tests/Unit")); + assert!(!is_test_command("echo 'cargo test'")); + assert!(!is_test_command("grep -rn \"pytest\" .")); + assert!(!is_test_command("cat jest.config.js")); + } + // --- shorten_path additional tests --- #[test] @@ -806,12 +939,9 @@ mod tests { #[test] fn test_truncate_multibyte_utf8() { - // Russian text: each char is 2 bytes. - let text = "Привет мир"; - let truncated = truncate(text, 6); - // 6 bytes = 3 cyrillic chars ("При") - assert!(truncated.ends_with('…')); - assert!(truncated.starts_with("При")); + // Russian text: each char is 2 bytes, but the limit counts chars. + assert_eq!(truncate("Привет мир", 6), "Привет…"); + assert_eq!(truncate("Привет", 6), "Привет"); } #[test] diff --git a/crates/tj-core/src/session_id.rs b/crates/tj-core/src/session_id.rs index e561345..d8942e6 100644 --- a/crates/tj-core/src/session_id.rs +++ b/crates/tj-core/src/session_id.rs @@ -1,4 +1,4 @@ -//! Live Claude Code session id helpers. +//! Live agent session id helpers (Claude Code and Codex). //! //! task-journal already *parses* session ids out of Claude Code //! transcripts (`session::parser`) — that is a passive, read-only @@ -9,8 +9,9 @@ //! without time-window heuristics. //! //! Source order: hook payload field `session_id` → `CLAUDE_CODE_SESSION_ID` -//! env var → `None`. `None` means standalone behaviour is unchanged — -//! nothing is added to `meta`. +//! env var → `CODEX_THREAD_ID` → `CODEX_SESSION_ID` (Codex exports both, +//! with the same value, to the processes it starts) → `None`. `None` means +//! standalone behaviour is unchanged — nothing is added to `meta`. use serde_json::Value; @@ -24,11 +25,31 @@ pub fn session_id_from_payload(payload: &Value) -> Option<String> { .map(str::to_string) } -/// Read `CLAUDE_CODE_SESSION_ID` from the environment. Empty counts as absent. +/// Pull the session id out of an MCP request's `_meta`. Codex sends it on +/// every `tools/call` as `x-codex-turn-metadata.session_id` (and the same +/// value as `thread_id`); its MCP servers get no session env var, so this is +/// the only way a Codex event learns its session. Empty counts as absent. +pub fn session_id_from_mcp_meta(meta: &serde_json::Map<String, Value>) -> Option<String> { + let turn = meta.get("x-codex-turn-metadata")?; + + ["session_id", "thread_id"] + .iter() + .filter_map(|key| turn.get(*key).and_then(|v| v.as_str())) + .find(|s| !s.is_empty()) + .map(str::to_string) +} + +/// Read the live session id from the environment: `CLAUDE_CODE_SESSION_ID`, +/// then Codex's `CODEX_THREAD_ID`, then `CODEX_SESSION_ID`. Empty counts as +/// absent. pub fn session_id_from_env() -> Option<String> { - std::env::var("CLAUDE_CODE_SESSION_ID") - .ok() - .filter(|s| !s.is_empty()) + [ + "CLAUDE_CODE_SESSION_ID", + "CODEX_THREAD_ID", + "CODEX_SESSION_ID", + ] + .into_iter() + .find_map(|var| std::env::var(var).ok().filter(|s| !s.is_empty())) } /// Resolve the live session id: hook payload first, env var as fallback. @@ -52,11 +73,8 @@ pub fn stamp_session_id(meta: &mut Value, sid: Option<&str>) { #[cfg(test)] mod tests { use super::*; + use crate::test_env_lock; use serde_json::json; - use std::sync::Mutex; - - // Serialises the env-touching tests — std env is process-global. - static ENV_LOCK: Mutex<()> = Mutex::new(()); #[test] fn payload_session_id_extracted() { @@ -64,6 +82,31 @@ mod tests { assert_eq!(session_id_from_payload(&p).as_deref(), Some("abc-123")); } + #[test] + fn mcp_meta_reads_codex_turn_metadata() { + let meta = json!({ + "callId": "exec-1", + "x-codex-turn-metadata": { "session_id": "c-1", "thread_id": "c-1", "model": "x" } + }); + let meta = meta.as_object().unwrap(); + assert_eq!(session_id_from_mcp_meta(meta).as_deref(), Some("c-1")); + + let thread_only = + json!({ "x-codex-turn-metadata": { "session_id": "", "thread_id": "t-2" } }); + assert_eq!( + session_id_from_mcp_meta(thread_only.as_object().unwrap()).as_deref(), + Some("t-2") + ); + } + + #[test] + fn mcp_meta_without_codex_metadata_is_none() { + // Claude Code sends only its tool-use id. + let claude = json!({ "claudecode/toolUseId": "toolu_1", "progressToken": 2 }); + assert_eq!(session_id_from_mcp_meta(claude.as_object().unwrap()), None); + assert_eq!(session_id_from_mcp_meta(&serde_json::Map::new()), None); + } + #[test] fn payload_empty_or_missing_is_none() { assert_eq!(session_id_from_payload(&json!({"session_id": ""})), None); @@ -95,7 +138,7 @@ mod tests { #[test] fn live_payload_wins_over_env() { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); let p = json!({"session_id": "from-payload"}); assert_eq!(live_session_id(Some(&p)).as_deref(), Some("from-payload")); @@ -104,7 +147,7 @@ mod tests { #[test] fn live_falls_back_to_env() { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); let p = json!({"hook_event_name": "Stop"}); assert_eq!(live_session_id(Some(&p)).as_deref(), Some("from-env")); @@ -114,9 +157,63 @@ mod tests { #[test] fn live_none_when_no_source() { - let _g = ENV_LOCK.lock().unwrap(); - std::env::remove_var("CLAUDE_CODE_SESSION_ID"); + let _g = test_env_lock(); + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } assert_eq!(live_session_id(None), None); assert_eq!(live_session_id(Some(&json!({}))), None); } + + const SESSION_ENV_VARS: [&str; 3] = [ + "CLAUDE_CODE_SESSION_ID", + "CODEX_THREAD_ID", + "CODEX_SESSION_ID", + ]; + + /// Run `f` with exactly the given session env vars set (the rest unset). + fn with_session_env(set: &[(&str, &str)], f: impl FnOnce()) { + let _g = test_env_lock(); + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } + for (k, v) in set { + std::env::set_var(k, v); + } + + f(); + + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } + } + + #[test] + fn env_falls_back_to_codex_thread_then_session_id() { + with_session_env( + &[ + ("CODEX_THREAD_ID", "thread"), + ("CODEX_SESSION_ID", "session"), + ], + || assert_eq!(session_id_from_env().as_deref(), Some("thread")), + ); + with_session_env(&[("CODEX_SESSION_ID", "session")], || { + assert_eq!(session_id_from_env().as_deref(), Some("session")) + }); + with_session_env( + &[("CODEX_THREAD_ID", ""), ("CODEX_SESSION_ID", "session")], + || assert_eq!(session_id_from_env().as_deref(), Some("session")), + ); + } + + #[test] + fn env_prefers_claude_code_over_codex() { + with_session_env( + &[ + ("CLAUDE_CODE_SESSION_ID", "claude"), + ("CODEX_THREAD_ID", "thread"), + ], + || assert_eq!(session_id_from_env().as_deref(), Some("claude")), + ); + } } diff --git a/crates/tj-core/src/session_state.rs b/crates/tj-core/src/session_state.rs new file mode 100644 index 0000000..d3c4b6c --- /dev/null +++ b/crates/tj-core/src/session_state.rs @@ -0,0 +1,267 @@ +//! What an integration shows for one session: the session's active task, +//! what that task holds so far, and how many tasks are open. Read by the +//! Claude Code mod through `task-journal state`; any client can call it. +//! +//! The active task is the open task this session last wrote to (by +//! `meta.session_id`), never "the newest open task", so two sessions in one +//! project don't share a task. + +use std::collections::BTreeMap; + +use rusqlite::{Connection, OptionalExtension}; +use serde::Serialize; + +/// Bumped when a field changes meaning; readers refuse other schemas. +pub const SCHEMA: &str = "tj-state/1"; + +/// How many of the task's latest entries ride along: enough for a model to +/// tell what is already recorded, small enough to stay cheap. +const RECENT: i64 = 20; + +#[derive(Debug, Serialize)] +pub struct SessionState { + pub schema: &'static str, + pub session_id: Option<String>, + pub open_tasks: i64, + pub active: Option<ActiveTask>, +} + +#[derive(Debug, Serialize)] +pub struct ActiveTask { + pub task_id: String, + pub title: String, + pub goal: Option<String>, + /// Entries per event type, e.g. `{"decision": 4, "evidence": 1}`. + pub counts: BTreeMap<String, i64>, + /// The latest entries, oldest first. No `open` / `amend` bookkeeping. + pub recent: Vec<RecentEntry>, +} + +#[derive(Debug, Serialize)] +pub struct RecentEntry { + #[serde(rename = "type")] + pub kind: String, + pub text: String, +} + +/// Assemble the state of `session_id` in one project. The caller has +/// ingested the journal into `conn` already. `prefer` names the task the +/// caller already shows: it stays the active one while it is open, so a +/// session logging to a parent and a subtask in turn doesn't flip between +/// them. +pub fn session_state( + conn: &Connection, + project_hash: &str, + session_id: Option<&str>, + prefer: Option<&str>, +) -> anyhow::Result<SessionState> { + let open_tasks = conn.query_row( + "SELECT COUNT(*) FROM tasks WHERE project_hash = ?1 AND status = 'open'", + [project_hash], + |r| r.get::<_, i64>(0), + )?; + + let preferred = match prefer { + Some(id) => conn + .query_row( + "SELECT task_id FROM tasks WHERE task_id = ?1 AND project_hash = ?2 AND status = 'open'", + [id, project_hash], + |r| r.get::<_, String>(0), + ) + .optional()?, + None => None, + }; + let active_id = match (preferred, session_id) { + (Some(id), _) => Some(id), + (None, Some(sid)) => crate::db::active_task_for_session(conn, project_hash, sid)?, + (None, None) => None, + }; + let active = active_id.map(|id| active_task(conn, &id)).transpose()?; + + Ok(SessionState { + schema: SCHEMA, + session_id: session_id.map(str::to_string), + open_tasks, + active, + }) +} + +fn active_task(conn: &Connection, task_id: &str) -> anyhow::Result<ActiveTask> { + let (title, goal) = conn.query_row( + "SELECT title, goal FROM tasks WHERE task_id = ?1", + [task_id], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?)), + )?; + + let mut counts = BTreeMap::new(); + // Corrected entries and bookkeeping (compaction markers, model switches) + // are not part of what the task holds. + let mut stmt = conn.prepare( + "SELECT type, COUNT(*) FROM events_index + WHERE task_id = ?1 AND corrected_by IS NULL AND bookkeeping = 0 + GROUP BY type", + )?; + for row in stmt.query_map([task_id], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)) + })? { + let (kind, n) = row?; + counts.insert(kind, n); + } + + let mut stmt = conn.prepare( + "SELECT e.type, f.text FROM events_index e + JOIN search_fts f ON f.event_id = e.event_id + WHERE e.task_id = ?1 AND e.type NOT IN ('open', 'amend') + AND e.corrected_by IS NULL AND e.bookkeeping = 0 + ORDER BY e.timestamp DESC, e.event_id DESC LIMIT ?2", + )?; + let mut recent = stmt + .query_map(rusqlite::params![task_id, RECENT], |r| { + Ok(RecentEntry { + kind: r.get(0)?, + text: r.get(1)?, + }) + })? + .collect::<Result<Vec<_>, _>>()?; + recent.reverse(); + + Ok(ActiveTask { + task_id: task_id.to_string(), + title, + goal: goal.filter(|g| !g.trim().is_empty()), + counts, + recent, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::{Author, Event, EventType, Source}; + + fn write(conn: &Connection, task: &str, kind: EventType, text: &str, session: Option<&str>) { + let mut event = Event::new(task, kind, Author::Agent, Source::Chat, text.into()); + if kind == EventType::Open { + event.meta = + serde_json::json!({ "title": format!("Title of {task}"), "goal": "Ship it" }); + } + crate::session_id::stamp_session_id(&mut event.meta, session); + + crate::db::upsert_task_from_event(conn, &event, "p").unwrap(); + crate::db::index_event(conn, &event).unwrap(); + } + + // The TempDir rides along so the database outlives the test body. + fn conn() -> (tempfile::TempDir, Connection) { + let dir = tempfile::tempdir().unwrap(); + let conn = crate::db::open(dir.path().join("s.sqlite")).unwrap(); + (dir, conn) + } + + #[test] + fn active_task_is_the_sessions_own_not_the_newest() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + write(&conn, "tj-a", EventType::Decision, "Use <=", Some("s1")); + write(&conn, "tj-b", EventType::Open, "b", Some("s2")); + + let s1 = session_state(&conn, "p", Some("s1"), None).unwrap(); + let active = s1.active.expect("s1 has a task"); + assert_eq!(active.task_id, "tj-a"); + assert_eq!(active.title, "Title of tj-a"); + assert_eq!(active.goal.as_deref(), Some("Ship it")); + assert_eq!(active.counts.get("decision"), Some(&1)); + assert_eq!(active.recent.len(), 1); + assert_eq!(active.recent[0].kind, "decision"); + assert_eq!(s1.open_tasks, 2); + + let other = session_state(&conn, "p", Some("s3"), None).unwrap(); + assert!(other.active.is_none()); + assert_eq!(other.open_tasks, 2); + } + + #[test] + fn a_preferred_open_task_stays_active() { + let (_dir, conn) = conn(); + write(&conn, "tj-parent", EventType::Open, "p", Some("s1")); + write(&conn, "tj-sub", EventType::Open, "s", Some("s1")); + + // Without a preference the newest write wins; with one, the pin holds. + let plain = session_state(&conn, "p", Some("s1"), None).unwrap(); + assert_eq!(plain.active.unwrap().task_id, "tj-sub"); + let pinned = session_state(&conn, "p", Some("s1"), Some("tj-parent")).unwrap(); + assert_eq!(pinned.active.unwrap().task_id, "tj-parent"); + + // A closed or unknown preference falls back to the session's own task. + write(&conn, "tj-parent", EventType::Close, "done", Some("s1")); + let closed = session_state(&conn, "p", Some("s1"), Some("tj-parent")).unwrap(); + assert_eq!(closed.active.unwrap().task_id, "tj-sub"); + let unknown = session_state(&conn, "p", Some("s1"), Some("tj-nope")).unwrap(); + assert_eq!(unknown.active.unwrap().task_id, "tj-sub"); + } + + #[test] + fn corrected_entries_are_left_out_of_counts_and_recent() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + let mut wrong = Event::new( + "tj-a", + EventType::Decision, + Author::Agent, + Source::Chat, + "Use X".into(), + ); + crate::session_id::stamp_session_id(&mut wrong.meta, Some("s1")); + crate::db::upsert_task_from_event(&conn, &wrong, "p").unwrap(); + crate::db::index_event(&conn, &wrong).unwrap(); + let mut fix = Event::new( + "tj-a", + EventType::Correction, + Author::Agent, + Source::Chat, + "Not X, Y".into(), + ); + fix.corrects = Some(wrong.event_id.clone()); + crate::db::upsert_task_from_event(&conn, &fix, "p").unwrap(); + crate::db::index_event(&conn, &fix).unwrap(); + + let active = session_state(&conn, "p", Some("s1"), None) + .unwrap() + .active + .unwrap(); + assert_eq!( + active.counts.get("decision"), + None, + "the corrected decision is not counted" + ); + assert!(active.recent.iter().all(|r| r.text != "Use X")); + assert!(active.recent.iter().any(|r| r.text == "Not X, Y")); + } + + #[test] + fn a_closed_task_is_no_longer_active() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + write(&conn, "tj-a", EventType::Close, "done", Some("s1")); + + let s = session_state(&conn, "p", Some("s1"), None).unwrap(); + assert!(s.active.is_none()); + assert_eq!(s.open_tasks, 0); + } + + #[test] + fn json_shape_is_the_contract() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + + let v = serde_json::to_value(session_state(&conn, "p", Some("s1"), None).unwrap()).unwrap(); + assert_eq!(v["schema"], "tj-state/1"); + assert_eq!(v["session_id"], "s1"); + assert_eq!(v["active"]["task_id"], "tj-a"); + assert!(v["active"]["recent"].as_array().unwrap().is_empty()); + + let none = serde_json::to_value(session_state(&conn, "p", None, None).unwrap()).unwrap(); + assert!(none["active"].is_null()); + assert!(none["session_id"].is_null()); + } +} diff --git a/crates/tj-core/src/storage.rs b/crates/tj-core/src/storage.rs index d57a48f..d1ad73a 100644 --- a/crates/tj-core/src/storage.rs +++ b/crates/tj-core/src/storage.rs @@ -2,7 +2,7 @@ use crate::event::Event; use anyhow::Context; use fd_lock::RwLock as FdLock; use std::fs::{File, OpenOptions}; -use std::io::Write; +use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; /// Append-only writer for the events JSONL log. Holds an advisory @@ -45,12 +45,22 @@ impl JsonlWriter { } pub fn append(&mut self, event: &Event) -> anyhow::Result<()> { - let line = serde_json::to_string(event).context("serialize event")?; + let mut line = serde_json::to_string(event).context("serialize event")?; + line.push('\n'); let mut guard = self.lock.write().context("acquire exclusive file lock")?; + + // A crash mid-append leaves a torn last line without its newline. + // Terminate it first, so only that line is lost and not this event, + // which would otherwise glue onto it and be skipped as malformed too. + if ends_without_newline(&mut guard).context("check the log's last byte")? { + guard + .write_all(b"\n") + .context("terminate a torn last line")?; + } + guard .write_all(line.as_bytes()) .context("write event line")?; - guard.write_all(b"\n").context("write newline")?; Ok(()) } @@ -68,6 +78,21 @@ impl JsonlWriter { } } +/// True when the file is non-empty and its last byte is not `\n`. Writes +/// still go to the end afterwards: the file is opened in append mode. +fn ends_without_newline(file: &mut File) -> std::io::Result<bool> { + let len = file.metadata()?.len(); + if len == 0 { + return Ok(false); + } + + file.seek(SeekFrom::Start(len - 1))?; + let mut last = [0u8; 1]; + file.read_exact(&mut last)?; + + Ok(last[0] != b'\n') +} + #[cfg(test)] mod tests { use super::*; @@ -124,6 +149,30 @@ mod tests { assert_eq!(body.lines().count(), 2); } + #[test] + fn append_after_a_torn_last_line_loses_only_that_line() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("events.jsonl"); + + // A crash mid-append left a whole event and then half of the next one. + let whole = serde_json::to_string(&sample_event("whole")).unwrap(); + std::fs::write(&path, format!("{whole}\n{{\"event_id\":\"01TORN")).unwrap(); + + let mut w = JsonlWriter::open(&path).unwrap(); + w.append(&sample_event("after crash")).unwrap(); + w.flush_durable().unwrap(); + drop(w); + + let body = std::fs::read_to_string(&path).unwrap(); + let parsed: Vec<Event> = body + .lines() + .filter_map(|l| serde_json::from_str(l).ok()) + .collect(); + let texts: Vec<&str> = parsed.iter().map(|e| e.text.as_str()).collect(); + assert_eq!(texts, ["whole", "after crash"], "file:\n{body}"); + assert!(body.ends_with('\n')); + } + #[test] fn concurrent_appends_do_not_interleave_bytes() { // Eight threads, each owning its own JsonlWriter (own File handle diff --git a/crates/tj-core/tests/golden_pack.rs b/crates/tj-core/tests/golden_pack.rs index da80f13..fa01f36 100644 --- a/crates/tj-core/tests/golden_pack.rs +++ b/crates/tj-core/tests/golden_pack.rs @@ -25,6 +25,19 @@ fn fixture_a_compact_pack_for_simple_task() { insta_assert_contains(&pack.text, "Adopt PKCE flow"); insta_assert_contains(&pack.text, "Recent events"); assert_eq!(pack.metadata.source_event_count, 5); + + assert!( + pack.text + .starts_with("# Add OAuth login [status: open]\n\n**Goal**: (not set)\n"), + "header:\n{}", + pack.text + ); + assert_eq!( + section_order(&pack.text), + ["Active decisions", "Recent events", "Completeness"], + "{}", + pack.text + ); } fn build_fixture_a() -> Vec<Event> { @@ -107,6 +120,35 @@ fn fixture_b_full_pack_with_supersede_and_correction() { ); assert_eq!(pack.metadata.source_event_count, 12); + + assert!( + pack.text.starts_with( + "# Stack choice for journal [status: closed]\n\n**Goal**: (not set)\n**Outcome**: (not recorded)\n" + ), + "header:\n{}", + pack.text + ); + assert_eq!( + section_order(&pack.text), + [ + "Lifecycle", + "Active decisions", + "Rejected", + "Evidence", + "Recent events", + "Completeness" + ], + "{}", + pack.text + ); +} + +/// The pack's `## ` section headings in order, without a `(count)` suffix. +fn section_order(text: &str) -> Vec<&str> { + text.lines() + .filter_map(|l| l.strip_prefix("## ")) + .map(|h| h.split(" (").next().unwrap_or(h)) + .collect() } fn build_fixture_b() -> Vec<Event> { diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index 7c71bf6..bf50032 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] # Lean: the MCP server doesn't embed yet, so it skips the model2vec backend. -tj-core = { package = "task-journal-core", version = "0.29.0", path = "../tj-core", default-features = false } +tj-core = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index c374c90..f5bbb61 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -5,8 +5,8 @@ use anyhow::{Context, Result}; use clap::Parser; use rmcp::{ - handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_handler, - tool_router, transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, + handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_router, + transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, }; use rusqlite::Connection; use serde::{Deserialize, Serialize}; @@ -131,9 +131,10 @@ tools — so call them, every session, without being asked. THE RITUAL — do this on EVERY coding session, not optional: -1. START. task_search(status="open") for related work → task_pack to resume it; - if nothing fits, task_create(title, goal=<one sentence: what the user is - trying to accomplish>). Hold the returned task_id for the whole session. +1. START. task_search(status="open") lists open tasks, newest first (add a + `query` to narrow; each entry in `tasks` has title + goal) → task_pack to + resume one; if nothing fits, task_create(title, goal=<one sentence: what the + user is trying to accomplish>). Hold the returned task_id for the whole session. 2. AT THE MOMENT you commit to an approach → event_add(event_type="decision", ...) and pass `alternatives` (the options you weighed). Right then — not at the end, or you will forget. @@ -178,12 +179,20 @@ pub struct TaskPackResult { pub struct TaskPackMetadata { pub source_event_count: Option<usize>, pub cache_hit: Option<bool>, + /// RFC 3339 time the pack text was assembled. + pub generated_at: Option<String>, + /// True when the pack was cut to fit its size budget. + pub truncated: Option<bool>, } #[derive(Debug, Deserialize, schemars::JsonSchema)] pub struct TaskSearchParams { + /// Full-text query. Empty or absent lists the project's tasks instead. + #[serde(default)] pub query: String, + /// `open`, `closed`, or `any` (the default). pub status: Option<String>, + /// Absolute path of another project directory to search instead of this one. pub project: Option<String>, /// v0.10.3+: restrict matches to a single event type /// (`decision`, `evidence`, `finding`, `rejection`, ...). @@ -193,7 +202,29 @@ pub struct TaskSearchParams { #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskSearchResult { pub query: String, + /// Matching task ids. Kept for older clients; `tasks` has the same ids + /// in the same order with enough detail to pick one. pub results: Vec<String>, + pub tasks: Vec<TaskSearchHit>, +} + +#[derive(Debug, Serialize, schemars::JsonSchema)] +pub struct TaskSearchHit { + pub task_id: String, + pub title: String, + pub status: String, + pub last_event_at: String, + pub goal: Option<String>, +} + +fn task_search_hit(r: &rusqlite::Row) -> rusqlite::Result<TaskSearchHit> { + Ok(TaskSearchHit { + task_id: r.get(0)?, + title: r.get(1)?, + status: r.get(2)?, + last_event_at: r.get(3)?, + goal: r.get(4)?, + }) } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -208,6 +239,8 @@ pub struct TaskCreateParams { /// Parent task id — makes this a subtask of the given id. Validated: the /// parent must exist and the link must not introduce a cycle. pub parent: Option<String>, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option<String>, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskCreateResult { @@ -227,6 +260,8 @@ pub struct EventAddParams { /// options and the final choice explicit. Stamped onto /// `meta.alternatives`. Rejected with an error on any non-decision type. pub alternatives: Option<serde_json::Value>, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option<String>, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct EventAddResult { @@ -244,6 +279,8 @@ pub struct ArtifactAddParams { pub url: String, /// Human label shown on the card. pub label: String, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option<String>, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct ArtifactAddResult { @@ -259,6 +296,8 @@ pub struct TaskCloseParams { /// v0.4.0+: structured outcome tag — `done`, `abandoned`, or /// `superseded`. Filterable; the free-form text lives in `outcome`. pub outcome_tag: Option<String>, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option<String>, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskCloseResult { @@ -318,12 +357,47 @@ fn resolve_project_paths( Ok((project_hash, events, state)) } -fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { - let dir = match PROJECT_DIR_OVERRIDE.get() { +/// The project directory every tool works on: `--project-dir`, else the cwd. +fn project_dir() -> anyhow::Result<PathBuf> { + Ok(match PROJECT_DIR_OVERRIDE.get() { Some(p) => p.clone(), None => std::env::current_dir()?, - }; - resolve_project_paths(&dir) + }) +} + +fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { + resolve_project_paths(&project_dir()?) +} + +/// The session id to stamp on an event: the caller's `session_id` param when +/// given (empty counts as absent), else the live one from the environment. +fn session_id_or_env(explicit: Option<&str>) -> Option<String> { + explicit + .filter(|s| !s.is_empty()) + .map(str::to_string) + .or_else(tj_core::session_id::session_id_from_env) +} + +/// Ingest the journal tail, then fail on an unknown `task_id` so a typo never +/// writes an orphan event. +fn require_task( + project_hash: &str, + events_path: &Path, + state_path: &Path, + task_id: &str, +) -> anyhow::Result<()> { + let conn_arc = cached_open(state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + } + if !tj_core::db::task_exists(&conn, task_id)? { + anyhow::bail!("task not found: {task_id}"); + } + + Ok(()) } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -381,6 +455,8 @@ impl TaskJournalServer { metadata: TaskPackMetadata { source_event_count: Some(pack.metadata.source_event_count), cache_hit: Some(pack.metadata.cache_hit), + generated_at: Some(pack.metadata.generated_at), + truncated: Some(pack.metadata.truncated), }, }) }) @@ -413,10 +489,7 @@ impl TaskJournalServer { &p.task_id, tj_core::completeness::pending_count(), )?; - let dir = match PROJECT_DIR_OVERRIDE.get() { - Some(d) => d.clone(), - None => std::env::current_dir()?, - }; + let dir = project_dir()?; let arts = tj_core::db::task_artifacts(&conn, &p.task_id)?; report .gaps @@ -449,7 +522,7 @@ impl TaskJournalServer { #[tool( name = "task_search", - description = "Full-text search tasks by query (FTS5)." + description = "Search tasks. `query` is full-text over event text; empty or absent lists the project's tasks, newest first. `status`: open | closed | any (default). `project`: absolute dir of another project. `tasks` gives id, title, status, last_event_at, goal per hit." )] async fn task_search( &self, @@ -459,79 +532,99 @@ impl TaskJournalServer { let query = p.query.clone(); let raw_query = p.query.clone(); let event_type = p.event_type.clone(); - let results = run_blocking(move || { - let (project_hash, events_path, state_path) = project_paths()?; + let tasks = run_blocking(move || { + let status = match p.status.as_deref() { + None | Some("any") => None, + Some(s @ ("open" | "closed")) => Some(s.to_string()), + Some(other) => { + anyhow::bail!("invalid status `{other}` (expected: open | closed | any)") + } + }; + let (project_hash, events_path, state_path) = match p.project.as_deref() { + Some(dir) if !Path::new(dir).is_absolute() => { + anyhow::bail!("project must be an absolute path, got `{dir}`") + } + Some(dir) => resolve_project_paths(Path::new(dir))?, + None => project_paths()?, + }; + // No journal, no tasks — and no empty state DB left behind to + // show up in project lists later. + if !events_path.exists() { + return Ok(Vec::new()); + } + let conn_arc = cached_open(&state_path)?; let conn = conn_arc .lock() .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + + // No query: list the project's tasks, newest first. FTS5 + // rejects an empty MATCH, so it must not reach it. + if raw_query.trim().is_empty() { + let mut stmt = conn.prepare( + "SELECT task_id, title, status, last_event_at, goal FROM tasks \ + WHERE (?1 IS NULL OR status = ?1) \ + AND (?2 IS NULL OR task_id IN \ + (SELECT task_id FROM events_index WHERE type = ?2)) \ + ORDER BY last_event_at DESC, rowid DESC LIMIT 50", + )?; + let hits = stmt + .query_map(rusqlite::params![status, event_type], task_search_hit)? + .collect::<Result<_, _>>()?; + + return Ok(hits); } // v0.10.3: sanitize FTS5 query. Hyphenated IDs like // `OPS-306` previously crashed with "no such column: 306" - // because FTS5 reads `-` as column-prefix syntax. Wrap - // such queries in phrase quotes; safe queries pass - // through unchanged so AND semantics are preserved. + // because FTS5 reads `-` as column-prefix syntax. Every + // token is quoted on its own, so punctuation is literal + // and multi-word queries keep their AND semantics. let fts_query = tj_core::fts::sanitize_query(&raw_query); - let (sql, fts_only) = match &event_type { - Some(_) => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE search_fts MATCH ?1 AND type = ?2 LIMIT 50", - false, - ), - None => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE search_fts MATCH ?1 LIMIT 50", - true, - ), - }; - let mut stmt = conn.prepare(sql)?; - let mut ids: Vec<String> = if fts_only { - stmt.query_map(rusqlite::params![fts_query], |r| r.get::<_, String>(0))? - .collect::<Result<_, _>>()? - } else { - let ty = event_type.as_deref().unwrap(); - stmt.query_map(rusqlite::params![fts_query, ty], |r| r.get::<_, String>(0))? - .collect::<Result<_, _>>()? - }; + let mut stmt = conn.prepare( + "SELECT DISTINCT t.task_id, t.title, t.status, t.last_event_at, t.goal \ + FROM search_fts JOIN tasks t ON t.task_id = search_fts.task_id \ + WHERE search_fts MATCH ?1 \ + AND (?2 IS NULL OR search_fts.type = ?2) \ + AND (?3 IS NULL OR t.status = ?3) LIMIT 50", + )?; + let mut hits: Vec<TaskSearchHit> = stmt + .query_map( + rusqlite::params![fts_query, event_type, status], + task_search_hit, + )? + .collect::<Result<_, _>>()?; // v0.10.3: LIKE fallback. FTS5 phrase search miss when // tokenizer split differs from the user's mental model // (e.g. `bulk-repack` in source vs `bulk repack` in // query). On zero FTS hits, scan event text directly so // hyphenated identifiers and partial-word recall work. - if ids.is_empty() { + if hits.is_empty() { let like = tj_core::fts::like_pattern(&raw_query); - let (sql_like, type_bind) = match &event_type { - Some(_) => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE text LIKE ?1 AND type = ?2 LIMIT 50", - true, - ), - None => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE text LIKE ?1 LIMIT 50", - false, - ), - }; - let mut stmt_like = conn.prepare(sql_like)?; - ids = if type_bind { - let ty = event_type.as_deref().unwrap(); - stmt_like - .query_map(rusqlite::params![like, ty], |r| r.get::<_, String>(0))? - .collect::<Result<_, _>>()? - } else { - stmt_like - .query_map(rusqlite::params![like], |r| r.get::<_, String>(0))? - .collect::<Result<_, _>>()? - }; + let mut stmt_like = conn.prepare( + "SELECT DISTINCT t.task_id, t.title, t.status, t.last_event_at, t.goal \ + FROM search_fts JOIN tasks t ON t.task_id = search_fts.task_id \ + WHERE search_fts.text LIKE ?1 \ + AND (?2 IS NULL OR search_fts.type = ?2) \ + AND (?3 IS NULL OR t.status = ?3) LIMIT 50", + )?; + hits = stmt_like + .query_map(rusqlite::params![like, event_type, status], task_search_hit)? + .collect::<Result<_, _>>()?; } - Ok(ids) + + Ok(hits) }) .await?; - Ok(Json(TaskSearchResult { query, results })) + + let results = tasks.iter().map(|t| t.task_id.clone()).collect(); + Ok(Json(TaskSearchResult { + query, + results, + tasks, + })) }) .await } @@ -602,39 +695,24 @@ impl TaskJournalServer { if let Some(ref parent_id) = p.parent { event.meta["parent_id"] = serde_json::Value::String(parent_id.clone()); } + // Goal and the Loom tag ride in the open event, so ingest (and + // a rebuild from the JSONL) restores them; later resolves and + // the board → journal link find the journal by `loom:<id>`. + if let Some(ref goal) = p.goal { + event.meta["goal"] = serde_json::Value::String(goal.clone()); + } + if let Some(ref r) = loom_ref { + event.meta["external"] = serde_json::json!([r]); + } tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; - // v0.6.0: persist goal column when caller passed --goal / - // params.goal. We must ingest into SQLite first so the - // task row exists; without ingestion set_task_goal hits - // an empty tasks table and silently no-ops. - if let Some(goal) = p.goal.as_deref() { - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::set_task_goal(&conn, &task_id, goal)?; - } - - // Tag the new journal with its Loom task id so later resolves - // (and the board → journal link) find it. - if let Some(ref r) = loom_ref { - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::add_task_external(&conn, &task_id, r)?; - } - Ok(TaskCreateResult { task_id, title: p.title.clone(), @@ -656,7 +734,7 @@ impl TaskJournalServer { ) -> Result<Json<EventAddResult>, McpError> { traced_tool("event_add", async move { run_blocking(move || { - let (_, events_path, _) = project_paths()?; + let (project_hash, events_path, state_path) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; let event_type = parse_event_type(&p.event_type)?; @@ -669,6 +747,8 @@ impl TaskJournalServer { p.event_type ); } + require_task(&project_hash, &events_path, &state_path, &p.task_id)?; + let mut event = tj_core::event::Event::new( &p.task_id, event_type, @@ -685,7 +765,7 @@ impl TaskJournalServer { } tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -714,8 +794,9 @@ impl TaskJournalServer { ) -> Result<Json<ArtifactAddResult>, McpError> { traced_tool("artifact_add", async move { run_blocking(move || { - let (_, events_path, _) = project_paths()?; + let (project_hash, events_path, state_path) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; + require_task(&project_hash, &events_path, &state_path, &p.task_id)?; let mut event = tj_core::event::Event::new( &p.task_id, @@ -727,7 +808,7 @@ impl TaskJournalServer { event.meta = tj_core::artifacts::link_event_meta(&p.kind, &p.url, &p.label); tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -770,10 +851,11 @@ impl TaskJournalServer { if !tj_core::db::task_exists(&conn, &p.task_id)? { anyhow::bail!("task not found: {}", p.task_id); } - // v0.6.0: validate outcome_tag enum and persist - // outcome+tag to the task row before writing the - // close event. Same enum + same ordering as the - // CLI close handler — keep them lockstep. + // v0.6.0: validate the outcome_tag enum before writing + // the close event (same enum as the CLI close handler). + // outcome+tag ride in the close event's meta and reach + // the task row only when that event is ingested, so a + // failed append never leaves an open task with an outcome. if let Some(tag) = p.outcome_tag.as_deref() { match tag { "done" | "abandoned" | "superseded" => {} @@ -782,9 +864,6 @@ impl TaskJournalServer { ), } } - if let Some(o) = p.outcome.as_deref() { - tj_core::db::set_task_outcome(&conn, &p.task_id, o, p.outcome_tag.as_deref())?; - } open_kids = tj_core::db::count_open_children(&conn, &p.task_id)?; } // release the connection lock before doing the JSONL append @@ -807,7 +886,7 @@ impl TaskJournalServer { // (commit, branch, PR) into the close event so the resume pack // reads as a clickable ledger of what shipped. Best-effort and // structured (merged in db::index_event) — never fails close. - if let Ok(dir) = std::env::current_dir() { + if let Ok(dir) = project_dir() { let arts = tj_core::harvest::harvest(&dir); if !arts.is_empty() { if let Ok(v) = serde_json::to_value(&arts) { @@ -818,7 +897,7 @@ impl TaskJournalServer { event.meta = serde_json::Value::Object(meta); tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -882,7 +961,8 @@ impl TaskJournalServer { } } -#[tool_handler(router = Self::tool_router())] +// Written out instead of `#[tool_handler]`, which generates the same two +// methods, so `call_tool` can read the request's `_meta` first. impl ServerHandler for TaskJournalServer { fn get_info(&self) -> rmcp::model::ServerInfo { rmcp::model::ServerInfo { @@ -897,6 +977,49 @@ impl ServerHandler for TaskJournalServer { ..Default::default() } } + + async fn call_tool( + &self, + mut request: rmcp::model::CallToolRequestParam, + context: rmcp::service::RequestContext<rmcp::RoleServer>, + ) -> Result<rmcp::model::CallToolResult, McpError> { + stamp_client_session(&mut request, &context.meta); + + let tcc = rmcp::handler::server::tool::ToolCallContext::new(self, request, context); + Self::tool_router().call(tcc).await + } + + async fn list_tools( + &self, + _request: Option<rmcp::model::PaginatedRequestParam>, + _context: rmcp::service::RequestContext<rmcp::RoleServer>, + ) -> Result<rmcp::model::ListToolsResult, McpError> { + Ok(rmcp::model::ListToolsResult::with_all_items( + Self::tool_router().list_all(), + )) + } +} + +/// Codex names the session in each call's `_meta`, never in the MCP +/// server's environment. Carry it into the `session_id` argument of the +/// tools that stamp one, unless the caller passed its own. +fn stamp_client_session(request: &mut rmcp::model::CallToolRequestParam, meta: &rmcp::model::Meta) { + const STAMPING: [&str; 4] = ["task_create", "event_add", "artifact_add", "task_close"]; + if !STAMPING.contains(&request.name.as_ref()) { + return; + } + let Some(session) = tj_core::session_id::session_id_from_mcp_meta(&meta.0) else { + return; + }; + + let args = request.arguments.get_or_insert_with(Default::default); + let has_own = args + .get("session_id") + .and_then(|v| v.as_str()) + .is_some_and(|s| !s.is_empty()); + if !has_own { + args.insert("session_id".into(), serde_json::Value::String(session)); + } } /// Resolve when the process should shut down: Ctrl-C on every platform, @@ -969,6 +1092,44 @@ mod tests { use super::*; + fn call(name: &'static str, args: serde_json::Value) -> rmcp::model::CallToolRequestParam { + rmcp::model::CallToolRequestParam { + name: name.into(), + arguments: args.as_object().cloned(), + } + } + + fn codex_meta(session: &str) -> rmcp::model::Meta { + let meta = serde_json::json!({ "x-codex-turn-metadata": { "session_id": session } }); + rmcp::model::Meta(meta.as_object().unwrap().clone()) + } + + #[test] + fn codex_session_from_meta_reaches_the_stamping_tools() { + let mut add = call("event_add", serde_json::json!({ "task_id": "tj-1" })); + stamp_client_session(&mut add, &codex_meta("c-1")); + assert_eq!(add.arguments.unwrap()["session_id"], "c-1"); + + let mut create = call("task_create", serde_json::json!(null)); + stamp_client_session(&mut create, &codex_meta("c-1")); + assert_eq!(create.arguments.unwrap()["session_id"], "c-1"); + } + + #[test] + fn an_explicit_session_id_and_other_tools_are_left_alone() { + let mut own = call("event_add", serde_json::json!({ "session_id": "mine" })); + stamp_client_session(&mut own, &codex_meta("c-1")); + assert_eq!(own.arguments.unwrap()["session_id"], "mine"); + + let mut search = call("task_search", serde_json::json!({ "query": "x" })); + stamp_client_session(&mut search, &codex_meta("c-1")); + assert!(search.arguments.unwrap().get("session_id").is_none()); + + let mut claude = call("event_add", serde_json::json!({ "task_id": "tj-1" })); + stamp_client_session(&mut claude, &rmcp::model::Meta::default()); + assert!(claude.arguments.unwrap().get("session_id").is_none()); + } + /// Handler tests touch process-global state (PROJECT_DIR_OVERRIDE OnceLock /// and the XDG_DATA_HOME env var), so they must run one at a time and share /// a single project dir. This guard serializes them and lazily pins the @@ -1021,6 +1182,8 @@ mod tests { metadata: TaskPackMetadata { source_event_count: None, cache_hit: None, + generated_at: None, + truncated: None, }, }; let pack_v = serde_json::to_value(&pack).unwrap(); @@ -1030,6 +1193,7 @@ mod tests { let search = TaskSearchResult { query: "q".into(), results: vec![], + tasks: vec![], }; assert!(!keys_of(&serde_json::to_value(&search).unwrap()).contains(&"stub".to_string())); @@ -1204,6 +1368,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1222,6 +1387,7 @@ mod tests { corrects: None, supersedes: None, alternatives: Some(alts.clone()), + session_id: None, })) .await .unwrap() @@ -1248,6 +1414,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1262,6 +1429,7 @@ mod tests { corrects: None, supersedes: None, alternatives: Some(serde_json::json!([{"option": "x", "chosen": true}])), + session_id: None, })) .await; let err = match res { @@ -1290,6 +1458,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1302,6 +1471,7 @@ mod tests { initial_context: None, goal: None, parent: Some(parent.clone()), + session_id: None, })) .await .unwrap() @@ -1332,6 +1502,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1345,6 +1516,7 @@ mod tests { initial_context: None, goal: None, parent: Some(parent.clone()), + session_id: None, })) .await .unwrap(); @@ -1355,6 +1527,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap() @@ -1362,6 +1535,42 @@ mod tests { assert_eq!(res.note.as_deref(), Some("note: 1 open subtask(s)")); } + #[tokio::test] + async fn task_pack_reports_generated_at_and_truncated() { + let _env = handler_env(); + let server = TaskJournalServer; + let task = server + .task_create(Parameters(TaskCreateParams { + title: "Pack metadata".into(), + initial_context: None, + goal: Some("g".into()), + parent: None, + session_id: None, + })) + .await + .unwrap() + .0 + .task_id; + + let pack = server + .task_pack(Parameters(TaskPackParams { + task_id: task, + mode: None, + })) + .await + .unwrap() + .0; + + let meta = serde_json::to_value(&pack.metadata).unwrap(); + assert!( + meta["generated_at"].as_str().is_some_and(|s| !s.is_empty()), + "{meta}" + ); + assert_eq!(meta["truncated"], false, "{meta}"); + assert!(meta["cache_hit"].is_boolean(), "{meta}"); + assert!(meta["source_event_count"].is_number(), "{meta}"); + } + #[tokio::test] async fn task_close_reports_completeness_gaps() { let _env = handler_env(); @@ -1374,6 +1583,7 @@ mod tests { initial_context: None, goal: Some("ship it".into()), parent: None, + session_id: None, })) .await .unwrap() @@ -1387,6 +1597,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap() @@ -1413,6 +1624,7 @@ mod tests { initial_context: None, goal: Some("ship it".into()), parent: None, + session_id: None, })) .await .unwrap() @@ -1428,6 +1640,7 @@ mod tests { corrects: None, supersedes: None, alternatives: None, + session_id: None, })) .await .unwrap(); @@ -1450,6 +1663,478 @@ mod tests { ); } + async fn create_task(server: &TaskJournalServer, title: &str) -> String { + server + .task_create(Parameters(TaskCreateParams { + title: title.into(), + initial_context: None, + goal: Some(format!("goal of {title}")), + parent: None, + session_id: None, + })) + .await + .unwrap() + .0 + .task_id + } + + async fn search( + server: &TaskJournalServer, + query: &str, + status: Option<&str>, + project: Option<&str>, + event_type: Option<&str>, + ) -> Result<TaskSearchResult, McpError> { + server + .task_search(Parameters(TaskSearchParams { + query: query.into(), + status: status.map(Into::into), + project: project.map(Into::into), + event_type: event_type.map(Into::into), + })) + .await + .map(|j| j.0) + } + + #[test] + fn task_search_params_accept_a_missing_query() { + let p: TaskSearchParams = + serde_json::from_value(serde_json::json!({"status": "open"})).unwrap(); + assert_eq!(p.query, ""); + } + + #[tokio::test] + async fn task_search_empty_query_lists_tasks_filtered_by_status() { + let _env = handler_env(); + let server = TaskJournalServer; + + let open = create_task(&server, "Empty query open").await; + let closed = create_task(&server, "Empty query closed").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: closed.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + session_id: None, + })) + .await + .unwrap(); + + let res = search(&server, " ", Some("open"), None, None) + .await + .unwrap(); + assert!(res.results.contains(&open), "{:?}", res.results); + assert!(!res.results.contains(&closed), "{:?}", res.results); + let ids: Vec<_> = res.tasks.iter().map(|t| t.task_id.clone()).collect(); + assert_eq!(ids, res.results, "`tasks` must follow `results` order"); + let hit = res.tasks.iter().find(|t| t.task_id == open).unwrap(); + assert_eq!(hit.title, "Empty query open"); + assert_eq!(hit.status, "open"); + assert_eq!(hit.goal.as_deref(), Some("goal of Empty query open")); + + let res = search(&server, "", Some("closed"), None, None) + .await + .unwrap(); + assert!(res.results.contains(&closed) && !res.results.contains(&open)); + + let res = search(&server, "", None, None, None).await.unwrap(); + assert!(res.results.contains(&closed) && res.results.contains(&open)); + assert!( + res.tasks + .windows(2) + .all(|w| w[0].last_event_at >= w[1].last_event_at), + "newest last_event_at first" + ); + + let res = search(&server, "", Some("any"), None, Some("close")) + .await + .unwrap(); + assert!(res.results.contains(&closed) && !res.results.contains(&open)); + } + + #[tokio::test] + async fn task_search_status_filters_fts_hits_and_rejects_unknown_values() { + let _env = handler_env(); + let server = TaskJournalServer; + + let open = create_task(&server, "Quagga open").await; + let closed = create_task(&server, "Quagga closed").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: closed.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + session_id: None, + })) + .await + .unwrap(); + + let res = search(&server, "quagga", Some("open"), None, None) + .await + .unwrap(); + assert_eq!(res.results, vec![open.clone()]); + assert_eq!(res.tasks[0].title, "Quagga open"); + + let res = search(&server, "quagga", Some("closed"), None, None) + .await + .unwrap(); + assert_eq!(res.results, vec![closed.clone()]); + + let res = search(&server, "quagga", None, None, None).await.unwrap(); + assert_eq!(res.results.len(), 2); + + let err = search(&server, "quagga", Some("pending"), None, None) + .await + .expect_err("unknown status must be rejected"); + assert!(err.message.contains("status"), "{}", err.message); + } + + #[tokio::test] + async fn task_search_project_searches_that_projects_journal() { + let _env = handler_env(); + let server = TaskJournalServer; + + let other = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(other.path().join(".git")).unwrap(); + let (_, other_events, _) = resolve_project_paths(other.path()).unwrap(); + std::fs::create_dir_all(other_events.parent().unwrap()).unwrap(); + let task_id = tj_core::new_task_id(); + let mut ev = tj_core::event::Event::new( + task_id.clone(), + tj_core::event::EventType::Open, + tj_core::event::Author::Agent, + tj_core::event::Source::Chat, + "Elsewhere quokka".into(), + ); + ev.meta = serde_json::json!({"title": "Elsewhere quokka"}); + let mut writer = tj_core::storage::JsonlWriter::open(&other_events).unwrap(); + writer.append(&ev).unwrap(); + writer.flush_durable().unwrap(); + let other_dir = other.path().to_str().unwrap(); + + let res = search(&server, "quokka", None, Some(other_dir), None) + .await + .unwrap(); + assert_eq!(res.results, vec![task_id.clone()]); + + let res = search(&server, "", Some("open"), Some(other_dir), None) + .await + .unwrap(); + assert_eq!(res.results, vec![task_id.clone()]); + + let res = search(&server, "quokka", None, None, None).await.unwrap(); + assert!(!res.results.contains(&task_id), "{:?}", res.results); + + assert!(search(&server, "", None, Some("relative/dir"), None) + .await + .is_err()); + } + + #[tokio::test] + async fn task_close_harvests_the_project_dir_not_the_cwd() { + let _env = handler_env(); + let server = TaskJournalServer; + + // The test binary runs inside this repo's checkout, while the project + // dir is a temp dir. A close must never stamp the cwd repo's commit. + let cwd_commit = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()); + let Some(cwd_commit) = cwd_commit else { + return; // not run from a git checkout: nothing to tell apart + }; + + let task = create_task(&server, "Harvest dir").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + session_id: None, + })) + .await + .unwrap(); + + let (_, events_path, _) = project_paths().unwrap(); + let jsonl = std::fs::read_to_string(&events_path).unwrap(); + let close = jsonl + .lines() + .map(|l| serde_json::from_str::<serde_json::Value>(l).unwrap()) + .find(|v| v["task_id"] == task.as_str() && v["type"] == "close") + .expect("close event in jsonl"); + assert!( + !close["meta"].to_string().contains(&cwd_commit), + "close harvested the cwd repo ({cwd_commit}): {}", + close["meta"] + ); + } + + #[tokio::test] + async fn task_close_records_outcome_only_through_the_close_event() { + let _env = handler_env(); + let server = TaskJournalServer; + + let task = create_task(&server, "Append fails").await; + let close = |outcome: &str| TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: Some(outcome.into()), + outcome_tag: Some("done".into()), + session_id: None, + }; + + // Make the close append fail: the journal is read-only. + let (_, events_path, state_path) = project_paths().unwrap(); + let writable = std::fs::metadata(&events_path).unwrap().permissions(); + let mut read_only = writable.clone(); + read_only.set_readonly(true); + std::fs::set_permissions(&events_path, read_only).unwrap(); + let res = server.task_close(Parameters(close("must not stick"))).await; + std::fs::set_permissions(&events_path, writable).unwrap(); + assert!(res.is_err(), "append to a read-only journal must fail"); + + let conn_arc = cached_open(&state_path).unwrap(); + { + let conn = conn_arc.lock().unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.outcome, None, "failed close left an outcome behind"); + let status = tj_core::db::task_status(&conn, &task).unwrap(); + assert_eq!(status.as_deref(), Some("open")); + } + + server + .task_close(Parameters(close("shipped"))) + .await + .unwrap(); + let conn = conn_arc.lock().unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.outcome.as_deref(), Some("shipped")); + assert_eq!(meta.outcome_tag.as_deref(), Some("done")); + } + + #[tokio::test] + async fn event_add_and_artifact_add_reject_an_unknown_task() { + let _env = handler_env(); + let server = TaskJournalServer; + + create_task(&server, "Known task").await; + let (_, events_path, _) = project_paths().unwrap(); + let before = std::fs::read_to_string(&events_path).unwrap(); + + let res = server + .event_add(Parameters(EventAddParams { + task_id: "tj-typo000000".into(), + event_type: "finding".into(), + text: "orphan".into(), + corrects: None, + supersedes: None, + alternatives: None, + session_id: None, + })) + .await; + let err = match res { + Ok(_) => panic!("event_add on an unknown task must fail"), + Err(e) => e, + }; + assert!( + err.message.contains("task not found: tj-typo000000"), + "{}", + err.message + ); + + let res = server + .artifact_add(Parameters(ArtifactAddParams { + task_id: "tj-typo000000".into(), + kind: "doc".into(), + url: "https://example.com/spec".into(), + label: "Spec".into(), + session_id: None, + })) + .await; + let err = match res { + Ok(_) => panic!("artifact_add on an unknown task must fail"), + Err(e) => e, + }; + assert!( + err.message.contains("task not found: tj-typo000000"), + "{}", + err.message + ); + + let after = std::fs::read_to_string(&events_path).unwrap(); + assert_eq!(after, before, "no orphan event may reach the journal"); + } + + #[tokio::test] + async fn an_explicit_session_id_is_stamped_instead_of_the_env_one() { + let _env = handler_env(); + let server = TaskJournalServer; + let prev = std::env::var("CLAUDE_CODE_SESSION_ID").ok(); + std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); + + let task = server + .task_create(Parameters(TaskCreateParams { + title: "Session task".into(), + initial_context: None, + goal: None, + parent: None, + session_id: Some("s-create".into()), + })) + .await + .unwrap() + .0 + .task_id; + let event = |session_id: Option<&str>| EventAddParams { + task_id: task.clone(), + event_type: "finding".into(), + text: "seen".into(), + corrects: None, + supersedes: None, + alternatives: None, + session_id: session_id.map(Into::into), + }; + let explicit = server.event_add(Parameters(event(Some("s-event")))).await; + let from_env = server.event_add(Parameters(event(None))).await; + let artifact = server + .artifact_add(Parameters(ArtifactAddParams { + task_id: task.clone(), + kind: "doc".into(), + url: "https://example.com/doc".into(), + label: "Doc".into(), + session_id: Some("s-artifact".into()), + })) + .await; + let closed = server + .task_close(Parameters(TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + session_id: Some("s-close".into()), + })) + .await; + match prev { + Some(v) => std::env::set_var("CLAUDE_CODE_SESSION_ID", v), + None => std::env::remove_var("CLAUDE_CODE_SESSION_ID"), + } + let (explicit, from_env, artifact) = ( + explicit.unwrap().0.event_id, + from_env.unwrap().0.event_id, + artifact.unwrap().0.event_id, + ); + closed.unwrap(); + + let (_, events_path, _) = project_paths().unwrap(); + let session_of = |pick: &dyn Fn(&serde_json::Value) -> bool| { + std::fs::read_to_string(&events_path) + .unwrap() + .lines() + .map(|l| serde_json::from_str::<serde_json::Value>(l).unwrap()) + .find(|v| pick(v)) + .and_then(|v| v["meta"]["session_id"].as_str().map(str::to_string)) + }; + let by_id = |id: &str| session_of(&|v| v["event_id"] == id); + let by_type = |ty: &str| session_of(&|v| v["task_id"] == task.as_str() && v["type"] == ty); + assert_eq!(by_type("open").as_deref(), Some("s-create")); + assert_eq!(by_id(&explicit).as_deref(), Some("s-event")); + assert_eq!(by_id(&from_env).as_deref(), Some("from-env")); + assert_eq!(by_id(&artifact).as_deref(), Some("s-artifact")); + assert_eq!(by_type("close").as_deref(), Some("s-close")); + } + + #[test] + fn session_id_params_are_optional() { + let p: EventAddParams = serde_json::from_value(serde_json::json!({ + "task_id": "tj-x", "event_type": "finding", "text": "t" + })) + .unwrap(); + assert_eq!(p.session_id, None); + } + + #[tokio::test] + async fn task_search_of_a_project_without_a_journal_creates_no_state_db() { + let _env = handler_env(); + let server = TaskJournalServer; + + let other = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(other.path().join(".git")).unwrap(); + let (_, _, other_state) = resolve_project_paths(other.path()).unwrap(); + let other_dir = other.path().to_str().unwrap(); + + for query in ["", "anything"] { + let res = search(&server, query, None, Some(other_dir), None) + .await + .unwrap(); + assert!(res.results.is_empty(), "{:?}", res.results); + } + assert!(!other_state.exists(), "created {other_state:?}"); + } + + #[tokio::test] + async fn task_create_goal_and_loom_ref_survive_a_state_rebuild() { + let _env = handler_env(); + let server = TaskJournalServer; + + let loom_id = format!("t-rebuild-{}", ulid::Ulid::new()); + std::env::set_var("LOOM_TASK_ID", &loom_id); + let created = server + .task_create(Parameters(TaskCreateParams { + title: "Loom task".into(), + initial_context: None, + goal: Some("Wire the board".into()), + parent: None, + session_id: None, + })) + .await; + std::env::remove_var("LOOM_TASK_ID"); + let task = created.unwrap().0.task_id; + + // A fresh SQLite rebuilt from the journal alone. + let (project_hash, events_path, _) = project_paths().unwrap(); + let dir = tempfile::TempDir::new().unwrap(); + let conn = tj_core::db::open(dir.path().join("fresh.sqlite")).unwrap(); + tj_core::db::rebuild_state(&conn, &events_path, &project_hash).unwrap(); + + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("Wire the board")); + let by_loom = tj_core::db::task_id_by_external(&conn, &format!("loom:{loom_id}")).unwrap(); + assert_eq!(by_loom.as_deref(), Some(task.as_str())); + } + + #[tokio::test] + async fn event_add_rejects_the_internal_amend_type() { + let _env = handler_env(); + let server = TaskJournalServer; + + let task = create_task(&server, "No amend").await; + let res = server + .event_add(Parameters(EventAddParams { + task_id: task, + event_type: "amend".into(), + text: "goal: sneaky".into(), + corrects: None, + supersedes: None, + alternatives: None, + session_id: None, + })) + .await; + let err = match res { + Ok(_) => panic!("amend is internal and must be rejected"), + Err(e) => e, + }; + assert!( + err.message.contains("unknown event type"), + "{}", + err.message + ); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index ccd67dd..a2b653e 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "task-journal", - "version": "0.29.0", + "version": "0.30.0", "description": "Append-only journal of AI-coding task reasoning chains: hypotheses, decisions, rejections, evidence. Renders compact resume packs so an agent can pick up a 2-week-old task with full context.", "author": { "name": "Mher Shahinyan" @@ -18,5 +18,21 @@ "command": "task-journal-mcp" } }, - "skills": "./skills/" + "skills": "./skills/", + "userConfig": { + "nudge_after_turns": { + "type": "number", + "title": "Remind after turns", + "description": "With the mod on (Claude Code 2.1.287+): remind the agent to log after this many of your prompts without a journal entry. 0 turns the reminder off.", + "default": 6, + "min": 0, + "max": 50 + }, + "distill_on_compact": { + "type": "boolean", + "title": "Catch up before compaction", + "description": "With the mod on: before the conversation is compacted, ask the model which decisions, rejections and findings were not logged, and record them as suggested events. One extra model call per compaction, mostly served from the prompt cache.", + "default": true + } + } } diff --git a/plugin/hooks/hooks.json b/plugin/hooks/hooks.json new file mode 100644 index 0000000..b614c24 --- /dev/null +++ b/plugin/hooks/hooks.json @@ -0,0 +1,4 @@ +{ + "modules": ["./register.ts"], + "hooks": {} +} diff --git a/plugin/hooks/journal.test.ts b/plugin/hooks/journal.test.ts new file mode 100644 index 0000000..1c269b6 --- /dev/null +++ b/plugin/hooks/journal.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from 'claude-code/testing' + +import { + journalTool, + nudgeText, + parseDistill, + parseState, + sectionText, + statusText, + transcriptExcerpt, + type JournalState, +} from './journal' + +const ACTIVE = JSON.stringify({ + schema: 'tj-state/1', + open_tasks: 2, + active: { + task_id: 'tj-abc', + title: 'Fix refresh', + goal: 'Stop dropping the token', + counts: { decision: 3, rejection: 1, evidence: 0 }, + recent: [{ type: 'decision', text: 'Use <= on expiry' }], + }, +}) + +describe('parseState', () => { + test('reads the active task', () => { + const s = parseState(ACTIVE) + expect(s?.active?.task_id).toBe('tj-abc') + expect(s?.active?.recent.length).toBe(1) + expect(s?.open_tasks).toBe(2) + }) + + test('no active task is a state, not a failure', () => { + const s = parseState(JSON.stringify({ schema: 'tj-state/1', open_tasks: 0, active: null })) + expect(s).toEqual({ active: null, open_tasks: 0 }) + }) + + test('rejects garbage and other schemas', () => { + expect(parseState('error: unrecognized subcommand')).toBe(null) + expect(parseState(JSON.stringify({ schema: 'tj-state/9', active: null }))).toBe(null) + }) + + test('an empty goal reads as not set', () => { + const s = parseState(JSON.stringify({ schema: 'tj-state/1', open_tasks: 1, active: { task_id: 'tj-x', goal: ' ' } })) + expect(s?.active?.goal).toBe(null) + expect(s?.active?.title).toBe('tj-x') + }) +}) + +describe('texts', () => { + const s = parseState(ACTIVE) as JournalState + const none: JournalState = { active: null, open_tasks: 3 } + + test('status line names the task and its counts', () => { + expect(statusText(s)).toBe('📓 tj-abc · 3 decisions · 1 rejected · 0 evidence') + expect(statusText(none)).toBe('📓 no task in this session · 3 open') + expect(statusText({ active: null, open_tasks: 0 })).toBe('📓 no task') + }) + + test('the prompt section holds nothing that changes per entry', () => { + const text = sectionText(s) + expect(text).toContain('tj-abc') + expect(text).toContain('Stop dropping the token') + expect(text).not.toContain('3 decisions') + expect(text).not.toContain('Use <= on expiry') + }) + + test('nudge names the task, or asks to open one', () => { + expect(nudgeText(s, 6)).toContain('6 turns since the last journal entry on tj-abc') + expect(nudgeText(none, 6)).toContain('task_create') + }) +}) + +describe('journalTool', () => { + test('matches the plugin server and a hand-wired one', () => { + expect(journalTool('mcp__plugin_task-journal_task-journal__event_add')).toBe('event_add') + expect(journalTool('mcp__task-journal__task_close')).toBe('task_close') + expect(journalTool('mcp__other__event_add')).toBe(undefined) + expect(journalTool('Bash')).toBe(undefined) + }) +}) + +describe('parseDistill', () => { + const known = [{ type: 'decision', text: 'Use <= on expiry' }] + + test('keeps typed, new events from a fenced reply', () => { + const reply = 'Here:\n```json\n[{"type":"rejection","text":"Ruled out a cache: stale tokens"},{"type":"finding","text":"refresh.rs:42 uses <"}]\n```' + expect(parseDistill(reply, known)).toEqual([ + { type: 'rejection', text: 'Ruled out a cache: stale tokens' }, + { type: 'finding', text: 'refresh.rs:42 uses <' }, + ]) + }) + + test('drops known, duplicate, untyped and empty entries', () => { + const reply = JSON.stringify([ + { type: 'decision', text: ' use <= ON expiry ' }, + { type: 'open', text: 'not a distill type' }, + { type: 'finding', text: '' }, + { type: 'finding', text: 'A' }, + { type: 'finding', text: 'a' }, + 'junk', + ]) + expect(parseDistill(reply, known)).toEqual([{ type: 'finding', text: 'A' }]) + }) + + test('caps the count and the length', () => { + const many = Array.from({ length: 20 }, (_, i) => ({ type: 'finding', text: `f${i} ${'x'.repeat(600)}` })) + const out = parseDistill(JSON.stringify(many), []) + expect(out.length).toBe(8) + expect(out[0]?.text.length).toBe(500) + }) + + test('no array, no events', () => { + expect(parseDistill('Nothing to add.', known)).toEqual([]) + expect(parseDistill('[not json', known)).toEqual([]) + }) +}) + +describe('transcriptExcerpt', () => { + const messages = [ + { role: 'user', text: 'first question' }, + { role: 'assistant', text: '' }, + { role: 'assistant', text: 'first answer' }, + { role: 'user', text: 'second question' }, + ] + + test('keeps the turns in order and skips empty ones', () => { + expect(transcriptExcerpt(messages, 1000)).toBe('user: first question\nassistant: first answer\nuser: second question') + }) + + test('keeps the newest turns when it has to cut', () => { + expect(transcriptExcerpt(messages, 50)).toBe('assistant: first answer\nuser: second question') + }) + + test('cuts a single huge turn from its start, keeping its end', () => { + const out = transcriptExcerpt([{ role: 'user', text: 'x'.repeat(100) + 'END' }], 20) + expect(out.length).toBe(20) + expect(out.endsWith('END')).toBe(true) + }) +}) diff --git a/plugin/hooks/journal.ts b/plugin/hooks/journal.ts new file mode 100644 index 0000000..64f7b9f --- /dev/null +++ b/plugin/hooks/journal.ts @@ -0,0 +1,201 @@ +// Pure helpers of the Task Journal mod: everything here is plain data in, +// plain data out, so journal.test.ts covers it without an engine. + +export type JournalEvent = { type: string; text: string } + +export type ActiveTask = { + task_id: string + title: string + goal: string | null + counts: Record<string, number> + recent: JournalEvent[] +} + +export type JournalState = { + active: ActiveTask | null + open_tasks: number +} + +export const STATE_SCHEMA = 'tj-state/1' + +const DISTILL_TYPES = new Set(['decision', 'rejection', 'finding', 'evidence', 'constraint', 'hypothesis']) + +const MAX_DISTILLED = 8 + +const MAX_EVENT_CHARS = 500 + +// `mcp__plugin_task-journal_task-journal__event_add` when the server comes +// with the plugin, `mcp__task-journal__event_add` when it was wired by hand. +const JOURNAL_TOOL = /^mcp__(?:plugin_task-journal_)?task-journal__([a-z_]+)$/ + +export const WRITE_TOOLS = new Set(['task_create', 'event_add', 'artifact_add', 'task_close']) + +export function journalTool(tool: string): string | undefined { + return JOURNAL_TOOL.exec(tool)?.[1] +} + +/** The `task-journal state --json` output, or null when it is not one. */ +export function parseState(stdout: string): JournalState | null { + let raw: unknown + try { + raw = JSON.parse(stdout) + } catch { + return null + } + + if (typeof raw !== 'object' || raw === null) return null + const v = raw as Record<string, unknown> + if (v.schema !== STATE_SCHEMA) return null + + const open = typeof v.open_tasks === 'number' ? v.open_tasks : 0 + const a = v.active as Record<string, unknown> | null | undefined + if (a === null || a === undefined || typeof a.task_id !== 'string') { + return { active: null, open_tasks: open } + } + + const recent = Array.isArray(a.recent) + ? a.recent.filter( + (r): r is JournalEvent => + typeof r === 'object' && r !== null && typeof r.type === 'string' && typeof r.text === 'string', + ) + : [] + + return { + active: { + task_id: a.task_id, + title: typeof a.title === 'string' ? a.title : a.task_id, + goal: typeof a.goal === 'string' && a.goal.trim() !== '' ? a.goal : null, + counts: (a.counts as Record<string, number>) ?? {}, + recent, + }, + open_tasks: open, + } +} + +export function statusText(state: JournalState): string { + const a = state.active + if (a === null) { + return state.open_tasks > 0 ? `📓 no task in this session · ${state.open_tasks} open` : '📓 no task' + } + + const n = (k: string) => a.counts[k] ?? 0 + + return `📓 ${a.task_id} · ${n('decision')} decisions · ${n('rejection')} rejected · ${n('evidence')} evidence` +} + +/** + * The system-prompt section. It sits before the conversation, so every + * change re-sends the whole conversation uncached: it names only what + * changes rarely (the task, its title and goal), never counts or times. + */ +export function sectionText(state: JournalState): string { + const a = state.active + if (a === null) { + return [ + 'Task Journal: no journal task is open in this session.', + 'When real work starts, resume one with task_search(status="open") or open one with task_create(goal=...).', + ].join('\n') + } + + return [ + `Task Journal: the active task in this session is ${a.task_id} — "${a.title}".`, + `Goal: ${a.goal ?? 'not set yet — pass it when you next call task_create, or ask the user.'}`, + `Record each decision, rejection, finding and test result with event_add(task_id="${a.task_id}") the moment it happens; close it with task_close when the work is done.`, + ].join('\n') +} + +export function nudgeText(state: JournalState, turns: number): string { + const a = state.active + if (a === null) { + return 'Task Journal: work is happening but no journal task is open in this session. Resume one with task_search(status="open") or open one with task_create(goal=...).' + } + + return `Task Journal: ${turns} turns since the last journal entry on ${a.task_id}. If a decision, rejection, finding or test result happened since, record it now with event_add.` +} + +export function compactInstruction(task: ActiveTask): string { + return `Keep the Task Journal task id ${task.task_id} ("${task.title}") and its goal in the summary, so the work can go on logging to it.` +} + +export function distillPrompt(task: ActiveTask): string { + const known = task.recent.map(r => `- [${r.type}] ${r.text}`).join('\n') || '- (nothing yet)' + + return [ + `You are auditing the Task Journal of task ${task.task_id} ("${task.title}").`, + `Goal: ${task.goal ?? 'not set'}.`, + 'Already recorded:', + known, + '', + 'From the conversation above, list the decisions, rejections, findings, evidence, constraints and hypotheses that matter for this task and are NOT already recorded.', + 'Do not restate an entry already recorded in other words, and do not turn a recorded decision into a hypothesis or finding.', + `Each one sentence, specific (names, files, numbers). At most ${MAX_DISTILLED}. Nothing worth adding → [].`, + 'Reply with a JSON array only: [{"type": "decision", "text": "..."}]. Allowed types: decision, rejection, finding, evidence, constraint, hypothesis.', + ].join('\n') +} + +export type TranscriptLine = { role: string; text: string } + +/** + * The newest turns of a conversation as `role: text` lines, cut to + * `maxChars` from the end: what a model reads when it cannot fork the + * session's own cached transcript. + */ +export function transcriptExcerpt(messages: readonly TranscriptLine[], maxChars: number): string { + const lines: string[] = [] + let size = 0 + + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + const text = m?.text.trim() ?? '' + if (m === undefined || text === '') continue + + const line = `${m.role}: ${text}` + if (size + line.length > maxChars) { + if (lines.length === 0) lines.push(line.slice(line.length - maxChars)) + break + } + + lines.push(line) + size += line.length + 1 + } + + return lines.reverse().join('\n') +} + +function normalized(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ').trim() +} + +/** The model's distill reply → the events worth recording, already-known ones dropped. */ +export function parseDistill(reply: string, known: JournalEvent[]): JournalEvent[] { + const start = reply.indexOf('[') + const end = reply.lastIndexOf(']') + if (start === -1 || end <= start) return [] + + let raw: unknown + try { + raw = JSON.parse(reply.slice(start, end + 1)) + } catch { + return [] + } + if (!Array.isArray(raw)) return [] + + const seen = new Set(known.map(k => normalized(k.text))) + const out: JournalEvent[] = [] + + for (const item of raw) { + if (typeof item !== 'object' || item === null) continue + const { type, text } = item as Record<string, unknown> + if (typeof type !== 'string' || typeof text !== 'string') continue + if (!DISTILL_TYPES.has(type)) continue + + const clean = text.trim().slice(0, MAX_EVENT_CHARS) + if (clean === '' || seen.has(normalized(clean))) continue + + seen.add(normalized(clean)) + out.push({ type, text: clean }) + if (out.length === MAX_DISTILLED) break + } + + return out +} diff --git a/plugin/hooks/register.ts b/plugin/hooks/register.ts new file mode 100644 index 0000000..3d8e700 --- /dev/null +++ b/plugin/hooks/register.ts @@ -0,0 +1,304 @@ +// Task Journal mod (Claude Code 2.1.287+). A thin layer over the +// `task-journal` CLI: the Rust core stays the only writer of the journal. +// +// - keeps the session's active task in the system prompt, so a compaction +// cannot lose it, and asks the compaction summary to keep its id; +// - stamps the session id on the journal's MCP calls, so each session has +// its own active task; +// - shows the task in the status line and a toast on every entry; +// - reminds the agent to log only after N turns without an entry; +// - before a compaction, asks the model (from its own prompt cache) what +// was not logged and records it as suggested events. +// +// It sets TJ_MOD_ACTIVE=1 for the hooks Claude Code starts, so the classic +// hooks skip what the mod does here (the reminder, per-message +// classification, the transcript catch-ups) and keep the rest. + +import type { EngineInterface, Register } from 'claude-code' + +import { + compactInstruction, + distillPrompt, + journalTool, + nudgeText, + parseDistill, + parseState, + sectionText, + statusText, + transcriptExcerpt, + WRITE_TOOLS, + type JournalState, + type TranscriptLine, +} from './journal' + +const WORK_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'Bash']) + +// How much of the conversation the no-fork distill reads: the tail that +// fits a small model's prompt comfortably. +const EXCERPT_CHARS = 60_000 + +// The longest a compaction waits for the catch-up before going on without it. +const DISTILL_TIMEOUT_MS = 60_000 + +// After a failed read (not a missing or old CLI), wait this long before +// trying again, so a broken journal costs one spawn a minute, not one a request. +const RETRY_AFTER_MS = 60_000 + +// What the mod knows about the session it runs in. A module variable: it +// lives as long as the process, which is as long as the session. +const mod = { + isOff: false, + hasWarned: false, + // The directory the session started in. The journal's MCP server keeps + // the directory it was started in too, so both resolve the same project + // even after /cd or a worktree move. + cwd: '', + state: null as JournalState | null, + session: '', + retryAt: 0, + // The task the system prompt names. It changes on task_create, on closing + // it, or with the session — never because the agent logged to another + // task — so the prompt (and the cache behind it) stays put. + pinned: null as string | null, + turnsSinceEntry: 0, + workSinceEntry: 0, +} + +type CliResult = + | { kind: 'ok'; stdout: string } + | { kind: 'missing' } + | { kind: 'old' } + | { kind: 'failed'; detail: string } + +async function cli($: EngineInterface, argv: string[]): Promise<CliResult> { + let run + try { + run = await $.process.run(['task-journal', ...argv], { + cwd: mod.cwd || (await $.session.cwd()), + timeoutMs: 30_000, + }) + } catch (err) { + const detail = String(err) + + return /ENOENT|not found|No such file/i.test(detail) ? { kind: 'missing' } : { kind: 'failed', detail } + } + + if (run.exitCode === 0) return { kind: 'ok', stdout: run.stdout } + if (/unrecognized subcommand|unexpected argument/i.test(run.stderr)) return { kind: 'old' } + + return { kind: 'failed', detail: run.stderr.trim().split('\n')[0] ?? `exit ${run.exitCode}` } +} + +function warnOnce($: EngineInterface, result: CliResult): void { + if (mod.hasWarned || result.kind === 'ok') return + mod.hasWarned = true + + const text = + result.kind === 'missing' + ? 'task-journal: the `task-journal` CLI is not on PATH, so the journal mod is off. Install it with `cargo install task-journal-cli task-journal-mcp`.' + : result.kind === 'old' + ? 'task-journal: the installed `task-journal` CLI is older than this plugin (0.30+ needed), so the journal mod is off. Update it with `cargo install task-journal-cli task-journal-mcp --force`.' + : `task-journal: reading the journal failed (${result.detail}); the mod will retry in a minute.` + $.ui.log(text) +} + +// Reads the state of the session the engine names now. A missing or old CLI +// is final for the session; any other failure is retried after a pause. +async function refresh($: EngineInterface): Promise<void> { + const session = await $.session.id() + const prefer = mod.session === session && mod.pinned !== null ? ['--prefer', mod.pinned] : [] + const result = await cli($, ['state', '--session', session, ...prefer]) + const state = result.kind === 'ok' ? parseState(result.stdout) : null + + if (state === null) { + warnOnce($, result) + mod.state = null + if (result.kind === 'failed') { + mod.retryAt = (await $.clock.now()) + RETRY_AFTER_MS + } else { + mod.session = session + } + + return + } + + if (mod.session !== session) mod.pinned = null + mod.session = session + mod.state = state + mod.pinned = state.active?.task_id ?? null + $.ui.status(statusText(state)) +} + +// The session id can change under the mod: a /clear starts a new one with +// no session.start, and a resumed session may only take its id after +// session.start ran. Re-read whenever the id moved, or a retry is due. +async function current($: EngineInterface): Promise<JournalState | null> { + const session = await $.session.id() + const isRetryDue = mod.state === null && mod.retryAt > 0 && (await $.clock.now()) >= mod.retryAt + if (mod.session !== session || isRetryDue) { + mod.retryAt = 0 + await refresh($) + } + + return mod.state +} + +// Resolves to undefined once `ms` passed. The wait costs the hook nothing +// while the raced call is in flight. +async function within<T>($: EngineInterface, ms: number, call: Promise<T>): Promise<T | undefined> { + let timer: { cancel: () => void } | undefined + const timeout = new Promise<undefined>(resolve => { + timer = $.clock.after(ms, () => resolve(undefined)) + }) + + try { + return await Promise.race([call, timeout]) + } finally { + timer?.cancel() + } +} + +// Asks what the conversation decided that the journal lacks. A fork reuses +// the session's own cached transcript; a process that has sent nothing yet +// (a session resumed straight into /compact) has nothing to fork, and then +// the messages being compacted go to a small model instead. +async function distill($: EngineInterface, state: JournalState, messages: readonly TranscriptLine[]): Promise<void> { + const task = state.active + if (task === null) return + + const ask = distillPrompt(task) + let reply = await within($, DISTILL_TIMEOUT_MS, $.model.fork({ prompt: ask })) + if (reply !== undefined && !reply.isAnswered && reply.reason === 'nothing-to-fork') { + const conversation = transcriptExcerpt(messages, EXCERPT_CHARS) + reply = await $.model.complete({ + model: 'haiku', + prompt: `<conversation>\n${conversation}\n</conversation>\n\n${ask}`, + maxTokens: 1024, + timeoutMs: DISTILL_TIMEOUT_MS, + }) + } + if (reply === undefined || !reply.isAnswered) return + + const session = await $.session.id() + for (const event of parseDistill(reply.text, task.recent)) { + await cli($, [ + 'event', task.task_id, + '--type', event.type, + '--text', event.text, + '--suggested', + '--session', session, + '--origin', 'mod-distill', + ]) + } + + await refresh($) +} + +export const register: Register = (on, options) => { + const nudgeAfter = Number(options.nudge_after_turns ?? 6) + const distillOnCompact = options.distill_on_compact !== false + + on('session.start', async ($, e, next) => { + if ((await $.env.get('TJ_IN_CLASSIFIER')) !== undefined) { + mod.isOff = true + + return next(e) + } + + mod.cwd = e.cwd + await refresh($) + if (mod.state !== null) await $.env.set('TJ_MOD_ACTIVE', '1') + + return next(e) + }) + + on('session.end', async ($, e, next) => { + mod.state = null + mod.session = '' + mod.pinned = null + mod.retryAt = 0 + mod.turnsSinceEntry = 0 + mod.workSinceEntry = 0 + + return next(e) + }) + + on('prompt.compose', async ($, e, next) => { + const composed = await next(e) + if (mod.isOff) return composed + + const state = await current($) + if (state === null) return composed + + return { + sections: [...composed.sections, { id: 'task-journal:active', text: sectionText(state), scope: 'session' }], + } + }) + + on('prompt.submit', async ($, e, next) => { + // Only the person's own prompts count as turns: not a background task's + // notification, a peer's message or a plugin's prompt. + const isPersons = e.origin === undefined || e.origin.kind === 'composer' || e.origin.kind === 'bridge' + if (mod.isOff || !isPersons) return next(e) + + const state = await current($) + if (state === null) return next(e) + + mod.turnsSinceEntry += 1 + const isDue = + nudgeAfter > 0 && mod.turnsSinceEntry > nudgeAfter && (state.active !== null || mod.workSinceEntry > 0) + if (!isDue) return next(e) + + mod.turnsSinceEntry = 0 + + return next({ ...e, context: [...(e.context ?? []), nudgeText(state, nudgeAfter)] }) + }) + + on('tool.call', async ($, e, next) => { + const name = journalTool(String(e.tool)) + if (mod.isOff || name === undefined) { + if (WORK_TOOLS.has(String(e.tool))) mod.workSinceEntry += 1 + + return next(e) + } + + // An MCP tool's arguments ride on `e` itself, untyped. + const args = e as unknown as Record<string, unknown> + const session = await $.session.id() + const call = WRITE_TOOLS.has(name) && args.session_id === undefined ? { ...e, session_id: session } : e + const ran = await next(call) + if (ran.deny !== undefined || ran.isError === true || !WRITE_TOOLS.has(name)) return ran + + // A new task, or closing the named one, moves the pin; logging to + // another task does not. + if (name === 'task_create' || (name === 'task_close' && args.task_id === mod.pinned)) mod.pinned = null + + mod.turnsSinceEntry = 0 + mod.workSinceEntry = 0 + await refresh($) + if (name === 'event_add' && typeof args.event_type === 'string') $.ui.toast(`📓 ${args.event_type} recorded`) + + return ran + }) + + on('session.compact', async ($, e, next) => { + if (mod.isOff || e.agentId !== undefined) return next(e) + + const state = await current($) + if (state === null || state.active === null) return next(e) + + // A precompute builds a summary ahead of time that may never be used; + // the catch-up waits for a compaction that happens. + if (distillOnCompact && e.trigger !== 'precompute') { + try { + await distill($, state, e.messages) + } catch { + // The compaction matters more than the catch-up: go on without it. + } + } + + const keep = compactInstruction(state.active) + + return next({ ...e, instructions: e.instructions ? `${e.instructions}\n\n${keep}` : keep }) + }) +} diff --git a/plugin/package.json b/plugin/package.json index a062918..6aba016 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -1,6 +1,6 @@ { "name": "task-journal", - "version": "0.29.0", + "version": "0.30.0", "description": "Append-only journal of AI-coding task reasoning chains. Captures hypotheses, decisions, rejections, evidence — renders compact resume packs so an agent can pick up a 2-week-old task with full context.", "author": { "name": "Mher Shahinyan", @@ -35,10 +35,11 @@ }, "files": [ ".claude-plugin/", + "agents/", "commands/", + "hooks/", "skills/", - "scripts/", - "README.md" + "scripts/" ], "scripts": { "postinstall": "node scripts/postinstall.mjs" diff --git a/plugin/skills/task-journal/SKILL.md b/plugin/skills/task-journal/SKILL.md index d4e4029..9087e9a 100644 --- a/plugin/skills/task-journal/SKILL.md +++ b/plugin/skills/task-journal/SKILL.md @@ -23,8 +23,9 @@ treat it as if it does nothing and record explicitly. ## Session-start ritual (do this before real work) -1. `task_search(query=<a few words about the work>, status="open")` — is there an open - task for this? If yes, `task_pack(task_id)` and continue it. **Do not** open a duplicate. +1. `task_search(status="open")` (add `query=<a few words about the work>` to narrow) — is + there an open task for this? Each entry in `tasks` shows its title and goal. If yes, + `task_pack(task_id)` and continue it. **Do not** open a duplicate. 2. If nothing fits, `task_create(title=<short>, goal=<one sentence: what the user is trying to accomplish>)`. **Always pass `goal`** — it is the first line of every pack and the anchor for "why was this done?". 3. Hold the returned `task_id` for the whole task. One task = one logical objective. @@ -100,7 +101,11 @@ task_close( | Resuming an existing project | `task_search(status="open")` → `task_pack` on the match | `task_pack` mode: `compact` (~2KB, default for resume) or `full` (~10KB, full trail). -`task_search` filters: `status`, `project`, `event_type` (decision/finding/evidence/…). +`task_search` filters: `status` (`open` | `closed` | `any`, default `any`), `project` +(absolute path of another project directory), `event_type` (decision/finding/evidence/…). +An empty or absent `query` lists the tasks, newest activity first. The result has +`results` (task ids) and `tasks` — the same ids in the same order, each with `title`, +`status`, `last_event_at` and `goal`. ## The 5 MCP tools (exact params) @@ -110,7 +115,8 @@ task_close( correction | reopen | supersede | redirect. `alternatives` is decision-only. - `task_close(task_id, reason, outcome?, outcome_tag?)` — **always pass `outcome` + `outcome_tag`.** - `task_pack(task_id, mode?)` — `compact` | `full`. -- `task_search(query, status?, project?, event_type?)` — FTS5 over this project's events. +- `task_search(query?, status?, project?, event_type?)` — full-text search over the project's + events; no `query` lists its tasks. Returns `results` (ids) and `tasks` (id, title, status, goal). ## Invariants