From 1ba546e1f106ed0043b19a0b2ed7ac91c51c215c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:23:54 +0400 Subject: [PATCH 001/104] fix(plugin): ship agents/ and hooks/ in package files, drop missing README --- plugin/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugin/package.json b/plugin/package.json index a062918..c29a72c 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -35,10 +35,11 @@ }, "files": [ ".claude-plugin/", + "agents/", "commands/", + "hooks/", "skills/", - "scripts/", - "README.md" + "scripts/" ], "scripts": { "postinstall": "node scripts/postinstall.mjs" From 7ce2f0e2f8748b7b48f6f77fa60ddaf6b74db7c7 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:25:25 +0400 Subject: [PATCH 002/104] build: pin task-journal-core once in workspace deps and test it against the crate version --- Cargo.toml | 3 +++ crates/tj-cli/Cargo.toml | 2 +- crates/tj-cli/tests/plugin_metadata.rs | 32 ++++++++++++++++++++++++++ crates/tj-mcp/Cargo.toml | 2 +- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c150d5..e626480 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,9 @@ keywords = ["task-journal", "ai-coding", "mcp", "memory", "agent"] categories = ["command-line-utilities", "development-tools"] [workspace.dependencies] +# `version` must equal workspace.package.version: `cargo publish` uploads it as +# the requirement on core. Cargo can't inherit it; plugin_metadata.rs checks it. +tj-core = { package = "task-journal-core", version = "0.29.0", path = "crates/tj-core", default-features = false } anyhow = "1" thiserror = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index c89b0a8..7ad5c3d 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -23,7 +23,7 @@ default = ["embed"] embed = ["tj-core/embed"] [dependencies] -tj-core = { package = "task-journal-core", version = "0.29.0", path = "../tj-core", default-features = false } +tj-core = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } diff --git a/crates/tj-cli/tests/plugin_metadata.rs b/crates/tj-cli/tests/plugin_metadata.rs index 8c9ebb2..4532c79 100644 --- a/crates/tj-cli/tests/plugin_metadata.rs +++ b/crates/tj-cli/tests/plugin_metadata.rs @@ -42,6 +42,38 @@ fn plugin_manifests_carry_the_crate_version() { ); } +/// `cargo publish` uploads tj-cli and tj-mcp with their `task-journal-core` +/// requirement taken from the version literal, which Cargo cannot inherit +/// from `workspace.package.version`. A stale literal ships binaries built +/// against the previous core release. +#[test] +fn internal_dependency_pins_carry_the_crate_version() { + let want = format!("version = \"{}\"", env!("CARGO_PKG_VERSION")); + + for rel in [ + "Cargo.toml", + "crates/tj-cli/Cargo.toml", + "crates/tj-mcp/Cargo.toml", + ] { + let body = std::fs::read_to_string(repo_root().join(rel)) + .unwrap_or_else(|e| panic!("read {rel}: {e}")); + + let pins = body.lines().filter(|l| { + l.contains("package = \"task-journal-") || l.trim_start().starts_with("task-journal-") + }); + + for line in pins { + if line.contains("workspace = true") { + continue; + } + assert!( + line.contains(&want), + "{rel}: internal dependency pin is stale: {line}" + ); + } + } +} + /// The MCP server is declared once, in `plugin.json`. A second declaration in /// `plugin/.mcp.json` made Claude Code see the same stdio server twice. #[test] diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index 7c71bf6..bf50032 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] # Lean: the MCP server doesn't embed yet, so it skips the model2vec backend. -tj-core = { package = "task-journal-core", version = "0.29.0", path = "../tj-core", default-features = false } +tj-core = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } From dfadf080d5e53ac43cc4b51b82a95debd10964fa Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:26:07 +0400 Subject: [PATCH 003/104] docs(releasing): align secret, sleeps and bump checklist with publish.yml --- RELEASING.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 173885a..f2fac3c 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,9 +5,9 @@ ## Pre-release (один раз для проекта) 1. **GitHub repo создан**, origin подключён. -2. **crates.io аккаунт** заведён → Settings → API Tokens → создать токен `task-journal-publish` с scope `publish-new`. +2. **crates.io аккаунт** заведён → Settings → API Tokens → создать токен `task-journal-publish` со scopes `publish-new` и `publish-update` (без `publish-update` новые версии уже существующих crate'ов не опубликуются). 3. **GitHub Secrets** (Settings → Secrets and variables → Actions → New repository secret): - - `CRATES_IO_TOKEN` = токен из шага 2. + - `CRATES_IO_TOKEN` = токен из шага 2. `publish.yml` читает именно этот secret и передаёт его cargo как env `CARGO_REGISTRY_TOKEN` — поэтому ошибка «`CARGO_REGISTRY_TOKEN is empty`» значит, что не задан `CRATES_IO_TOKEN`. 4. **Crate names проверены** — `cargo search task-journal-core task-journal-cli task-journal-mcp` показывает что свободны. Если заняты — поменять в Cargo.toml каждого crate (`name = "..."`). ## Per-release flow @@ -26,17 +26,19 @@ cargo fmt --all --check # ВАЖНО: версия плагина НЕ наследуется из Cargo.toml — синхронь вручную, иначе # `/plugin update` будет показывать старую версию (см. историю: 0.10.3 vs 0.12.0). # Обнови до той же версии: +# - Cargo.toml [workspace.dependencies] → tj-core version (cargo publish +# заливает её как требование на core) # - plugin/.claude-plugin/plugin.json → "version" # - plugin/package.json → "version" # - .claude-plugin/marketplace.json → metadata.version И plugins[0].version # -# Проверка (падает, если хоть один манифест отстал): +# Проверка (падает, если хоть один манифест или пин отстал): cargo test -p task-journal-cli --test plugin_metadata # Плагин целиком — манифест, MCP-объявления, frontmatter скиллов и команд: claude plugin validate plugin --json -git add Cargo.toml plugin/.claude-plugin/plugin.json plugin/package.json .claude-plugin/marketplace.json +git add Cargo.toml Cargo.lock plugin/.claude-plugin/plugin.json plugin/package.json .claude-plugin/marketplace.json git commit -m "chore: bump version to v0.1.1" git push ``` @@ -61,8 +63,9 @@ git push origin v0.1.1 ```bash cargo login # paste crates.io token cargo publish -p task-journal-core -sleep 30 +sleep 60 # как в publish.yml: crates.io должен проиндексировать core cargo publish -p task-journal-cli +sleep 30 cargo publish -p task-journal-mcp ``` @@ -84,7 +87,7 @@ cargo publish -p task-journal-mcp ## Post-release Для каждой следующей версии: -- Обнови `version` в `Cargo.toml` workspace +- Пройди шаг 1 целиком: `Cargo.toml` (workspace `version` и пин `tj-core`) **и** три манифеста плагина, `plugin_metadata` зелёный - `git tag vX.Y.Z && git push origin vX.Y.Z` - CI делает остальное @@ -94,5 +97,5 @@ cargo publish -p task-journal-mcp |---------|---------------| | `cargo publish` падает с "name already taken" | Имя crate занято на crates.io. Поменяй `name = "..."` в Cargo.toml на другое. | | Release workflow не запустился | Тэг должен начинаться с `v` (например `v0.1.0`, не `0.1.0` или `release-0.1.0`). | -| `cargo publish -p task-journal-cli` падает с "task-journal-core not found" | Не подождал `sleep 30` после публикации `task-journal-core` — повтори через минуту. | +| `cargo publish -p task-journal-cli` падает с "task-journal-core not found" | Не подождал `sleep 60` после публикации `task-journal-core` — повтори через минуту. | | Pre-built binary не работает на macOS — "killed" | Apple Silicon vs Intel — скачай правильный target (`aarch64-apple-darwin` для M1/M2/M3, `x86_64-apple-darwin` для Intel). | From 7ccd09e0061b98eb175e7d333b332b174f21ee65 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:26:47 +0400 Subject: [PATCH 004/104] ci: gate crates.io publish and release builds on fmt, clippy and tests --- .github/workflows/publish.yml | 26 ++++++++++++++++++++++++++ .github/workflows/release.yml | 26 ++++++++++++++++++++++++++ RELEASING.md | 2 ++ 3 files changed, 54 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d313280..5c88038 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,8 +7,34 @@ on: workflow_dispatch: jobs: + # A tag doesn't wait for CI, so a tag on a red commit would publish. Gate on + # the same checks as ci.yml's test job. + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: cargo fmt + run: cargo fmt --all --check + + - name: cargo clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: cargo test + run: cargo test --workspace --all-targets + publish: name: cargo publish + needs: test runs-on: ubuntu-latest env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c6fd1d..f1a55ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,8 +17,34 @@ env: CARGO_TERM_COLOR: always jobs: + # A tag doesn't wait for CI, so a tag on a red commit would ship binaries. + # Gate on the same checks as ci.yml's test job. + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: cargo fmt + run: cargo fmt --all --check + + - name: cargo clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: cargo test + run: cargo test --workspace --all-targets + build: name: build (${{ matrix.target }}) + needs: test strategy: fail-fast: false matrix: diff --git a/RELEASING.md b/RELEASING.md index f2fac3c..56ddf6f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -58,6 +58,8 @@ git push origin v0.1.1 - **`.github/workflows/release.yml`** — собирает pre-built бинарники под Linux/macOS-x86_64/macOS-arm64/Windows и публикует GitHub Release с прикреплёнными `tar.gz`/`.zip` + `checksums.txt`. - **`.github/workflows/publish.yml`** — публикует все 3 crate'а на crates.io (нужен `CRATES_IO_TOKEN` secret). +Оба workflow сначала гоняют свой job `test` (fmt, clippy `-D warnings`, `cargo test --workspace --all-targets`); если он красный — ни бинарники, ни crate'ы не публикуются. + ### 3. Если автопубликация не нужна — руками ```bash From 39c4d2bb34e18d7b980a56f518b106ac7fcea821 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:28:00 +0400 Subject: [PATCH 005/104] fix(core): quote each FTS5 query token so punctuation never breaks search --- crates/tj-core/src/fts.rs | 107 +++++++++++++++++++++++++++----------- crates/tj-mcp/src/main.rs | 6 +-- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/crates/tj-core/src/fts.rs b/crates/tj-core/src/fts.rs index 5e34640..d96e639 100644 --- a/crates/tj-core/src/fts.rs +++ b/crates/tj-core/src/fts.rs @@ -1,30 +1,24 @@ //! FTS5 query sanitization. //! -//! SQLite FTS5 treats `-`, `:`, `*`, `(`, `)`, `"`, and `/` as syntax. -//! A raw query like `OPS-306` parses as column-prefix `OPS-` followed by -//! reference `306`, raising `no such column: 306` at runtime. +//! SQLite FTS5 treats almost any punctuation (`-`, `:`, `*`, `.`, `,`, `'`, +//! `+`, `^`, `{`, `?`, …) and the bare words `AND`/`OR`/`NOT`/`NEAR` as +//! syntax. A raw query like `OPS-306` parses as column-prefix `OPS-` +//! followed by reference `306`, raising `no such column: 306` at runtime. //! -//! [`sanitize_query`] wraps queries containing any metacharacter in -//! double quotes, turning them into FTS5 phrase queries that match the -//! sequence of tokens as the unicode61 tokenizer split them. Internal -//! `"` characters are doubled per FTS5 escape rules. - -/// Wrap a query in FTS5 phrase quotes if it contains any character the -/// FTS5 parser treats as syntax. Strings that only contain word -/// characters / whitespace pass through untouched so multi-term -/// queries keep their default AND semantics. +//! [`sanitize_query`] splits the query on whitespace and wraps every token +//! in double quotes, so each one is an FTS5 string matched as the unicode61 +//! tokenizer splits it, and the tokens are implicitly AND-ed. Internal `"` +//! characters are doubled per FTS5 escape rules. + +/// Quote every whitespace-separated token as its own FTS5 string. Any +/// punctuation inside a token is then literal, operator words lose their +/// meaning, and a multi-token query keeps its default AND semantics. pub fn sanitize_query(query: &str) -> String { - if query.is_empty() { - return String::new(); - } - let needs_quote = query - .chars() - .any(|c| matches!(c, '-' | '"' | '*' | ':' | '(' | ')' | '/')); - if !needs_quote { - return query.to_string(); - } - let escaped = query.replace('"', "\"\""); - format!("\"{escaped}\"") + query + .split_whitespace() + .map(|tok| format!("\"{}\"", tok.replace('"', "\"\""))) + .collect::>() + .join(" ") } /// Build a `LIKE` pattern equivalent to a free-text search — used as a @@ -40,19 +34,71 @@ pub fn like_pattern(query: &str) -> String { mod tests { use super::{like_pattern, sanitize_query}; + /// Count rows of a throwaway FTS5 table matching `sanitize_query(query)`. + /// `Err` means FTS5 rejected the sanitized query as syntax. + fn fts_hits(rows: &[&str], query: &str) -> rusqlite::Result { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE VIRTUAL TABLE t USING fts5(text);") + .unwrap(); + for row in rows { + conn.execute("INSERT INTO t(text) VALUES (?1)", [row]) + .unwrap(); + } + + conn.query_row( + "SELECT COUNT(*) FROM t WHERE t MATCH ?1", + [sanitize_query(query)], + |r| r.get(0), + ) + } + + #[test] + fn plain_word_is_quoted_as_one_token() { + assert_eq!(sanitize_query("hello"), "\"hello\""); + } + + #[test] + fn multi_word_quotes_each_token_for_default_and_search() { + assert_eq!(sanitize_query("bulk repack"), "\"bulk\" \"repack\""); + } + #[test] - fn plain_word_passes_through() { - assert_eq!(sanitize_query("hello"), "hello"); + fn cyrillic_tokens_are_quoted() { + assert_eq!(sanitize_query("слим модели"), "\"слим\" \"модели\""); } #[test] - fn multi_word_passes_through_for_default_and_search() { - assert_eq!(sanitize_query("bulk repack"), "bulk repack"); + fn punctuation_and_operator_words_never_raise_fts5_syntax_errors() { + let rows = ["release v1.2, don't c++ AND or NOT near"]; + for q in [ + "v1.2", + "a, b", + "don't", + "c++", + "^start", + "{x}", + "why?", + "AND", + "OR", + "NOT", + "NEAR", + "this AND", + "NEAR(a b)", + "a + b", + ] { + assert!( + fts_hits(&rows, q).is_ok(), + "FTS5 rejected sanitized query {q:?}: {:?}", + fts_hits(&rows, q) + ); + } } #[test] - fn cyrillic_passes_through() { - assert_eq!(sanitize_query("слим модели"), "слим модели"); + fn multi_word_query_with_hyphen_is_and_of_tokens_not_a_phrase() { + let rows = ["OPS-306 needs a quick fix"]; + assert_eq!(fts_hits(&rows, "fix OPS-306").unwrap(), 1); + assert_eq!(fts_hits(&rows, "fix OPS-307").unwrap(), 0); } #[test] @@ -82,12 +128,13 @@ mod tests { #[test] fn embedded_quote_is_doubled() { - assert_eq!(sanitize_query("say \"hi\""), "\"say \"\"hi\"\"\""); + assert_eq!(sanitize_query("say \"hi\""), "\"say\" \"\"\"hi\"\"\""); } #[test] fn empty_query_stays_empty() { assert_eq!(sanitize_query(""), ""); + assert_eq!(sanitize_query(" "), ""); } #[test] diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index c374c90..fa9aefd 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -471,9 +471,9 @@ impl TaskJournalServer { // v0.10.3: sanitize FTS5 query. Hyphenated IDs like // `OPS-306` previously crashed with "no such column: 306" - // because FTS5 reads `-` as column-prefix syntax. Wrap - // such queries in phrase quotes; safe queries pass - // through unchanged so AND semantics are preserved. + // because FTS5 reads `-` as column-prefix syntax. Every + // token is quoted on its own, so punctuation is literal + // and multi-word queries keep their AND semantics. let fts_query = tj_core::fts::sanitize_query(&raw_query); let (sql, fts_only) = match &event_type { Some(_) => ( From a3b1ff429e2f4a9dc648b6c29ad3576a4abe5a49 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:28:07 +0400 Subject: [PATCH 006/104] refactor(core): remove unused argv ClaudeBinaryRunner --- crates/tj-core/src/classifier/agent_sdk.rs | 56 +++++++--------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/crates/tj-core/src/classifier/agent_sdk.rs b/crates/tj-core/src/classifier/agent_sdk.rs index 777a54c..695b7cb 100644 --- a/crates/tj-core/src/classifier/agent_sdk.rs +++ b/crates/tj-core/src/classifier/agent_sdk.rs @@ -50,10 +50,10 @@ pub trait CommandRunner: Send + Sync { } } -/// Build the base `claude` invocation shared by both runners: print mode, the -/// pinned model, the JSON envelope, an isolated MCP config, and — critically — -/// the [`IN_CLASSIFIER_ENV`] recursion marker. The argv runner appends the -/// prompt as a positional arg; the stdin runner feeds it on stdin. Extracted so +/// Build the base `claude` invocation: print mode, the pinned model, the JSON +/// envelope, an isolated MCP config, and — critically — the +/// [`IN_CLASSIFIER_ENV`] recursion marker. The prompt goes on stdin (see +/// [`ClaudeBinaryStdinRunner`]), never as a positional arg. Extracted so /// a unit test can assert the marker is present without spawning `claude` (the /// missing marker is exactly what let the fork bomb through before). fn base_claude_command(model: &str) -> Command { @@ -79,12 +79,6 @@ fn base_claude_command(model: &str) -> Command { const DISABLED_TOOLS: &str = "Bash Read Edit Write Glob Grep Task WebFetch \ WebSearch NotebookEdit TodoWrite BashOutput KillBash"; -/// Production runner: invokes the local `claude` binary in print mode, pinned -/// to the given model, asking for the JSON envelope and an isolated MCP config -/// (`--strict-mcp-config` keeps the project's own MCP servers — including this -/// very journal — out of the classification subprocess). -pub struct ClaudeBinaryRunner; - /// Build the error for a non-zero `claude -p` exit. With `--output-format /// json` claude reports the real cause (invalid model, usage limit, auth) as /// JSON on **stdout**, not stderr — so surface both, capped, or the user just @@ -170,31 +164,15 @@ pub(crate) fn wait_with_timeout( }) } -impl CommandRunner for ClaudeBinaryRunner { - fn run(&self, model: &str, prompt: &str) -> anyhow::Result { - let child = base_claude_command(model) - .arg(prompt) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn `claude` (is Claude Code installed and on PATH?)")?; - let output = wait_with_timeout(child, claude_timeout())?; - if !output.status.success() { - return Err(claude_exit_error( - output.status, - &output.stdout, - &output.stderr, - )); - } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } -} - -/// Like [`ClaudeBinaryRunner`] but feeds the prompt on **stdin** instead of as -/// an argv argument. Use for large prompts (e.g. a whole session transcript in -/// dream backfill) that would otherwise blow the per-argument size limit -/// (`E2BIG`, ~128 KiB on Linux). `claude -p` with no positional prompt reads -/// the prompt from stdin. +/// Production runner: invokes the local `claude` binary in print mode, pinned +/// to the given model, asking for the JSON envelope and an isolated MCP config +/// (`--strict-mcp-config` keeps the project's own MCP servers — including this +/// very journal — out of the classification subprocess). The prompt goes on +/// **stdin**, not as an argv argument: that sidesteps the `--disallowed-tools` +/// collision (see [`CommandRunner::feeds_prompt_on_stdin`]) and the +/// per-argument size limit (`E2BIG`, ~128 KiB on Linux) that a whole session +/// transcript in dream backfill would blow. `claude -p` with no positional +/// prompt reads the prompt from stdin. pub struct ClaudeBinaryStdinRunner; impl CommandRunner for ClaudeBinaryStdinRunner { @@ -511,9 +489,9 @@ mod tests { assert!(format!("{err}").contains("error"), "got: {err}"); } - /// Regression: the production classifier MUST feed the prompt on stdin. With - /// the argv runner the prompt lands right after `--disallowed-tools` and the - /// current `claude` CLI swallows it as bogus deny-rules, failing every + /// Regression: the production classifier MUST feed the prompt on stdin. As + /// an argv positional the prompt lands right after `--disallowed-tools` and + /// the current `claude` CLI swallows it as bogus deny-rules, failing every /// classification (the "139 pending" backlog). Lock the choice here. #[test] fn production_runner_feeds_prompt_on_stdin() { @@ -521,8 +499,6 @@ mod tests { default_runner().feeds_prompt_on_stdin(), "classifier prompt must go on stdin, not as an argv positional" ); - // the argv runner is the one that collides — keep the contrast explicit - assert!(!ClaudeBinaryRunner.feeds_prompt_on_stdin()); assert!(ClaudeBinaryStdinRunner.feeds_prompt_on_stdin()); } } From 97296c934f072fed4aa59e605580227041d13ad0 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:28:29 +0400 Subject: [PATCH 007/104] docs(install): describe what hook uninstall removes for each client --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 9f8d366..4f6293f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -99,7 +99,7 @@ task-journal install-hooks --scope user --uninstall # add --client codex to remove them from ~/.codex/hooks.json ``` -This removes only the `hooks` block from `settings.json` (keeps theme, other servers, everything else). +This removes only task-journal's own hook entries (and its `TJ_CLASSIFIER_CLI` env key) from `~/.claude/settings.json`, or from `~/.codex/hooks.json` with `--client codex`. Other plugins' hooks, theme, servers and everything else stay as they are. With `--scope project` the file is `.claude/settings.json` or `.codex/hooks.json` in the current directory. ## Where data lives From 5e8f7de899b855d91c6a4468eea3b67cdef0c89c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:29:49 +0400 Subject: [PATCH 008/104] feat(plugin): Claude Code mod keeps the session's task in the system prompt --- .gitignore | 3 + plugin/.claude-plugin/plugin.json | 18 ++- plugin/hooks/hooks.json | 3 + plugin/hooks/journal.test.ts | 117 +++++++++++++++++ plugin/hooks/journal.ts | 171 +++++++++++++++++++++++++ plugin/hooks/register.ts | 204 ++++++++++++++++++++++++++++++ 6 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 plugin/hooks/hooks.json create mode 100644 plugin/hooks/journal.test.ts create mode 100644 plugin/hooks/journal.ts create mode 100644 plugin/hooks/register.ts diff --git a/.gitignore b/.gitignore index c19b5bd..d10d988 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ # Local design specs & plans (not versioned) .docs/ + +# Types Claude Code lays beside a mod it loads from this folder +plugin/.claude-plugin/types/ diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index ccd67dd..7ba1073 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -18,5 +18,21 @@ "command": "task-journal-mcp" } }, - "skills": "./skills/" + "skills": "./skills/", + "userConfig": { + "nudge_after_turns": { + "type": "number", + "title": "Remind after turns", + "description": "With the mod on (Claude Code 2.1.287+): remind the agent to log after this many of your prompts without a journal entry. 0 turns the reminder off.", + "default": 6, + "min": 0, + "max": 50 + }, + "distill_on_compact": { + "type": "boolean", + "title": "Catch up before compaction", + "description": "With the mod on: before the conversation is compacted, ask the model which decisions, rejections and findings were not logged, and record them as suggested events. One extra model call per compaction, mostly served from the prompt cache.", + "default": true + } + } } diff --git a/plugin/hooks/hooks.json b/plugin/hooks/hooks.json new file mode 100644 index 0000000..9f45101 --- /dev/null +++ b/plugin/hooks/hooks.json @@ -0,0 +1,3 @@ +{ + "modules": ["./register.ts"] +} diff --git a/plugin/hooks/journal.test.ts b/plugin/hooks/journal.test.ts new file mode 100644 index 0000000..28c9e66 --- /dev/null +++ b/plugin/hooks/journal.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from 'claude-code/testing' + +import { + journalTool, + nudgeText, + parseDistill, + parseState, + sectionText, + statusText, + type JournalState, +} from './journal' + +const ACTIVE = JSON.stringify({ + schema: 'tj-state/1', + open_tasks: 2, + active: { + task_id: 'tj-abc', + title: 'Fix refresh', + goal: 'Stop dropping the token', + counts: { decision: 3, rejection: 1, evidence: 0 }, + recent: [{ type: 'decision', text: 'Use <= on expiry' }], + }, +}) + +describe('parseState', () => { + test('reads the active task', () => { + const s = parseState(ACTIVE) + expect(s?.active?.task_id).toBe('tj-abc') + expect(s?.active?.recent.length).toBe(1) + expect(s?.open_tasks).toBe(2) + }) + + test('no active task is a state, not a failure', () => { + const s = parseState(JSON.stringify({ schema: 'tj-state/1', open_tasks: 0, active: null })) + expect(s).toEqual({ active: null, open_tasks: 0 }) + }) + + test('rejects garbage and other schemas', () => { + expect(parseState('error: unrecognized subcommand')).toBe(null) + expect(parseState(JSON.stringify({ schema: 'tj-state/9', active: null }))).toBe(null) + }) + + test('an empty goal reads as not set', () => { + const s = parseState(JSON.stringify({ schema: 'tj-state/1', open_tasks: 1, active: { task_id: 'tj-x', goal: ' ' } })) + expect(s?.active?.goal).toBe(null) + expect(s?.active?.title).toBe('tj-x') + }) +}) + +describe('texts', () => { + const s = parseState(ACTIVE) as JournalState + const none: JournalState = { active: null, open_tasks: 3 } + + test('status line names the task and its counts', () => { + expect(statusText(s)).toBe('📓 tj-abc · 3 decisions · 1 rejected · 0 evidence') + expect(statusText(none)).toBe('📓 no task in this session · 3 open') + expect(statusText({ active: null, open_tasks: 0 })).toBe('📓 no task') + }) + + test('the prompt section holds nothing that changes per entry', () => { + const text = sectionText(s) + expect(text).toContain('tj-abc') + expect(text).toContain('Stop dropping the token') + expect(text).not.toContain('3 decisions') + expect(text).not.toContain('Use <= on expiry') + }) + + test('nudge names the task, or asks to open one', () => { + expect(nudgeText(s, 6)).toContain('6 turns since the last journal entry on tj-abc') + expect(nudgeText(none, 6)).toContain('task_create') + }) +}) + +describe('journalTool', () => { + test('matches the plugin server and a hand-wired one', () => { + expect(journalTool('mcp__plugin_task-journal_task-journal__event_add')).toBe('event_add') + expect(journalTool('mcp__task-journal__task_close')).toBe('task_close') + expect(journalTool('mcp__other__event_add')).toBe(undefined) + expect(journalTool('Bash')).toBe(undefined) + }) +}) + +describe('parseDistill', () => { + const known = [{ type: 'decision', text: 'Use <= on expiry' }] + + test('keeps typed, new events from a fenced reply', () => { + const reply = 'Here:\n```json\n[{"type":"rejection","text":"Ruled out a cache: stale tokens"},{"type":"finding","text":"refresh.rs:42 uses <"}]\n```' + expect(parseDistill(reply, known)).toEqual([ + { type: 'rejection', text: 'Ruled out a cache: stale tokens' }, + { type: 'finding', text: 'refresh.rs:42 uses <' }, + ]) + }) + + test('drops known, duplicate, untyped and empty entries', () => { + const reply = JSON.stringify([ + { type: 'decision', text: ' use <= ON expiry ' }, + { type: 'open', text: 'not a distill type' }, + { type: 'finding', text: '' }, + { type: 'finding', text: 'A' }, + { type: 'finding', text: 'a' }, + 'junk', + ]) + expect(parseDistill(reply, known)).toEqual([{ type: 'finding', text: 'A' }]) + }) + + test('caps the count and the length', () => { + const many = Array.from({ length: 20 }, (_, i) => ({ type: 'finding', text: `f${i} ${'x'.repeat(600)}` })) + const out = parseDistill(JSON.stringify(many), []) + expect(out.length).toBe(8) + expect(out[0]?.text.length).toBe(500) + }) + + test('no array, no events', () => { + expect(parseDistill('Nothing to add.', known)).toEqual([]) + expect(parseDistill('[not json', known)).toEqual([]) + }) +}) diff --git a/plugin/hooks/journal.ts b/plugin/hooks/journal.ts new file mode 100644 index 0000000..7e4073d --- /dev/null +++ b/plugin/hooks/journal.ts @@ -0,0 +1,171 @@ +// Pure helpers of the Task Journal mod: everything here is plain data in, +// plain data out, so journal.test.ts covers it without an engine. + +export type JournalEvent = { type: string; text: string } + +export type ActiveTask = { + task_id: string + title: string + goal: string | null + counts: Record + recent: JournalEvent[] +} + +export type JournalState = { + active: ActiveTask | null + open_tasks: number +} + +export const STATE_SCHEMA = 'tj-state/1' + +const DISTILL_TYPES = new Set(['decision', 'rejection', 'finding', 'evidence', 'constraint', 'hypothesis']) + +const MAX_DISTILLED = 8 + +const MAX_EVENT_CHARS = 500 + +// `mcp__plugin_task-journal_task-journal__event_add` when the server comes +// with the plugin, `mcp__task-journal__event_add` when it was wired by hand. +const JOURNAL_TOOL = /^mcp__(?:plugin_task-journal_)?task-journal__([a-z_]+)$/ + +export const WRITE_TOOLS = new Set(['task_create', 'event_add', 'artifact_add', 'task_close']) + +export function journalTool(tool: string): string | undefined { + return JOURNAL_TOOL.exec(tool)?.[1] +} + +/** The `task-journal state --json` output, or null when it is not one. */ +export function parseState(stdout: string): JournalState | null { + let raw: unknown + try { + raw = JSON.parse(stdout) + } catch { + return null + } + + if (typeof raw !== 'object' || raw === null) return null + const v = raw as Record + if (v.schema !== STATE_SCHEMA) return null + + const open = typeof v.open_tasks === 'number' ? v.open_tasks : 0 + const a = v.active as Record | null | undefined + if (a === null || a === undefined || typeof a.task_id !== 'string') { + return { active: null, open_tasks: open } + } + + const recent = Array.isArray(a.recent) + ? a.recent.filter( + (r): r is JournalEvent => + typeof r === 'object' && r !== null && typeof r.type === 'string' && typeof r.text === 'string', + ) + : [] + + return { + active: { + task_id: a.task_id, + title: typeof a.title === 'string' ? a.title : a.task_id, + goal: typeof a.goal === 'string' && a.goal.trim() !== '' ? a.goal : null, + counts: (a.counts as Record) ?? {}, + recent, + }, + open_tasks: open, + } +} + +export function statusText(state: JournalState): string { + const a = state.active + if (a === null) { + return state.open_tasks > 0 ? `📓 no task in this session · ${state.open_tasks} open` : '📓 no task' + } + + const n = (k: string) => a.counts[k] ?? 0 + + return `📓 ${a.task_id} · ${n('decision')} decisions · ${n('rejection')} rejected · ${n('evidence')} evidence` +} + +/** + * The system-prompt section. It sits before the conversation, so every + * change re-sends the whole conversation uncached: it names only what + * changes rarely (the task, its title and goal), never counts or times. + */ +export function sectionText(state: JournalState): string { + const a = state.active + if (a === null) { + return [ + 'Task Journal: no journal task is open in this session.', + 'When real work starts, resume one with task_search(status="open") or open one with task_create(goal=...).', + ].join('\n') + } + + return [ + `Task Journal: the active task in this session is ${a.task_id} — "${a.title}".`, + `Goal: ${a.goal ?? 'not set yet — pass it when you next call task_create, or ask the user.'}`, + `Record each decision, rejection, finding and test result with event_add(task_id="${a.task_id}") the moment it happens; close it with task_close when the work is done.`, + ].join('\n') +} + +export function nudgeText(state: JournalState, turns: number): string { + const a = state.active + if (a === null) { + return 'Task Journal: work is happening but no journal task is open in this session. Resume one with task_search(status="open") or open one with task_create(goal=...).' + } + + return `Task Journal: ${turns} turns since the last journal entry on ${a.task_id}. If a decision, rejection, finding or test result happened since, record it now with event_add.` +} + +export function compactInstruction(task: ActiveTask): string { + return `Keep the Task Journal task id ${task.task_id} ("${task.title}") and its goal in the summary, so the work can go on logging to it.` +} + +export function distillPrompt(task: ActiveTask): string { + const known = task.recent.map(r => `- [${r.type}] ${r.text}`).join('\n') || '- (nothing yet)' + + return [ + `You are auditing the Task Journal of task ${task.task_id} ("${task.title}").`, + `Goal: ${task.goal ?? 'not set'}.`, + 'Already recorded:', + known, + '', + 'From the conversation above, list the decisions, rejections, findings, evidence, constraints and hypotheses that matter for this task and are NOT already recorded.', + `Each one sentence, specific (names, files, numbers). At most ${MAX_DISTILLED}. Nothing worth adding → [].`, + 'Reply with a JSON array only: [{"type": "decision", "text": "..."}]. Allowed types: decision, rejection, finding, evidence, constraint, hypothesis.', + ].join('\n') +} + +function normalized(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ').trim() +} + +/** The model's distill reply → the events worth recording, already-known ones dropped. */ +export function parseDistill(reply: string, known: JournalEvent[]): JournalEvent[] { + const start = reply.indexOf('[') + const end = reply.lastIndexOf(']') + if (start === -1 || end <= start) return [] + + let raw: unknown + try { + raw = JSON.parse(reply.slice(start, end + 1)) + } catch { + return [] + } + if (!Array.isArray(raw)) return [] + + const seen = new Set(known.map(k => normalized(k.text))) + const out: JournalEvent[] = [] + + for (const item of raw) { + if (typeof item !== 'object' || item === null) continue + const { type, text } = item as Record + if (typeof type !== 'string' || typeof text !== 'string') continue + if (!DISTILL_TYPES.has(type)) continue + + const clean = text.trim().slice(0, MAX_EVENT_CHARS) + if (clean === '' || seen.has(normalized(clean))) continue + + seen.add(normalized(clean)) + out.push({ type, text: clean }) + if (out.length === MAX_DISTILLED) break + } + + return out +} diff --git a/plugin/hooks/register.ts b/plugin/hooks/register.ts new file mode 100644 index 0000000..6f7053c --- /dev/null +++ b/plugin/hooks/register.ts @@ -0,0 +1,204 @@ +// Task Journal mod (Claude Code 2.1.287+). A thin layer over the +// `task-journal` CLI: the Rust core stays the only writer of the journal. +// +// - keeps the session's active task in the system prompt, so a compaction +// cannot lose it, and asks the compaction summary to keep its id; +// - stamps the session id on the journal's MCP calls, so each session has +// its own active task; +// - shows the task in the status line and a toast on every entry; +// - reminds the agent to log only after N turns without an entry; +// - before a compaction, asks the model (from its own prompt cache) what +// was not logged and records it as suggested events. +// +// It sets TJ_MOD_ACTIVE=1 for the hooks Claude Code starts, so the classic +// `task-journal ingest-hook` capture and `nudge` stand down instead of +// doing the same work twice. + +import type { EngineInterface, Register } from 'claude-code' + +import { + compactInstruction, + distillPrompt, + journalTool, + nudgeText, + parseDistill, + parseState, + sectionText, + statusText, + WRITE_TOOLS, + type JournalState, +} from './journal' + +const WORK_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'Bash']) + +// What the mod knows about the session it runs in. A module variable: it +// lives as long as the process, which is as long as the session. +const mod = { + isOff: false, + hasWarned: false, + state: null as JournalState | null, + session: '', + turnsSinceEntry: 0, + workSinceEntry: 0, +} + +async function cli($: EngineInterface, argv: string[]): Promise { + try { + const run = await $.process.run(['task-journal', ...argv], { + cwd: await $.session.root(), + timeoutMs: 10_000, + }) + + return run.exitCode === 0 ? run.stdout : null + } catch { + return null + } +} + +async function refresh($: EngineInterface): Promise { + const session = await $.session.id() + const out = await cli($, ['state', '--session', session, '--json']) + const state = out === null ? null : parseState(out) + + if (state === null) { + if (mod.state === null && !mod.hasWarned) { + mod.hasWarned = true + $.ui.log( + 'task-journal: the task-journal CLI 0.30+ was not found on PATH, so the journal mod is off. Install it with `cargo install task-journal-cli task-journal-mcp --force`.', + ) + } + + return + } + + mod.state = state + mod.session = session + $.ui.status(statusText(state)) +} + +// A /clear ends the session without a new session.start: catch up on the +// first prompt under the new id. +async function current($: EngineInterface): Promise { + if (mod.state === null || mod.session !== (await $.session.id())) await refresh($) + + return mod.state +} + +async function distill($: EngineInterface, state: JournalState): Promise { + const task = state.active + if (task === null) return + + const reply = await $.model.fork({ prompt: distillPrompt(task) }) + if (!reply.isAnswered) return + + const session = await $.session.id() + for (const event of parseDistill(reply.text, task.recent)) { + await cli($, [ + 'event', task.task_id, + '--type', event.type, + '--text', event.text, + '--suggested', + '--session', session, + '--origin', 'mod-distill', + ]) + } + + await refresh($) +} + +export const register: Register = (on, options) => { + const nudgeAfter = Number(options.nudge_after_turns ?? 6) + const distillOnCompact = options.distill_on_compact !== false + + on('session.start', async ($, e, next) => { + if ((await $.env.get('TJ_IN_CLASSIFIER')) !== undefined) { + mod.isOff = true + + return next(e) + } + + await refresh($) + if (mod.state !== null) await $.env.set('TJ_MOD_ACTIVE', '1') + + return next(e) + }) + + on('session.end', async ($, e, next) => { + mod.state = null + mod.session = '' + mod.turnsSinceEntry = 0 + mod.workSinceEntry = 0 + + return next(e) + }) + + on('prompt.compose', async ($, e, next) => { + const composed = await next(e) + if (mod.isOff || mod.state === null) return composed + + return { + sections: [...composed.sections, { id: 'task-journal:active', text: sectionText(mod.state), scope: 'session' }], + } + }) + + on('prompt.submit', async ($, e, next) => { + // Only the person's own prompts count as turns: not a background task's + // notification, a peer's message or a plugin's prompt. + const isPersons = e.origin === undefined || e.origin.kind === 'composer' || e.origin.kind === 'bridge' + if (mod.isOff || !isPersons) return next(e) + + const state = await current($) + if (state === null) return next(e) + + mod.turnsSinceEntry += 1 + const isDue = + nudgeAfter > 0 && mod.turnsSinceEntry > nudgeAfter && (state.active !== null || mod.workSinceEntry > 0) + if (!isDue) return next(e) + + mod.turnsSinceEntry = 0 + + return next({ ...e, context: [...(e.context ?? []), nudgeText(state, nudgeAfter)] }) + }) + + on('tool.call', async ($, e, next) => { + const name = journalTool(String(e.tool)) + if (mod.isOff || name === undefined) { + if (WORK_TOOLS.has(String(e.tool))) mod.workSinceEntry += 1 + + return next(e) + } + + // An MCP tool's arguments ride on `e` itself, untyped. + const args = e as unknown as Record + const session = await $.session.id() + const call = WRITE_TOOLS.has(name) && args.session_id === undefined ? { ...e, session_id: session } : e + const ran = await next(call) + if (ran.deny !== undefined || ran.isError === true || !WRITE_TOOLS.has(name)) return ran + + mod.turnsSinceEntry = 0 + mod.workSinceEntry = 0 + await refresh($) + if (name === 'event_add' && typeof args.event_type === 'string') $.ui.toast(`📓 ${args.event_type} recorded`) + + return ran + }) + + on('session.compact', async ($, e, next) => { + if (mod.isOff || e.agentId !== undefined) return next(e) + + const state = await current($) + if (state === null || state.active === null) return next(e) + + if (distillOnCompact) { + try { + await distill($, state) + } catch { + // The compaction matters more than the catch-up: go on without it. + } + } + + const keep = compactInstruction(state.active) + + return next({ ...e, instructions: e.instructions ? `${e.instructions}\n\n${keep}` : keep }) + }) +} From ddc82d3d0e1adf48baca2345eb7671afb78fbd7c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:31:44 +0400 Subject: [PATCH 009/104] fix(mcp): task_search honours status and project, lists tasks on empty query --- crates/tj-mcp/src/main.rs | 310 ++++++++++++++++++++++++++++++++------ 1 file changed, 260 insertions(+), 50 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index fa9aefd..4457f50 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -182,8 +182,12 @@ pub struct TaskPackMetadata { #[derive(Debug, Deserialize, schemars::JsonSchema)] pub struct TaskSearchParams { + /// Full-text query. Empty or absent lists the project's tasks instead. + #[serde(default)] pub query: String, + /// `open`, `closed`, or `any` (the default). pub status: Option, + /// Absolute path of another project directory to search instead of this one. pub project: Option, /// v0.10.3+: restrict matches to a single event type /// (`decision`, `evidence`, `finding`, `rejection`, ...). @@ -193,7 +197,29 @@ pub struct TaskSearchParams { #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskSearchResult { pub query: String, + /// Matching task ids. Kept for older clients; `tasks` has the same ids + /// in the same order with enough detail to pick one. pub results: Vec, + pub tasks: Vec, +} + +#[derive(Debug, Serialize, schemars::JsonSchema)] +pub struct TaskSearchHit { + pub task_id: String, + pub title: String, + pub status: String, + pub last_event_at: String, + pub goal: Option, +} + +fn task_search_hit(r: &rusqlite::Row) -> rusqlite::Result { + Ok(TaskSearchHit { + task_id: r.get(0)?, + title: r.get(1)?, + status: r.get(2)?, + last_event_at: r.get(3)?, + goal: r.get(4)?, + }) } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -449,7 +475,7 @@ impl TaskJournalServer { #[tool( name = "task_search", - description = "Full-text search tasks by query (FTS5)." + description = "Search tasks. `query` is full-text over event text; empty or absent lists the project's tasks, newest first. `status`: open | closed | any (default). `project`: absolute dir of another project. `tasks` gives id, title, status, last_event_at, goal per hit." )] async fn task_search( &self, @@ -459,8 +485,22 @@ impl TaskJournalServer { let query = p.query.clone(); let raw_query = p.query.clone(); let event_type = p.event_type.clone(); - let results = run_blocking(move || { - let (project_hash, events_path, state_path) = project_paths()?; + let tasks = run_blocking(move || { + let status = match p.status.as_deref() { + None | Some("any") => None, + Some(s @ ("open" | "closed")) => Some(s.to_string()), + Some(other) => { + anyhow::bail!("invalid status `{other}` (expected: open | closed | any)") + } + }; + let (project_hash, events_path, state_path) = match p.project.as_deref() { + Some(dir) if !Path::new(dir).is_absolute() => { + anyhow::bail!("project must be an absolute path, got `{dir}`") + } + Some(dir) => resolve_project_paths(Path::new(dir))?, + None => project_paths()?, + }; + let conn_arc = cached_open(&state_path)?; let conn = conn_arc .lock() @@ -469,69 +509,72 @@ impl TaskJournalServer { tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } + // No query: list the project's tasks, newest first. FTS5 + // rejects an empty MATCH, so it must not reach it. + if raw_query.trim().is_empty() { + let mut stmt = conn.prepare( + "SELECT task_id, title, status, last_event_at, goal FROM tasks \ + WHERE (?1 IS NULL OR status = ?1) \ + AND (?2 IS NULL OR task_id IN \ + (SELECT task_id FROM events_index WHERE type = ?2)) \ + ORDER BY last_event_at DESC LIMIT 50", + )?; + let hits = stmt + .query_map(rusqlite::params![status, event_type], task_search_hit)? + .collect::>()?; + + return Ok(hits); + } + // v0.10.3: sanitize FTS5 query. Hyphenated IDs like // `OPS-306` previously crashed with "no such column: 306" // because FTS5 reads `-` as column-prefix syntax. Every // token is quoted on its own, so punctuation is literal // and multi-word queries keep their AND semantics. let fts_query = tj_core::fts::sanitize_query(&raw_query); - let (sql, fts_only) = match &event_type { - Some(_) => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE search_fts MATCH ?1 AND type = ?2 LIMIT 50", - false, - ), - None => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE search_fts MATCH ?1 LIMIT 50", - true, - ), - }; - let mut stmt = conn.prepare(sql)?; - let mut ids: Vec = if fts_only { - stmt.query_map(rusqlite::params![fts_query], |r| r.get::<_, String>(0))? - .collect::>()? - } else { - let ty = event_type.as_deref().unwrap(); - stmt.query_map(rusqlite::params![fts_query, ty], |r| r.get::<_, String>(0))? - .collect::>()? - }; + let mut stmt = conn.prepare( + "SELECT DISTINCT t.task_id, t.title, t.status, t.last_event_at, t.goal \ + FROM search_fts JOIN tasks t ON t.task_id = search_fts.task_id \ + WHERE search_fts MATCH ?1 \ + AND (?2 IS NULL OR search_fts.type = ?2) \ + AND (?3 IS NULL OR t.status = ?3) LIMIT 50", + )?; + let mut hits: Vec = stmt + .query_map( + rusqlite::params![fts_query, event_type, status], + task_search_hit, + )? + .collect::>()?; // v0.10.3: LIKE fallback. FTS5 phrase search miss when // tokenizer split differs from the user's mental model // (e.g. `bulk-repack` in source vs `bulk repack` in // query). On zero FTS hits, scan event text directly so // hyphenated identifiers and partial-word recall work. - if ids.is_empty() { + if hits.is_empty() { let like = tj_core::fts::like_pattern(&raw_query); - let (sql_like, type_bind) = match &event_type { - Some(_) => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE text LIKE ?1 AND type = ?2 LIMIT 50", - true, - ), - None => ( - "SELECT DISTINCT task_id FROM search_fts \ - WHERE text LIKE ?1 LIMIT 50", - false, - ), - }; - let mut stmt_like = conn.prepare(sql_like)?; - ids = if type_bind { - let ty = event_type.as_deref().unwrap(); - stmt_like - .query_map(rusqlite::params![like, ty], |r| r.get::<_, String>(0))? - .collect::>()? - } else { - stmt_like - .query_map(rusqlite::params![like], |r| r.get::<_, String>(0))? - .collect::>()? - }; + let mut stmt_like = conn.prepare( + "SELECT DISTINCT t.task_id, t.title, t.status, t.last_event_at, t.goal \ + FROM search_fts JOIN tasks t ON t.task_id = search_fts.task_id \ + WHERE search_fts.text LIKE ?1 \ + AND (?2 IS NULL OR search_fts.type = ?2) \ + AND (?3 IS NULL OR t.status = ?3) LIMIT 50", + )?; + hits = stmt_like + .query_map(rusqlite::params![like, event_type, status], task_search_hit)? + .collect::>()?; } - Ok(ids) + + Ok(hits) }) .await?; - Ok(Json(TaskSearchResult { query, results })) + + let results = tasks.iter().map(|t| t.task_id.clone()).collect(); + Ok(Json(TaskSearchResult { + query, + results, + tasks, + })) }) .await } @@ -1030,6 +1073,7 @@ mod tests { let search = TaskSearchResult { query: "q".into(), results: vec![], + tasks: vec![], }; assert!(!keys_of(&serde_json::to_value(&search).unwrap()).contains(&"stub".to_string())); @@ -1450,6 +1494,172 @@ mod tests { ); } + async fn create_task(server: &TaskJournalServer, title: &str) -> String { + server + .task_create(Parameters(TaskCreateParams { + title: title.into(), + initial_context: None, + goal: Some(format!("goal of {title}")), + parent: None, + })) + .await + .unwrap() + .0 + .task_id + } + + async fn search( + server: &TaskJournalServer, + query: &str, + status: Option<&str>, + project: Option<&str>, + event_type: Option<&str>, + ) -> Result { + server + .task_search(Parameters(TaskSearchParams { + query: query.into(), + status: status.map(Into::into), + project: project.map(Into::into), + event_type: event_type.map(Into::into), + })) + .await + .map(|j| j.0) + } + + #[test] + fn task_search_params_accept_a_missing_query() { + let p: TaskSearchParams = + serde_json::from_value(serde_json::json!({"status": "open"})).unwrap(); + assert_eq!(p.query, ""); + } + + #[tokio::test] + async fn task_search_empty_query_lists_tasks_filtered_by_status() { + let _env = handler_env(); + let server = TaskJournalServer; + + let open = create_task(&server, "Empty query open").await; + let closed = create_task(&server, "Empty query closed").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: closed.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + })) + .await + .unwrap(); + + let res = search(&server, " ", Some("open"), None, None) + .await + .unwrap(); + assert!(res.results.contains(&open), "{:?}", res.results); + assert!(!res.results.contains(&closed), "{:?}", res.results); + let ids: Vec<_> = res.tasks.iter().map(|t| t.task_id.clone()).collect(); + assert_eq!(ids, res.results, "`tasks` must follow `results` order"); + let hit = res.tasks.iter().find(|t| t.task_id == open).unwrap(); + assert_eq!(hit.title, "Empty query open"); + assert_eq!(hit.status, "open"); + assert_eq!(hit.goal.as_deref(), Some("goal of Empty query open")); + + let res = search(&server, "", Some("closed"), None, None) + .await + .unwrap(); + assert!(res.results.contains(&closed) && !res.results.contains(&open)); + + let res = search(&server, "", None, None, None).await.unwrap(); + assert!(res.results.contains(&closed) && res.results.contains(&open)); + assert!( + res.tasks + .windows(2) + .all(|w| w[0].last_event_at >= w[1].last_event_at), + "newest last_event_at first" + ); + + let res = search(&server, "", Some("any"), None, Some("close")) + .await + .unwrap(); + assert!(res.results.contains(&closed) && !res.results.contains(&open)); + } + + #[tokio::test] + async fn task_search_status_filters_fts_hits_and_rejects_unknown_values() { + let _env = handler_env(); + let server = TaskJournalServer; + + let open = create_task(&server, "Quagga open").await; + let closed = create_task(&server, "Quagga closed").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: closed.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + })) + .await + .unwrap(); + + let res = search(&server, "quagga", Some("open"), None, None) + .await + .unwrap(); + assert_eq!(res.results, vec![open.clone()]); + assert_eq!(res.tasks[0].title, "Quagga open"); + + let res = search(&server, "quagga", Some("closed"), None, None) + .await + .unwrap(); + assert_eq!(res.results, vec![closed.clone()]); + + let res = search(&server, "quagga", None, None, None).await.unwrap(); + assert_eq!(res.results.len(), 2); + + let err = search(&server, "quagga", Some("pending"), None, None) + .await + .expect_err("unknown status must be rejected"); + assert!(err.message.contains("status"), "{}", err.message); + } + + #[tokio::test] + async fn task_search_project_searches_that_projects_journal() { + let _env = handler_env(); + let server = TaskJournalServer; + + let other = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(other.path().join(".git")).unwrap(); + let (_, other_events, _) = resolve_project_paths(other.path()).unwrap(); + std::fs::create_dir_all(other_events.parent().unwrap()).unwrap(); + let task_id = tj_core::new_task_id(); + let mut ev = tj_core::event::Event::new( + task_id.clone(), + tj_core::event::EventType::Open, + tj_core::event::Author::Agent, + tj_core::event::Source::Chat, + "Elsewhere quokka".into(), + ); + ev.meta = serde_json::json!({"title": "Elsewhere quokka"}); + let mut writer = tj_core::storage::JsonlWriter::open(&other_events).unwrap(); + writer.append(&ev).unwrap(); + writer.flush_durable().unwrap(); + let other_dir = other.path().to_str().unwrap(); + + let res = search(&server, "quokka", None, Some(other_dir), None) + .await + .unwrap(); + assert_eq!(res.results, vec![task_id.clone()]); + + let res = search(&server, "", Some("open"), Some(other_dir), None) + .await + .unwrap(); + assert_eq!(res.results, vec![task_id.clone()]); + + let res = search(&server, "quokka", None, None, None).await.unwrap(); + assert!(!res.results.contains(&task_id), "{:?}", res.results); + + assert!(search(&server, "", None, Some("relative/dir"), None) + .await + .is_err()); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full From a75fb0deab31a1b3d8dcd382689475c7f5160f9e Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:32:27 +0400 Subject: [PATCH 010/104] fix(hooks): mark codex classifier child with TJ_IN_CLASSIFIER and silence nudge inside it --- crates/tj-cli/src/main.rs | 6 +++ crates/tj-cli/tests/cli.rs | 16 ++++++ crates/tj-core/src/llm.rs | 100 +++++++++++++++++++++++++++++++------ 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 571a696..3ce7473 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -4246,6 +4246,12 @@ fn count_session_events_tail(path: &std::path::Path, sid: &str, tail_lines: usiz /// substantial work but logged little — escalate. All signals are cheap (a file /// size + a tail scan); no model, never blocks the prompt. fn run_nudge() -> anyhow::Result<()> { + // Recursion guard, same as recall-hook: never inject into our own + // classifier child (`claude -p` / `codex exec` re-run the user's hooks). + if std::env::var(tj_core::classifier::agent_sdk::IN_CLASSIFIER_ENV).is_ok() { + return Ok(()); + } + let mut ctx = NUDGE_BASE.to_string(); let escalation = (|| -> Option { use std::io::{IsTerminal, Read}; diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index c768714..63f879a 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1966,6 +1966,22 @@ fn ingest_hook_short_circuits_when_in_classifier_env_set() { ); } +#[test] +fn nudge_is_silent_inside_a_classifier_child() { + // A `claude -p` / `codex exec` spawned by the classifier re-runs the + // user's UserPromptSubmit hooks; the nudge must not inject into it. + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_IN_CLASSIFIER", "1") + .args(["nudge"]) + .write_stdin("") + .assert() + .success() + .stdout(""); +} + #[test] fn ingest_hook_reads_user_prompt_submit_payload_from_stdin() { // Real Claude Code passes hook input as JSON over stdin, NOT via env diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs index c6f49a6..f96498b 100644 --- a/crates/tj-core/src/llm.rs +++ b/crates/tj-core/src/llm.rs @@ -217,7 +217,10 @@ impl LlmBackend for CodexCliBackend { .args(["--sandbox", "read-only"]) .args(["--color", "never"]) .arg("-o") - .arg(&out_path); + .arg(&out_path) + // `codex exec` runs the user's ~/.codex hooks; the marker makes our + // own `ingest-hook` no-op inside this child instead of recursing. + .env(crate::classifier::agent_sdk::IN_CLASSIFIER_ENV, "1"); if let Some(model) = &self.model { cmd.args(["-m", model]); } @@ -225,22 +228,27 @@ impl LlmBackend for CodexCliBackend { // transcript-sized prompt away from the argv size limit. cmd.arg("-"); - let mut child = cmd - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn `codex` (is the Codex CLI installed and on PATH?)")?; - child - .stdin - .take() - .context("codex stdin was not captured")? - .write_all(prompt.as_bytes()) - .context("failed to write prompt to codex stdin")?; - - let output = crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout())?; + // Run inside a closure so the output file is removed on every exit + // path — a failed stdin write and a timeout included. + let mut run = || -> anyhow::Result { + let mut child = cmd + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("failed to spawn `codex` (is the Codex CLI installed and on PATH?)")?; + child + .stdin + .take() + .context("codex stdin was not captured")? + .write_all(prompt.as_bytes()) + .context("failed to write prompt to codex stdin")?; + crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout()) + }; + let output = run(); let answer = std::fs::read_to_string(&out_path).unwrap_or_default(); let _ = std::fs::remove_file(&out_path); + let output = output?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -510,6 +518,68 @@ mod tests { } } + /// A stand-in `codex` that writes `TJ_IN_CLASSIFIER` into its `-o` file, + /// then sleeps `FAKE_CODEX_SLEEP` seconds when that is set. + #[cfg(unix)] + fn fake_codex_dir() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + let script = dir.path().join("codex"); + std::fs::write( + &script, + "#!/bin/sh\nout=\"\"\nwhile [ $# -gt 0 ]; do\n if [ \"$1\" = \"-o\" ]; then out=\"$2\"; shift; fi\n shift\ndone\ncat > /dev/null\nprintf 'in_classifier=%s' \"$TJ_IN_CLASSIFIER\" > \"$out\"\nif [ -n \"$FAKE_CODEX_SLEEP\" ]; then exec sleep \"$FAKE_CODEX_SLEEP\"; fi\n", + ) + .unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + dir + } + + #[cfg(unix)] + fn path_with(dir: &std::path::Path) -> EnvGuard { + let old = std::env::var("PATH").unwrap_or_default(); + EnvGuard::set("PATH", &format!("{}:{old}", dir.display())) + } + + /// `codex exec` runs the user's ~/.codex hooks; without the marker a + /// `dream`/`complete` run re-enters `ingest-hook` from inside itself. + #[cfg(unix)] + #[test] + fn codex_child_runs_with_the_recursion_marker() { + let _l = ENV_LOCK.lock().unwrap(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _marker = EnvGuard::unset("TJ_IN_CLASSIFIER"); + let _sleep = EnvGuard::unset("FAKE_CODEX_SLEEP"); + + let (answer, _) = CodexCliBackend { model: None } + .complete_usage("classify me", 64) + .unwrap(); + + assert_eq!(answer, "in_classifier=1"); + } + + #[cfg(unix)] + #[test] + fn codex_output_file_is_removed_when_the_call_times_out() { + let _l = ENV_LOCK.lock().unwrap(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); + let _sleep = EnvGuard::set("FAKE_CODEX_SLEEP", "5"); + + let res = CodexCliBackend { model: None }.complete_usage("classify me", 64); + + assert!(res.is_err(), "a wedged codex must time out"); + let prefix = format!("task-journal-codex-{}-", std::process::id()); + let left: Vec<_> = std::fs::read_dir(std::env::temp_dir()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix)) + .collect(); + assert!(left.is_empty(), "codex temp file leaked: {left:?}"); + } + #[test] fn unknown_backend_errors() { let _l = ENV_LOCK.lock().unwrap(); From 1c25888b24860b91d11639c53da78dbc8b54be98 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:33:01 +0400 Subject: [PATCH 011/104] fix(mcp): task_close harvests the project dir and records outcome via the close event --- crates/tj-mcp/src/main.rs | 115 ++++++++++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 16 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 4457f50..af8d3c0 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -344,12 +344,16 @@ fn resolve_project_paths( Ok((project_hash, events, state)) } -fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { - let dir = match PROJECT_DIR_OVERRIDE.get() { +/// The project directory every tool works on: `--project-dir`, else the cwd. +fn project_dir() -> anyhow::Result { + Ok(match PROJECT_DIR_OVERRIDE.get() { Some(p) => p.clone(), None => std::env::current_dir()?, - }; - resolve_project_paths(&dir) + }) +} + +fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { + resolve_project_paths(&project_dir()?) } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -439,10 +443,7 @@ impl TaskJournalServer { &p.task_id, tj_core::completeness::pending_count(), )?; - let dir = match PROJECT_DIR_OVERRIDE.get() { - Some(d) => d.clone(), - None => std::env::current_dir()?, - }; + let dir = project_dir()?; let arts = tj_core::db::task_artifacts(&conn, &p.task_id)?; report .gaps @@ -813,10 +814,11 @@ impl TaskJournalServer { if !tj_core::db::task_exists(&conn, &p.task_id)? { anyhow::bail!("task not found: {}", p.task_id); } - // v0.6.0: validate outcome_tag enum and persist - // outcome+tag to the task row before writing the - // close event. Same enum + same ordering as the - // CLI close handler — keep them lockstep. + // v0.6.0: validate the outcome_tag enum before writing + // the close event (same enum as the CLI close handler). + // outcome+tag ride in the close event's meta and reach + // the task row only when that event is ingested, so a + // failed append never leaves an open task with an outcome. if let Some(tag) = p.outcome_tag.as_deref() { match tag { "done" | "abandoned" | "superseded" => {} @@ -825,9 +827,6 @@ impl TaskJournalServer { ), } } - if let Some(o) = p.outcome.as_deref() { - tj_core::db::set_task_outcome(&conn, &p.task_id, o, p.outcome_tag.as_deref())?; - } open_kids = tj_core::db::count_open_children(&conn, &p.task_id)?; } // release the connection lock before doing the JSONL append @@ -850,7 +849,7 @@ impl TaskJournalServer { // (commit, branch, PR) into the close event so the resume pack // reads as a clickable ledger of what shipped. Best-effort and // structured (merged in db::index_event) — never fails close. - if let Ok(dir) = std::env::current_dir() { + if let Ok(dir) = project_dir() { let arts = tj_core::harvest::harvest(&dir); if !arts.is_empty() { if let Ok(v) = serde_json::to_value(&arts) { @@ -1660,6 +1659,90 @@ mod tests { .is_err()); } + #[tokio::test] + async fn task_close_harvests_the_project_dir_not_the_cwd() { + let _env = handler_env(); + let server = TaskJournalServer; + + // The test binary runs inside this repo's checkout, while the project + // dir is a temp dir. A close must never stamp the cwd repo's commit. + let cwd_commit = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()); + let Some(cwd_commit) = cwd_commit else { + return; // not run from a git checkout: nothing to tell apart + }; + + let task = create_task(&server, "Harvest dir").await; + server + .task_close(Parameters(TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + })) + .await + .unwrap(); + + let (_, events_path, _) = project_paths().unwrap(); + let jsonl = std::fs::read_to_string(&events_path).unwrap(); + let close = jsonl + .lines() + .map(|l| serde_json::from_str::(l).unwrap()) + .find(|v| v["task_id"] == task.as_str() && v["type"] == "close") + .expect("close event in jsonl"); + assert!( + !close["meta"].to_string().contains(&cwd_commit), + "close harvested the cwd repo ({cwd_commit}): {}", + close["meta"] + ); + } + + #[tokio::test] + async fn task_close_records_outcome_only_through_the_close_event() { + let _env = handler_env(); + let server = TaskJournalServer; + + let task = create_task(&server, "Append fails").await; + let close = |outcome: &str| TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: Some(outcome.into()), + outcome_tag: Some("done".into()), + }; + + // Make the close append fail: the journal is read-only. + let (_, events_path, state_path) = project_paths().unwrap(); + let writable = std::fs::metadata(&events_path).unwrap().permissions(); + let mut read_only = writable.clone(); + read_only.set_readonly(true); + std::fs::set_permissions(&events_path, read_only).unwrap(); + let res = server.task_close(Parameters(close("must not stick"))).await; + std::fs::set_permissions(&events_path, writable).unwrap(); + assert!(res.is_err(), "append to a read-only journal must fail"); + + let conn_arc = cached_open(&state_path).unwrap(); + { + let conn = conn_arc.lock().unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.outcome, None, "failed close left an outcome behind"); + let status = tj_core::db::task_status(&conn, &task).unwrap(); + assert_eq!(status.as_deref(), Some("open")); + } + + server + .task_close(Parameters(close("shipped"))) + .await + .unwrap(); + let conn = conn_arc.lock().unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.outcome.as_deref(), Some("shipped")); + assert_eq!(meta.outcome_tag.as_deref(), Some("done")); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full From c44a500fadbedf6de796bcba4d9a4b879134e600 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:33:46 +0400 Subject: [PATCH 012/104] fix(hooks): classify-worker lock treats a fresh empty lockfile and EPERM as held --- crates/tj-cli/src/main.rs | 41 ++++++++++++++++++----- crates/tj-cli/tests/cli.rs | 68 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 3ce7473..b304f23 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -5498,6 +5498,9 @@ fn spawn_classify_worker(backend: &str) -> anyhow::Result<()> { Ok(()) } +/// How long an empty (pid not yet written) worker lockfile counts as held. +const LOCK_PID_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + /// File-lock guard for the classify-worker. Holds the lockfile until /// dropped; ensures cleanup on panic. One worker per project_hash. struct WorkerLock { @@ -5525,16 +5528,25 @@ impl WorkerLock { } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { // Inspect existing lockfile. If PID is alive → another - // worker is running; back off. If dead/missing → - // remove stale file and retry. + // worker is running; back off. If dead → remove stale + // file and retry. let body = std::fs::read_to_string(&path).unwrap_or_default(); - let pid: Option = body.trim().parse().ok(); - if let Some(pid) = pid { - if pid_is_alive(pid) { - return Ok(None); + match body.trim().parse::() { + Ok(pid) if pid_is_alive(pid) => return Ok(None), + Ok(_) => {} + Err(_) => { + // No PID yet: the holder may sit between + // `create_new` and the pid write. Only a lock + // that stayed empty past the grace is stale. + let fresh = std::fs::metadata(&path) + .and_then(|m| m.modified()) + .map(|t| t.elapsed().unwrap_or_default() < LOCK_PID_GRACE) + .unwrap_or(false); + if fresh { + return Ok(None); + } } } - // Stale (no PID, or dead PID) — remove and retry. let _ = std::fs::remove_file(&path); continue; } @@ -5554,7 +5566,12 @@ impl Drop for WorkerLock { fn pid_is_alive(pid: u32) -> bool { // kill(pid, 0) probes existence without sending a signal. // SAFETY: libc::kill is a thin syscall wrapper, no aliasing concerns. - unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } + if unsafe { libc::kill(pid as libc::pid_t, 0) } == 0 { + return true; + } + + // EPERM: the process exists but belongs to another user. + std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) } #[cfg(not(unix))] @@ -6338,6 +6355,14 @@ mod inline_tests { ); } + /// PID 1 always exists; for a non-root user `kill(1, 0)` fails with EPERM, + /// which means "alive, not ours" — never "dead". + #[cfg(unix)] + #[test] + fn pid_is_alive_treats_eperm_as_alive() { + assert!(pid_is_alive(1)); + } + #[test] fn persist_pending_v2_omits_session_id_when_none() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 63f879a..8d7460b 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2882,6 +2882,74 @@ fn classify_worker_respects_existing_lock() { ); } +/// The lock is created empty and the pid is written a moment later. A +/// second worker that reads it in between must treat it as held, not as a +/// stale lock to delete and steal. An empty lock that has sat for a while +/// is a crashed worker's leftover and is taken over. +#[test] +fn classify_worker_treats_fresh_empty_lock_as_held() { + let dir = assert_fs::TempDir::new().unwrap(); + let cwd = std::env::current_dir().unwrap(); + let project_hash = tj_core::project_hash::from_path(&cwd).expect("compute project hash"); + + let pending = dir.path().join("task-journal").join("pending"); + std::fs::create_dir_all(&pending).unwrap(); + let events_path = dir + .path() + .join("task-journal") + .join("events") + .join(format!("{project_hash}.jsonl")); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let entry = pending.join("01emptylock.json"); + std::fs::write( + &entry, + serde_json::json!({ + "schema": "v2", + "kind": "PostToolUse", + "text": "empty lock marker", + "project_hash": project_hash, + "events_path": events_path.to_string_lossy(), + "backend": "heuristic", + "queued_at": "2026-05-08T00:00:00Z", + }) + .to_string(), + ) + .unwrap(); + + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + let lock_path = state.join(format!("classifier-{project_hash}.lock")); + std::fs::write(&lock_path, "").unwrap(); + + let worker = || { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + }; + + worker(); + assert!( + entry.exists(), + "a fresh empty lock is held — entry must stay" + ); + assert!(lock_path.exists(), "a fresh empty lock must not be deleted"); + + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60); + std::fs::File::options() + .write(true) + .open(&lock_path) + .unwrap() + .set_modified(old) + .unwrap(); + + worker(); + assert!(!entry.exists(), "a stale empty lock is taken over"); + assert!(!lock_path.exists(), "the new holder releases the lock"); +} + // ===================================================================== // v0.7.0: statusline / PreCompact / /rewind / rejected / export-pr // ===================================================================== From 2d162b2d19878cf0cc7747943cef3810795018ba Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:33:48 +0400 Subject: [PATCH 013/104] fix(mcp): event_add and artifact_add reject an unknown task_id --- crates/tj-mcp/src/main.rs | 80 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index af8d3c0..dd45895 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -356,6 +356,28 @@ fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::Pat resolve_project_paths(&project_dir()?) } +/// Ingest the journal tail, then fail on an unknown `task_id` so a typo never +/// writes an orphan event. +fn require_task( + project_hash: &str, + events_path: &Path, + state_path: &Path, + task_id: &str, +) -> anyhow::Result<()> { + let conn_arc = cached_open(state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + } + if !tj_core::db::task_exists(&conn, task_id)? { + anyhow::bail!("task not found: {task_id}"); + } + + Ok(()) +} + #[derive(Debug, Deserialize, schemars::JsonSchema)] pub struct TaskCheckParams { pub task_id: String, @@ -700,7 +722,7 @@ impl TaskJournalServer { ) -> Result, McpError> { traced_tool("event_add", async move { run_blocking(move || { - let (_, events_path, _) = project_paths()?; + let (project_hash, events_path, state_path) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; let event_type = parse_event_type(&p.event_type)?; @@ -713,6 +735,8 @@ impl TaskJournalServer { p.event_type ); } + require_task(&project_hash, &events_path, &state_path, &p.task_id)?; + let mut event = tj_core::event::Event::new( &p.task_id, event_type, @@ -758,8 +782,9 @@ impl TaskJournalServer { ) -> Result, McpError> { traced_tool("artifact_add", async move { run_blocking(move || { - let (_, events_path, _) = project_paths()?; + let (project_hash, events_path, state_path) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; + require_task(&project_hash, &events_path, &state_path, &p.task_id)?; let mut event = tj_core::event::Event::new( &p.task_id, @@ -1743,6 +1768,57 @@ mod tests { assert_eq!(meta.outcome_tag.as_deref(), Some("done")); } + #[tokio::test] + async fn event_add_and_artifact_add_reject_an_unknown_task() { + let _env = handler_env(); + let server = TaskJournalServer; + + create_task(&server, "Known task").await; + let (_, events_path, _) = project_paths().unwrap(); + let before = std::fs::read_to_string(&events_path).unwrap(); + + let res = server + .event_add(Parameters(EventAddParams { + task_id: "tj-typo000000".into(), + event_type: "finding".into(), + text: "orphan".into(), + corrects: None, + supersedes: None, + alternatives: None, + })) + .await; + let err = match res { + Ok(_) => panic!("event_add on an unknown task must fail"), + Err(e) => e, + }; + assert!( + err.message.contains("task not found: tj-typo000000"), + "{}", + err.message + ); + + let res = server + .artifact_add(Parameters(ArtifactAddParams { + task_id: "tj-typo000000".into(), + kind: "doc".into(), + url: "https://example.com/spec".into(), + label: "Spec".into(), + })) + .await; + let err = match res { + Ok(_) => panic!("artifact_add on an unknown task must fail"), + Err(e) => e, + }; + assert!( + err.message.contains("task not found: tj-typo000000"), + "{}", + err.message + ); + + let after = std::fs::read_to_string(&events_path).unwrap(); + assert_eq!(after, before, "no orphan event may reach the journal"); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full From 2c5238158ec0b747831f7b07f66b008f6391ddcb Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:34:28 +0400 Subject: [PATCH 014/104] docs(mcp): describe task_search status filter, empty query and tasks field --- crates/tj-mcp/src/main.rs | 7 ++++--- plugin/skills/task-journal/SKILL.md | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index dd45895..fbef18f 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -131,9 +131,10 @@ tools — so call them, every session, without being asked. THE RITUAL — do this on EVERY coding session, not optional: -1. START. task_search(status="open") for related work → task_pack to resume it; - if nothing fits, task_create(title, goal=). Hold the returned task_id for the whole session. +1. START. task_search(status="open") lists open tasks, newest first (add a + `query` to narrow; each entry in `tasks` has title + goal) → task_pack to + resume one; if nothing fits, task_create(title, goal=). Hold the returned task_id for the whole session. 2. AT THE MOMENT you commit to an approach → event_add(event_type="decision", ...) and pass `alternatives` (the options you weighed). Right then — not at the end, or you will forget. diff --git a/plugin/skills/task-journal/SKILL.md b/plugin/skills/task-journal/SKILL.md index d4e4029..9087e9a 100644 --- a/plugin/skills/task-journal/SKILL.md +++ b/plugin/skills/task-journal/SKILL.md @@ -23,8 +23,9 @@ treat it as if it does nothing and record explicitly. ## Session-start ritual (do this before real work) -1. `task_search(query=, status="open")` — is there an open - task for this? If yes, `task_pack(task_id)` and continue it. **Do not** open a duplicate. +1. `task_search(status="open")` (add `query=` to narrow) — is + there an open task for this? Each entry in `tasks` shows its title and goal. If yes, + `task_pack(task_id)` and continue it. **Do not** open a duplicate. 2. If nothing fits, `task_create(title=, goal=)`. **Always pass `goal`** — it is the first line of every pack and the anchor for "why was this done?". 3. Hold the returned `task_id` for the whole task. One task = one logical objective. @@ -100,7 +101,11 @@ task_close( | Resuming an existing project | `task_search(status="open")` → `task_pack` on the match | `task_pack` mode: `compact` (~2KB, default for resume) or `full` (~10KB, full trail). -`task_search` filters: `status`, `project`, `event_type` (decision/finding/evidence/…). +`task_search` filters: `status` (`open` | `closed` | `any`, default `any`), `project` +(absolute path of another project directory), `event_type` (decision/finding/evidence/…). +An empty or absent `query` lists the tasks, newest activity first. The result has +`results` (task ids) and `tasks` — the same ids in the same order, each with `title`, +`status`, `last_event_at` and `goal`. ## The 5 MCP tools (exact params) @@ -110,7 +115,8 @@ task_close( correction | reopen | supersede | redirect. `alternatives` is decision-only. - `task_close(task_id, reason, outcome?, outcome_tag?)` — **always pass `outcome` + `outcome_tag`.** - `task_pack(task_id, mode?)` — `compact` | `full`. -- `task_search(query, status?, project?, event_type?)` — FTS5 over this project's events. +- `task_search(query?, status?, project?, event_type?)` — full-text search over the project's + events; no `query` lists its tasks. Returns `results` (ids) and `tasks` (id, title, status, goal). ## Invariants From cc2664a063823120c7441f4942a4d3af9db6af6e Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:35:48 +0400 Subject: [PATCH 015/104] fix(hooks): drop dead FileChanged handler, watchPaths emission and asyncRewake backlog exit --- crates/tj-cli/src/main.rs | 154 +------------------------------ crates/tj-cli/tests/cli.rs | 182 +++++-------------------------------- 2 files changed, 26 insertions(+), 310 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index b304f23..10e7f21 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -376,9 +376,7 @@ fn run_pending_list() -> Result<()> { .take(72) .collect(); let attempts = v.get("attempts").and_then(|x| x.as_u64()).unwrap_or(0) as u32; - let dead_marker = if id.ends_with(".dead") { " [DEAD]" } else { "" }; entries.push((id, queued_at, text_preview, attempts)); - let _ = dead_marker; } if entries.is_empty() { println!("(no pending entries)"); @@ -2370,33 +2368,6 @@ runs in the background and won't block you; it only fills gaps and never closes bundle.push_str("\n\n"); } - // v0.10.2 X4: emit `watchPaths` so Claude Code starts - // monitoring our marker files (CLAUDE.md, README.md, - // .docs/plans). When any of them changes, Claude Code - // fires a FileChanged hook event — our ingest-hook - // handler below treats those as `evidence` entries on - // the active task so the journal captures - // "instructions were updated mid-session" without the - // user manually logging it. Only paths that exist at - // SessionStart time are emitted (no point watching a - // non-existent file — Claude Code logs `watcher error` - // and gives up on it). Gated by TJ_WATCH_PATHS=0. - let allow_watch_paths = std::env::var("TJ_WATCH_PATHS").as_deref() != Ok("0"); - let watch_candidates = [ - cwd.join("CLAUDE.md"), - cwd.join("README.md"), - cwd.join(".docs").join("plans"), - ]; - let watch_paths: Vec = if allow_watch_paths { - watch_candidates - .iter() - .filter(|p| p.exists()) - .map(|p| p.to_string_lossy().to_string()) - .collect() - } else { - Vec::new() - }; - // We deliberately DO NOT emit `sessionTitle` or // `initialUserMessage` here. The v0.10.1 X2 experiment set // `sessionTitle` to "TJ — ( open)", which @@ -2408,18 +2379,10 @@ runs in the background and won't block you; it only fills gaps and never closes // title. The resume context the model actually needs already // rides in `additionalContext`; the tab label belongs to // Claude Code, not to us. (0.14.3) - let mut hook_specific = serde_json::json!({ + let hook_specific = serde_json::json!({ "hookEventName": "SessionStart", "additionalContext": bundle.trim_end(), }); - if !watch_paths.is_empty() { - hook_specific["watchPaths"] = serde_json::Value::Array( - watch_paths - .into_iter() - .map(serde_json::Value::String) - .collect(), - ); - } let envelope = serde_json::json!({ "hookSpecificOutput": hook_specific, }); @@ -2427,65 +2390,6 @@ runs in the background and won't block you; it only fills gaps and never closes return Ok(()); } - // v0.10.2 X4: FileChanged. Claude Code 2.1.x fires this - // event whenever a path in `watchPaths` (emitted on - // SessionStart) changes. Payload: { file_path, event: - // "change"|"add"|"unlink" }. We translate it into an - // `evidence` event on the active task — captures - // "the user/agent edited CLAUDE.md mid-session" without - // anyone typing anything. Schema verified in 2.1.160: - // `literal("FileChanged"), file_path: y.string(), event: - // y.enum(["change","add","unlink"])`. - // - // No active task → drop silently (we're not opening a - // task just because a watched file moved). No events_path - // → ditto, fresh project. - if kind == "FileChanged" { - if !events_path.exists() { - return Ok(()); - } - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; - let Some(tc) = recent.into_iter().next() else { - return Ok(()); - }; - let file_path = payload - .get("file_path") - .and_then(|v| v.as_str()) - .unwrap_or("(unknown)"); - let change = payload - .get("event") - .and_then(|v| v.as_str()) - .unwrap_or("change"); - // Trim noisy absolute paths to project-relative when - // possible — the journal is per-project so the prefix - // is redundant and just steals tokens from the pack. - let display_path = cwd - .to_str() - .and_then(|c| file_path.strip_prefix(c)) - .map(|s| s.trim_start_matches('/').to_string()) - .unwrap_or_else(|| file_path.to_string()); - let evidence_text = format!("FileChanged ({change}): {display_path}"); - let mut event = tj_core::event::Event::new( - &tc.task_id, - tj_core::event::EventType::Evidence, - tj_core::event::Author::Classifier, - tj_core::event::Source::Hook, - evidence_text, - ); - event.confidence = Some(0.9); - event.status = tj_core::event::EventStatus::Confirmed; - tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); - let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - println!("{}", event.event_id); - return Ok(()); - } - // PostModelSwitch (Claude Code 2.1.251+). Payload: { from_model, // to_model, source: "user"|"auto"|"resume", requested_model }. // Which model did the work is a fact about the task that neither @@ -2493,7 +2397,7 @@ runs in the background and won't block you; it only fills gaps and never closes // "why did this stretch come out shallow" is usually answered by // "it ran on a fallback model". Recorded as a `constraint`: it's an // external condition the work happened under, not a decision. - // No active task → drop silently, same rule as FileChanged. + // No active task → drop silently. if kind == "PostModelSwitch" { if !events_path.exists() { return Ok(()); @@ -2829,25 +2733,6 @@ runs in the background and won't block you; it only fills gaps and never closes // a failure to spawn just means the entry sits in // pending/ until the next hook fires another spawn. let _ = spawn_classify_worker(&backend); - - // v0.10.0 asyncRewake backlog signal. Only the PostToolUse - // hook runs as asyncRewake (hooks.json sets TJ_ASYNC_REWAKE=1 - // there), so other kinds — and direct CLI invocations — - // never exit 2 even on overflow. Exit code 2 from a sync - // hook would BLOCK the operation; only asyncRewake hooks - // treat code 2 as "wake the model with rewakeMessage". stdout - // is appended to the wake message, so the user sees the - // drain command without us reaching into stderr. - let allow_wake = std::env::var("TJ_ASYNC_REWAKE").as_deref() == Ok("1"); - if allow_wake && kind == "PostToolUse" { - let pending_count = count_pending_entries(&events_path).unwrap_or(0); - if pending_count > PENDING_OVERFLOW_THRESHOLD { - println!( - "Task Journal pending queue: {pending_count} entries. Classifier behind — run `task-journal pending-gc --days 0` to drain.", - ); - std::process::exit(2); - } - } return Ok(()); } @@ -5346,41 +5231,6 @@ fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyh /// v0.6.2: queue an ingest event for the detached classify-worker. The /// hook returns immediately after writing this entry so it does not /// block Claude Code's hook timeout (was 5-30s, now <100ms). Schema "v2" -/// Threshold for the v0.10.0 asyncRewake backlog signal. When the -/// PostToolUse hook (configured with `asyncRewake: true` in -/// `hooks.json`) finds more than this many entries already queued -/// in `pending/`, it exits with code 2 to wake the model with a -/// system reminder pointing at `task-journal pending-gc`. Tuned so -/// that normal load (<5 in-flight at any moment) never trips, but -/// a stuck classifier surfaces visibly before the queue grows into -/// the hundreds (the v0.6.2 fork-bomb era saw 515 entries before a -/// user noticed). -const PENDING_OVERFLOW_THRESHOLD: usize = 25; - -/// Count `.json` (and `.json.dead`) entries currently sitting in -/// `pending/` next to `events_path`. Best-effort: any IO error -/// returns 0 so a borked filesystem never wakes the model with -/// noise. Used by the asyncRewake backlog signal. -fn count_pending_entries(events_path: &std::path::Path) -> anyhow::Result { - let dir = events_path - .parent() - .and_then(|p| p.parent()) - .ok_or_else(|| anyhow::anyhow!("events_path has no grandparent"))? - .join("pending"); - if !dir.exists() { - return Ok(0); - } - let mut count = 0usize; - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if let Some("json") = path.extension().and_then(|e| e.to_str()) { - count += 1; - } - } - Ok(count) -} - /// distinguishes async-ingest entries from legacy v1 (text+error) ones /// the `pending retry` path knows how to handle. fn persist_pending_v2( diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 8d7460b..a598ceb 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1645,8 +1645,8 @@ fn ingest_hook_session_start_emits_resume_pack_json() { #[test] fn session_start_emits_neither_session_title_nor_initial_message() { - // 0.14.3: the SessionStart envelope carries ONLY additionalContext - // (+ optional watchPaths). It must never set sessionTitle (which + // 0.14.3: the SessionStart envelope carries ONLY additionalContext. + // It must never set sessionTitle (which // overrode Claude Code's own session name with our task id) nor // initialUserMessage (which seeded garbage "[Task Journal resumed: …]" // task titles). @@ -3639,12 +3639,9 @@ fn export_pr_unknown_task_id_exits_one_with_stderr_message() { .stderr(contains("task not found: tj-zzzz")); } -// v0.10.0: asyncRewake backlog signal. PostToolUse hook configured with -// asyncRewake:true in hooks.json sets TJ_ASYNC_REWAKE=1; when pending/ -// has more than PENDING_OVERFLOW_THRESHOLD (25) entries already queued, -// ingest-hook exits 2 with a wake-message on stdout. Sync hooks (or -// CLI invocations without the env var) must NEVER exit 2 — that would -// block the operation in Claude Code's hook contract. +// The PostToolUse hook used to exit 2 on a large pending backlog to wake the +// model (TJ_ASYNC_REWAKE). Nothing installs that env var, so the signal was +// dead code; a backlog must never turn the hook into a non-zero exit. fn seed_pending_chunks(pending_dir: &std::path::Path, count: usize) { std::fs::create_dir_all(pending_dir).unwrap(); for i in 0..count { @@ -3676,82 +3673,19 @@ fn posttooluse_payload() -> String { } #[test] -fn session_start_emits_watch_paths_for_existing_marker_files() { - // v0.10.2 X4: SessionStart envelope must include `watchPaths` with - // existing marker files (CLAUDE.md, README.md, .docs/plans). Files - // that don't exist are skipped — Claude Code's watcher logs an - // error and gives up on missing paths, so we don't emit them. +fn session_start_does_not_emit_watch_paths() { + // install-hooks never registers FileChanged, so asking Claude Code to + // watch marker files only made it watch them for nothing. let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(workdir.join(".docs").join("plans")).unwrap(); std::fs::write(workdir.join("CLAUDE.md"), "# Project rules").unwrap(); - // README.md intentionally absent — must NOT appear in watchPaths. - - let task_id = String::from_utf8( - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["create", "Watch paths test"]) - .assert() - .success() - .get_output() - .stdout - .clone(), - ) - .unwrap() - .trim() - .to_string(); - let _ = task_id; - - let body = String::from_utf8( - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["ingest-hook", "--kind", "SessionStart", "--text", ""]) - .assert() - .success() - .get_output() - .stdout - .clone(), - ) - .unwrap(); - let v: serde_json::Value = serde_json::from_str(body.trim()).unwrap(); - let watches = v["hookSpecificOutput"]["watchPaths"] - .as_array() - .expect("watchPaths must be present when at least one marker exists"); - let joined: String = watches - .iter() - .filter_map(|x| x.as_str()) - .collect::>() - .join("|"); - assert!( - joined.contains("CLAUDE.md"), - "CLAUDE.md must be watched: {joined}" - ); - assert!( - joined.contains("plans"), - ".docs/plans must be watched: {joined}" - ); - assert!( - !joined.contains("README.md"), - "README.md does not exist, must NOT be watched: {joined}" - ); -} - -#[test] -fn session_start_omits_watch_paths_when_disabled_via_env() { - let dir = assert_fs::TempDir::new().unwrap(); - let workdir = dir.path().join("proj"); - std::fs::create_dir_all(&workdir).unwrap(); - std::fs::write(workdir.join("CLAUDE.md"), "# Project rules").unwrap(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "Watch paths env-disabled"]) + .args(["create", "Watch paths test"]) .assert() .success(); @@ -3759,7 +3693,6 @@ fn session_start_omits_watch_paths_when_disabled_via_env() { Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) - .env("TJ_WATCH_PATHS", "0") .current_dir(&workdir) .args(["ingest-hook", "--kind", "SessionStart", "--text", ""]) .assert() @@ -3772,19 +3705,18 @@ fn session_start_omits_watch_paths_when_disabled_via_env() { let v: serde_json::Value = serde_json::from_str(body.trim()).unwrap(); assert!( v["hookSpecificOutput"]["watchPaths"].is_null(), - "TJ_WATCH_PATHS=0 must suppress watchPaths emission" + "SessionStart must not emit watchPaths: {body}" ); assert!( v["hookSpecificOutput"]["additionalContext"].is_string(), - "additionalContext still emitted independently of watchPaths" + "resume context is still emitted: {body}" ); } #[test] -fn file_changed_hook_appends_evidence_to_active_task() { - // v0.10.2 X4: FileChanged hook handler should append an evidence - // event to the most-recent open task with the changed path - // (trimmed project-relative) and the change kind. +fn file_changed_payload_is_ignored() { + // No FileChanged handler any more: the payload carries no text, so the + // hook writes nothing even with an open task. let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); @@ -3794,7 +3726,7 @@ fn file_changed_hook_appends_evidence_to_active_task() { .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "FileChanged evidence test"]) + .args(["create", "Watched file ignored"]) .assert() .success() .get_output() @@ -3807,7 +3739,6 @@ fn file_changed_hook_appends_evidence_to_active_task() { let touched = workdir.join("CLAUDE.md"); std::fs::write(&touched, "# rules v2").unwrap(); - let stdin_payload = serde_json::json!({ "hook_event_name": "FileChanged", "file_path": touched.to_str().unwrap(), @@ -3818,8 +3749,9 @@ fn file_changed_hook_appends_evidence_to_active_task() { Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) + .args(["ingest-hook", "--backend", "heuristic"]) .write_stdin(stdin_payload) .assert() .success(); @@ -3831,8 +3763,7 @@ fn file_changed_hook_appends_evidence_to_active_task() { .args(["pack", &task_id, "--mode", "full"]) .assert() .success() - .stdout(contains("FileChanged (change)")) - .stdout(contains("CLAUDE.md")); + .stdout(contains("FileChanged").not()); } #[test] @@ -3872,7 +3803,7 @@ fn file_changed_hook_with_no_open_task_is_no_op() { } #[test] -fn asyncrewake_below_threshold_exits_zero() { +fn post_tool_use_backlog_never_exits_two() { let dir = assert_fs::TempDir::new().unwrap(); let workdir = dir.path().join("proj"); std::fs::create_dir_all(&workdir).unwrap(); @@ -3881,41 +3812,11 @@ fn asyncrewake_below_threshold_exits_zero() { .unwrap() .env("XDG_DATA_HOME", dir.path()) .current_dir(&workdir) - .args(["create", "Async wake test below threshold"]) + .args(["create", "Backlog test"]) .assert() .success(); - // Seed 5 entries — well under the 25 threshold. - let pending = dir.path().join("task-journal").join("pending"); - seed_pending_chunks(&pending, 5); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .env("TJ_ASYNC_REWAKE", "1") - .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") - .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) - .write_stdin(posttooluse_payload()) - .assert() - .success(); // exit 0, no wake -} - -#[test] -fn asyncrewake_overflow_exits_two_with_drain_hint() { - let dir = assert_fs::TempDir::new().unwrap(); - let workdir = dir.path().join("proj"); - std::fs::create_dir_all(&workdir).unwrap(); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["create", "Async wake overflow test"]) - .assert() - .success(); - - // Seed 30 entries — over the 25 threshold. + // Seed 30 entries — over the old 25-entry wake threshold. let pending = dir.path().join("task-journal").join("pending"); seed_pending_chunks(&pending, 30); @@ -3925,46 +3826,11 @@ fn asyncrewake_overflow_exits_two_with_drain_hint() { .env("TJ_ASYNC_REWAKE", "1") .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) + .args(["ingest-hook", "--backend", "heuristic"]) .write_stdin(posttooluse_payload()) .assert() - .failure() - .code(2) - .stdout(contains("Task Journal pending queue")) - .stdout(contains("pending-gc")); -} - -#[test] -fn asyncrewake_overflow_without_env_does_not_exit_two() { - // Sync hook safety: without TJ_ASYNC_REWAKE=1 we must NEVER exit 2 - // even on overflow, because exit 2 from a sync hook blocks the - // operation in Claude Code. CLI invocations and the PreCompact/Stop - // hooks (which stay sync) rely on this guarantee. - let dir = assert_fs::TempDir::new().unwrap(); - let workdir = dir.path().join("proj"); - std::fs::create_dir_all(&workdir).unwrap(); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .current_dir(&workdir) - .args(["create", "Sync hook safety test"]) - .assert() - .success(); - - let pending = dir.path().join("task-journal").join("pending"); - seed_pending_chunks(&pending, 30); - - Command::cargo_bin("task-journal") - .unwrap() - .env("XDG_DATA_HOME", dir.path()) - .env_remove("TJ_ASYNC_REWAKE") - .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") - .current_dir(&workdir) - .args(["ingest-hook", "--backend", "hybrid"]) - .write_stdin(posttooluse_payload()) - .assert() - .success(); // exit 0, no wake — must not block sync hooks + .success() + .stdout(contains("pending queue").not()); } // --------------------------------------------------------------------------- From 3d35d1c1ffd88330abbc7c876075e216c5911742 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:38:53 +0400 Subject: [PATCH 016/104] fix(core): count chars for extractor thresholds and require a real git commit command --- crates/tj-core/src/session/extractor.rs | 73 ++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/crates/tj-core/src/session/extractor.rs b/crates/tj-core/src/session/extractor.rs index 4544bdd..368c49f 100644 --- a/crates/tj-core/src/session/extractor.rs +++ b/crates/tj-core/src/session/extractor.rs @@ -94,7 +94,7 @@ pub fn extract_from_session(session: &ParsedSession) -> Option { // Git commit → evidence. if tool_name == "Bash" { if let Some(cmd) = input.get("command").and_then(|v| v.as_str()) { - if cmd.contains("git commit") && !cmd.contains("git commit --amend") { + if is_git_commit(cmd) { let mut ev = Event::new( &task_id, EventType::Evidence, @@ -230,7 +230,8 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option let lower = text.to_lowercase(); // Skip very short texts (< 50 chars) — usually just confirmations. - if text.len() < 50 { + let char_count = text.chars().count(); + if char_count < 50 { return None; } @@ -302,7 +303,7 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option "must be", ]; for pattern in &constraint_patterns { - if lower.contains(pattern) && text.len() < 500 { + if lower.contains(pattern) && char_count < 500 { let mut ev = Event::new( task_id, EventType::Constraint, @@ -334,6 +335,15 @@ fn is_test_command(cmd: &str) -> bool { || lower.contains("make test") } +/// True when a bash command actually runs `git commit` (not `--amend`): some +/// `&&`-chained step must start with it, so an `echo`/`grep` that merely +/// mentions "git commit" doesn't count. +fn is_git_commit(cmd: &str) -> bool { + cmd.split("&&").map(str::trim).any(|step| { + (step == "git commit" || step.starts_with("git commit ")) && !step.contains("--amend") + }) +} + /// Shorten a file path for display — keep last 2 components. fn shorten_path(path: &str) -> String { let parts: Vec<&str> = path.split(['/', '\\']).collect(); @@ -626,6 +636,49 @@ mod tests { ); } + fn git_commit_evidence_count(cmd: &str) -> usize { + let session = ParsedSession { + session_id: "git-cmd-session".into(), + file_path: "/tmp/gcmd.jsonl".into(), + entries: vec![ + make_user_entry("u1", "2026-01-01T00:00:00Z", "Do the thing"), + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::ToolUse { + name: "Bash".into(), + input: serde_json::json!({ "command": cmd }), + }], + ), + make_user_entry("u2", "2026-01-01T00:00:02Z", "Thanks"), + ], + first_timestamp: Some("2026-01-01T00:00:00Z".into()), + last_timestamp: Some("2026-01-01T00:00:02Z".into()), + }; + extract_from_session(&session) + .unwrap() + .events + .iter() + .filter(|e| e.text.starts_with("Git commit")) + .count() + } + + #[test] + fn git_commit_evidence_requires_a_real_commit_command() { + assert_eq!(git_commit_evidence_count(" git commit -m 'fix'"), 1); + assert_eq!(git_commit_evidence_count("cd repo && git commit -m 'x'"), 1); + assert_eq!( + git_commit_evidence_count("git add -A && git commit -m 'x'"), + 1 + ); + assert_eq!( + git_commit_evidence_count("echo 'remember to git commit'"), + 0 + ); + assert_eq!(git_commit_evidence_count("grep -rn \"git commit\" ."), 0); + assert_eq!(git_commit_evidence_count("git commit --amend --no-edit"), 0); + } + // --- strip_xml_tags() --- #[test] @@ -764,6 +817,20 @@ mod tests { assert_eq!(ev.unwrap().event_type, EventType::Constraint); } + #[test] + fn classify_thresholds_count_chars_not_bytes() { + // 29 Cyrillic chars but 50+ bytes: still a short confirmation. + let short = "Я выбрал вариант Б, он проще."; + assert!(short.chars().count() < 50 && short.len() >= 50); + assert!(classify_text_heuristic("tj-test", short, "2026-01-01T00:00:00Z").is_none()); + + // Under 500 chars but over 500 bytes: still a constraint. + let long = format!("Есть ограничение: {}", "а".repeat(300)); + assert!(long.chars().count() < 500 && long.len() >= 500); + let ev = classify_text_heuristic("tj-test", &long, "2026-01-01T00:00:00Z"); + assert_eq!(ev.unwrap().event_type, EventType::Constraint); + } + #[test] fn test_classify_no_match_returns_none() { let ev = classify_text_heuristic( From 379b1f644412697363db49fd23036f05761bf406 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:39:47 +0400 Subject: [PATCH 017/104] fix(hooks): scope the shared pending queue to the current project --- crates/tj-cli/src/main.rs | 200 ++++++++++++++-------- crates/tj-cli/tests/cli.rs | 329 +++++++++++++++++++++++++++++++++++++ 2 files changed, 464 insertions(+), 65 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 10e7f21..d6997ff 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -343,23 +343,60 @@ fn pending_dir() -> Result { Ok(dir) } -fn run_pending_list() -> Result<()> { - let dir = pending_dir()?; +/// The current project's entries in the global `pending/` dir. New entries +/// are named `..json` (`….dead.json` once retries are +/// exhausted); a legacy bare `.json` counts as ours unless its JSON +/// names another project. Callers filter schema / dead state on top. +fn project_pending_entries( + dir: &std::path::Path, + project_hash: &str, +) -> Result> { + let mut out = Vec::new(); if !dir.exists() { - println!("(no pending entries)"); - return Ok(()); + return Ok(out); } - let mut entries: Vec<(String, String, String, u32)> = Vec::new(); - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("json") { + + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + let Some(stem) = path + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".json")) + else { continue; + }; + let stem = stem.strip_suffix(".dead").unwrap_or(stem); + let ours = match stem.split_once('.') { + Some((prefix, _)) => prefix == project_hash, + None => !legacy_pending_is_foreign(&path, project_hash), + }; + if ours { + out.push(path); } - let id = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("?") + } + + Ok(out) +} + +/// A legacy (un-prefixed) entry belongs to another project only when its +/// JSON says so; one without `project_hash` stays visible here. +fn legacy_pending_is_foreign(path: &std::path::Path, project_hash: &str) -> bool { + std::fs::read_to_string(path) + .ok() + .and_then(|b| serde_json::from_str::(&b).ok()) + .and_then(|v| v.get("project_hash")?.as_str().map(|h| h != project_hash)) + .unwrap_or(false) +} + +fn run_pending_list() -> Result<()> { + let dir = pending_dir()?; + let project_hash = tj_core::project_hash::from_path(std::env::current_dir()?)?; + let mut entries: Vec<(String, String, String, u32)> = Vec::new(); + for path in project_pending_entries(&dir, &project_hash)? { + let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?"); + let id = stem + .strip_prefix(&format!("{project_hash}.")) + .unwrap_or(stem) .to_string(); let body = std::fs::read_to_string(&path)?; let v: serde_json::Value = serde_json::from_str(&body)?; @@ -406,12 +443,7 @@ fn run_pending_retry( let mut succeeded = 0usize; let mut died = 0usize; let mut still_pending = 0usize; - for entry in std::fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("json") { - continue; - } + for path in project_pending_entries(&dir, &project_hash)? { if path .file_stem() .and_then(|s| s.to_str()) @@ -759,14 +791,17 @@ enum Commands { #[arg(long, default_value_t = 7)] days: i64, }, - /// Garbage-collect the pending classifier queue. Removes entries - /// older than N days OR marked dead by retry exhaustion. Run after - /// classifier auth was broken for a while and the queue grew - /// stale. + /// Garbage-collect the current project's pending classifier queue. + /// Removes entries older than N days OR marked dead by retry + /// exhaustion. Run after classifier auth was broken for a while and + /// the queue grew stale. PendingGc { /// Age threshold in days. Default 7. #[arg(long, default_value_t = 7)] days: i64, + /// Collect every project's entries, not just the current one's. + #[arg(long)] + all: bool, }, /// Set or update the goal of an existing task. Goal { @@ -1640,7 +1675,7 @@ fn real_main() -> Result<()> { ); } } - Commands::PendingGc { days } => { + Commands::PendingGc { days, all } => { let pending_dir = tj_core::paths::events_dir()? .parent() .ok_or_else(|| anyhow::anyhow!("events_dir has no parent"))? @@ -1649,20 +1684,23 @@ fn real_main() -> Result<()> { println!("(no pending dir — nothing to gc)"); return Ok(()); } + let entries: Vec = if all { + std::fs::read_dir(&pending_dir)? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("json")) + .collect() + } else { + let project_hash = tj_core::project_hash::from_path(std::env::current_dir()?)?; + project_pending_entries(&pending_dir, &project_hash)? + }; let cutoff = chrono::Utc::now() - chrono::Duration::days(days); let mut removed = 0usize; - for entry in std::fs::read_dir(&pending_dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) != Some("json") { - continue; - } + for path in entries { // Prefer the file's mtime over JSON parsing — pending // payloads include their own queued_at but are not // guaranteed parseable when the classifier corrupted // input mid-stream. - let mtime = entry - .metadata() + let mtime = std::fs::metadata(&path) .and_then(|m| m.modified()) .ok() .and_then(|t| { @@ -2652,6 +2690,7 @@ runs in the background and won't block you; it only fills gaps and never closes // Drain any pending entries first (Task 10 fills the real-classifier branch). drain_pending( &events_path, + &project_hash, mock_event_type.as_deref(), mock_task_id.as_deref(), mock_confidence, @@ -2732,7 +2771,9 @@ runs in the background and won't block you; it only fills gaps and never closes // Fire-and-forget worker. Errors here are best-effort — // a failure to spawn just means the entry sits in // pending/ until the next hook fires another spawn. - let _ = spawn_classify_worker(&backend); + if std::env::var("TJ_DISABLE_CLASSIFY_SPAWN").is_err() { + let _ = spawn_classify_worker(&backend); + } return Ok(()); } @@ -2842,7 +2883,7 @@ runs in the background and won't block you; it only fills gaps and never closes let out = match classifier.classify(&input) { Ok(o) => o, Err(e) => { - persist_pending(&events_path, &text, &e.to_string())?; + persist_pending(&events_path, &project_hash, &text, &e.to_string())?; return Ok(()); } }; @@ -2872,6 +2913,7 @@ runs in the background and won't block you; it only fills gaps and never closes None => { persist_pending( &events_path, + &project_hash, &text, &format!("task_id_guess `{tid}` not found"), )?; @@ -2880,6 +2922,7 @@ runs in the background and won't block you; it only fills gaps and never closes Some(s) if s == "closed" => { persist_pending( &events_path, + &project_hash, &text, &format!("task_id_guess `{tid}` is closed"), )?; @@ -3441,20 +3484,19 @@ fn run_statusline() -> anyhow::Result { }) .count(); - // Pending dir is global — one entry per queued classifier failure. - // No project filter (matches the brief; per-project counting would - // need extra metadata in each pending file). + // Pending dir is global; this project's entries carry its hash as a + // filename prefix, so counting stays a cheap readdir with no JSON + // parsing. Legacy un-prefixed entries are not counted. + let prefix = format!("{project_hash}."); let pending_count = pending_dir() .ok() .and_then(|d| std::fs::read_dir(&d).ok()) .map(|rd| { rd.filter_map(|e| e.ok()) .filter(|e| { - e.path() - .extension() - .and_then(|x| x.to_str()) - .map(|x| x == "json") - .unwrap_or(false) + let name = e.file_name(); + let name = name.to_string_lossy(); + name.starts_with(&prefix) && name.ends_with(".json") }) .count() }) @@ -5211,7 +5253,12 @@ fn auto_open_task_from_prompt( })) } -fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyhow::Result<()> { +fn persist_pending( + events_path: &std::path::Path, + project_hash: &str, + text: &str, + err: &str, +) -> anyhow::Result<()> { let pending_dir = events_path .parent() .unwrap() @@ -5222,7 +5269,7 @@ fn persist_pending(events_path: &std::path::Path, text: &str, err: &str) -> anyh let id = ulid::Ulid::new().to_string(); let payload = serde_json::json!({"text": text, "error": err, "queued_at": chrono::Utc::now().to_rfc3339()}); std::fs::write( - pending_dir.join(format!("{id}.json")), + pending_dir.join(format!("{project_hash}.{id}.json")), serde_json::to_string_pretty(&payload)?, )?; Ok(()) @@ -5261,7 +5308,9 @@ fn persist_pending_v2( if let Some(sid) = session_id { payload["session_id"] = serde_json::Value::String(sid.to_string()); } - let path = pending_dir.join(format!("{id}.json")); + // `pending/` is shared by every project: the hash prefix tells each + // project's worker which entries are its own. + let path = pending_dir.join(format!("{project_hash}.{id}.json")); std::fs::write(&path, serde_json::to_string_pretty(&payload)?)?; Ok(path) } @@ -5456,14 +5505,9 @@ fn run_classify_worker(backend: &str) -> Result<()> { } // Snapshot entries up front so concurrent re-queues don't loop us. - let mut entries: Vec = Vec::new(); - for e in std::fs::read_dir(&pending)? { - let e = e?; - let p = e.path(); - if p.extension().and_then(|s| s.to_str()) == Some("json") { - entries.push(p); - } - } + // Only this project's entries: the lock is per project, so a worker + // of another project may be draining the same directory right now. + let entries = project_pending_entries(&pending, &project_hash)?; for path in entries { if let Err(err) = process_pending_entry(&path, &events_path, &project_hash, backend) { @@ -5493,6 +5537,11 @@ fn process_pending_entry( if schema != "v2" { return Ok(()); // legacy entry, handled by `pending retry` } + // A legacy un-prefixed entry is ours only when it names this project; + // one without `project_hash` is left for `pending retry`. + if v.get("project_hash").and_then(|x| x.as_str()) != Some(project_hash) { + return Ok(()); + } let kind = v .get("kind") @@ -5584,7 +5633,7 @@ fn process_pending_entry( Err(e) => { // Persist as legacy v1 pending entry so `pending retry` // surfaces it; remove the v2 source. - persist_pending(events_path, &text, &e.to_string())?; + persist_pending(events_path, project_hash, &text, &e.to_string())?; std::fs::remove_file(path)?; return Ok(()); } @@ -5604,6 +5653,7 @@ fn process_pending_entry( None => { persist_pending( events_path, + project_hash, &text, &format!("task_id_guess `{tid}` not found"), )?; @@ -5613,6 +5663,7 @@ fn process_pending_entry( Some(s) if s == "closed" => { persist_pending( events_path, + project_hash, &text, &format!("task_id_guess `{tid}` is closed"), )?; @@ -5671,6 +5722,7 @@ fn process_pending_entry( fn drain_pending( events_path: &std::path::Path, + project_hash: &str, mock_etype: Option<&str>, mock_tid: Option<&str>, mock_conf: Option, @@ -5681,17 +5733,9 @@ fn drain_pending( .parent() .unwrap() .join("pending"); - if !pending_dir.exists() { - return Ok(()); - } - for entry in std::fs::read_dir(&pending_dir)? { - let entry = entry?; - if entry.path().extension().and_then(|e| e.to_str()) != Some("json") { - continue; - } - - let body = std::fs::read_to_string(entry.path())?; + for path in project_pending_entries(&pending_dir, project_hash)? { + let body = std::fs::read_to_string(&path)?; let v: serde_json::Value = serde_json::from_str(&body)?; // v0.6.2: skip v2 entries — those are owned by classify-worker. // Removing them here would silently drop async-queued events. @@ -5719,7 +5763,7 @@ fn drain_pending( writer.flush_durable()?; } } - std::fs::remove_file(entry.path())?; + std::fs::remove_file(&path)?; } Ok(()) } @@ -6225,6 +6269,32 @@ mod inline_tests { assert!(v.get("session_id").is_none()); } + #[test] + fn pending_entries_are_named_after_their_project() { + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + + persist_pending_v2(&events_path, "PostToolUse", "txt", "h", "hybrid", None).unwrap(); + persist_pending(&events_path, "h", "txt", "err").unwrap(); + + let pending = dir.path().join("pending"); + let names: Vec = std::fs::read_dir(&pending) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names.len(), 2); + assert!(names.iter().all(|n| n.starts_with("h.")), "{names:?}"); + assert_eq!( + project_pending_entries(&pending, "h").unwrap().len(), + 2, + "both entries belong to project h" + ); + assert!(project_pending_entries(&pending, "other") + .unwrap() + .is_empty()); + } + #[test] fn is_rewind_prompt_simple() { assert!(is_rewind_prompt("/rewind")); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a598ceb..2e3a014 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2950,6 +2950,335 @@ fn classify_worker_treats_fresh_empty_lock_as_held() { assert!(!lock_path.exists(), "the new holder releases the lock"); } +// ---------------- pending/ is global: every consumer is project-scoped ---------------- + +/// Write a pending entry. `project_hash: None` leaves the field out (an +/// entry from before the queue knew about projects). +fn write_pending_entry( + pending: &std::path::Path, + file_name: &str, + project_hash: Option<&str>, + schema_v2: bool, + kind: &str, + text: &str, +) -> std::path::PathBuf { + std::fs::create_dir_all(pending).unwrap(); + let mut body = serde_json::json!({ + "kind": kind, + "text": text, + "backend": "heuristic", + "queued_at": "2026-05-08T00:00:00Z", + }); + if schema_v2 { + body["schema"] = serde_json::json!("v2"); + } + if let Some(h) = project_hash { + body["project_hash"] = serde_json::json!(h); + body["events_path"] = serde_json::json!(format!("/nowhere/{h}.jsonl")); + } + let path = pending.join(file_name); + std::fs::write(&path, body.to_string()).unwrap(); + path +} + +fn project_events(xdg: &std::path::Path, project_hash: &str) -> String { + std::fs::read_to_string( + xdg.join("task-journal") + .join("events") + .join(format!("{project_hash}.jsonl")), + ) + .unwrap_or_default() +} + +const OTHER_PROJECT: &str = "0123456789abcdef"; + +#[test] +fn classify_worker_only_processes_its_own_projects_entries() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + let pending = dir.path().join("task-journal").join("pending"); + + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + true, + "UserPromptSubmit", + "We decided to use sqlite for the project A store", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + true, + "UserPromptSubmit", + "We decided to use postgres for the project B store", + ); + let legacy_foreign = write_pending_entry( + &pending, + "01JA0000000000000000000002.json", + Some(OTHER_PROJECT), + true, + "UserPromptSubmit", + "We decided to use mongodb for the project B cache", + ); + let legacy_no_project = write_pending_entry( + &pending, + "01JA0000000000000000000003.json", + None, + true, + "UserPromptSubmit", + "We decided to use redis for the unknown project cache", + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + assert!(!own.exists(), "the project's own entry is processed"); + assert!( + foreign.exists(), + "another project's entry must be left alone" + ); + assert!( + legacy_foreign.exists(), + "a legacy entry of another project too" + ); + assert!( + legacy_no_project.exists(), + "a legacy entry without project_hash is left for `pending retry`" + ); + let journal = project_events(dir.path(), &hash_a); + assert!(journal.contains("sqlite"), "own entry recorded: {journal}"); + for other in ["postgres", "mongodb", "redis"] { + assert!( + !journal.contains(other), + "{other} belongs to another project: {journal}" + ); + } +} + +#[test] +fn statusline_counts_only_this_projects_pending_entries() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Statusline scope"]) + .assert() + .success(); + + let pending = dir.path().join("task-journal").join("pending"); + for i in 0..2 { + write_pending_entry( + &pending, + &format!("{hash_a}.01JA000000000000000000000{i}.json"), + Some(&hash_a), + true, + "PostToolUse", + "own", + ); + } + for i in 0..3 { + write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA000000000000000000000{i}.json"), + Some(OTHER_PROJECT), + true, + "PostToolUse", + "foreign", + ); + } + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["statusline"]) + .assert() + .success() + .stdout(contains("pending: 2")); +} + +#[test] +fn pending_list_retry_and_hook_drain_skip_other_projects() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Project A"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + let pending = dir.path().join("task-journal").join("pending"); + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + false, + "Stop", + "own legacy-v1 chunk alpha", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + false, + "Stop", + "foreign chunk bravo", + ); + let legacy_foreign = write_pending_entry( + &pending, + "01JA0000000000000000000002.json", + Some(OTHER_PROJECT), + false, + "Stop", + "foreign legacy chunk charlie", + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["pending", "list"]) + .assert() + .success() + .stdout(contains("alpha")) + .stdout(contains("bravo").not()) + .stdout(contains("charlie").not()); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args([ + "pending", + "retry", + "--mock-event-type", + "decision", + "--mock-task-id", + &task_id, + ]) + .assert() + .success() + .stdout(contains("1 drained")); + assert!(!own.exists(), "own entry drained"); + + // The hook's mock drain is project-scoped too. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args([ + "ingest-hook", + "--kind", + "Stop", + "--text", + "Live chunk", + "--mock-event-type", + "decision", + "--mock-task-id", + &task_id, + ]) + .assert() + .success(); + + assert!( + foreign.exists(), + "another project's entry must be left alone" + ); + assert!( + legacy_foreign.exists(), + "a legacy entry of another project too" + ); + let journal = project_events(dir.path(), &hash_a); + assert!(journal.contains("alpha"), "own entry recorded: {journal}"); + assert!( + !journal.contains("bravo") && !journal.contains("charlie"), + "foreign chunks leaked into project A: {journal}" + ); +} + +#[test] +fn pending_gc_is_project_scoped_unless_all() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj-a"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash_a = tj_core::project_hash::from_path(&workdir).unwrap(); + let pending = dir.path().join("task-journal").join("pending"); + + let own = write_pending_entry( + &pending, + &format!("{hash_a}.01JA0000000000000000000000.json"), + Some(&hash_a), + true, + "PostToolUse", + "own", + ); + let foreign = write_pending_entry( + &pending, + &format!("{OTHER_PROJECT}.01JA0000000000000000000001.json"), + Some(OTHER_PROJECT), + true, + "PostToolUse", + "foreign", + ); + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(3 * 86_400); + for p in [&own, &foreign] { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(old) + .unwrap(); + } + + let gc = |extra: &[&str]| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["pending-gc", "--days", "1"]) + .args(extra) + .assert() + .success(); + }; + + gc(&[]); + assert!(!own.exists(), "own stale entry collected"); + assert!( + foreign.exists(), + "another project's entry is not ours to collect" + ); + + gc(&["--all"]); + assert!(!foreign.exists(), "--all collects every project"); +} + // ===================================================================== // v0.7.0: statusline / PreCompact / /rewind / rejected / export-pr // ===================================================================== From 3e241225214bb54f7de2e6b192f38a0813c81040 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:40:15 +0400 Subject: [PATCH 018/104] fix(cli): export and events list skip malformed JSONL lines with a warning --- crates/tj-cli/src/main.rs | 40 ++++++++++++++++------- crates/tj-cli/tests/cli.rs | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 571a696..9887b58 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -112,6 +112,32 @@ fn dir_writable(dir: &std::path::Path) -> bool { r } +/// Read a project's JSONL event log. Malformed lines are skipped with a +/// warning on stderr, the same policy as `rebuild_state`, so one bad line +/// cannot abort a read-only command. +fn read_events_lenient( + path: &std::path::Path, + command: &str, +) -> Result> { + let body = std::fs::read_to_string(path)?; + let mut events = Vec::new(); + + for (i, line) in body.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + match serde_json::from_str(line) { + Ok(e) => events.push(e), + Err(err) => eprintln!( + "warning: skipping malformed JSONL line {} in {command}: {err}", + i + 1 + ), + } + } + + Ok(events) +} + /// Move all on-disk data for one project_hash to another. Used by the /// `migrate-project` subcommand when a project's directory has been /// moved on disk and the canonical-path hash no longer matches. @@ -1234,12 +1260,7 @@ fn real_main() -> Result<()> { println!("(no events yet)"); return Ok(()); } - let body = std::fs::read_to_string(&events_path)?; - let mut events: Vec = body - .lines() - .filter(|l| !l.trim().is_empty()) - .map(serde_json::from_str) - .collect::>()?; + let mut events = read_events_lenient(&events_path, "events list")?; events.reverse(); for e in events.into_iter().take(limit) { let title = e @@ -3095,12 +3116,7 @@ runs in the background and won't block you; it only fills gaps and never closes anyhow::bail!("no events file at {events_path:?}"); } - let body = std::fs::read_to_string(&events_path)?; - let all_events: Vec = body - .lines() - .filter(|l| !l.trim().is_empty()) - .map(serde_json::from_str) - .collect::>()?; + let all_events = read_events_lenient(&events_path, "export")?; // Filter to specific task if requested. let events: Vec<&tj_core::event::Event> = if let Some(ref tid) = task { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index c768714..ca80903 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -531,6 +531,46 @@ fn export_html_emits_self_contained_document() { assert!(!html.contains("https://"), "external https url leaked"); } +/// Append a line that is not a valid event to the only JSONL log under `xdg`. +fn append_malformed_jsonl_line(xdg: &std::path::Path) { + use std::io::Write; + + let events = xdg.join("task-journal").join("events"); + let log = std::fs::read_dir(&events) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("jsonl")) + .expect("events log present"); + let mut f = std::fs::OpenOptions::new().append(true).open(log).unwrap(); + writeln!(f, "{{not json").unwrap(); +} + +#[test] +fn export_skips_malformed_jsonl_lines_with_a_warning() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Survives a bad line"]) + .assert() + .success(); + append_malformed_jsonl_line(xdg.path()); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["export", "--format", "json"]) + .assert() + .success() + .stdout(contains("Survives a bad line")) + .stderr(contains("skipping malformed JSONL line 2")); +} + #[test] fn migrate_project_round_trips_data_to_new_path() { let xdg = assert_fs::TempDir::new().unwrap(); @@ -2224,6 +2264,31 @@ fn events_list_shows_recent_events() { .stdout(contains("First task").and(contains("Second task"))); } +#[test] +fn events_list_skips_malformed_jsonl_lines_with_a_warning() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Listed despite a bad line"]) + .assert() + .success(); + append_malformed_jsonl_line(xdg.path()); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["events", "list"]) + .assert() + .success() + .stdout(contains("Listed despite a bad line")) + .stderr(contains("skipping malformed JSONL line 2")); +} + #[test] fn rebuild_state_creates_sqlite_with_one_task() { let dir = assert_fs::TempDir::new().unwrap(); From d7595c9359a44d7c602973c2aec168391fc4c89d Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:40:24 +0400 Subject: [PATCH 019/104] refactor(core): remove unused dream backends and session discovery helpers --- crates/tj-core/src/dream/agent_sdk.rs | 139 ----------------------- crates/tj-core/src/dream/backend.rs | 2 + crates/tj-core/src/dream/http.rs | 143 ------------------------ crates/tj-core/src/dream/mod.rs | 2 - crates/tj-core/src/session/discovery.rs | 75 +------------ 5 files changed, 4 insertions(+), 357 deletions(-) delete mode 100644 crates/tj-core/src/dream/agent_sdk.rs delete mode 100644 crates/tj-core/src/dream/http.rs diff --git a/crates/tj-core/src/dream/agent_sdk.rs b/crates/tj-core/src/dream/agent_sdk.rs deleted file mode 100644 index 9c37806..0000000 --- a/crates/tj-core/src/dream/agent_sdk.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Claude CLI ("agent SDK") implementation of [`DreamBackend`]. -//! -//! Runs the local, already-authenticated `claude` binary in print mode, -//! **pinned to Haiku**, so offline dream backfill works on the Claude -//! subscription with no `ANTHROPIC_API_KEY`. Mirrors -//! [`crate::dream::http::AnthropicDreamBackend`] but reuses the classifier's -//! [`crate::classifier::agent_sdk`] command plumbing. -//! -//! Unlike the realtime classifier, dream feeds a whole session transcript, so -//! the prompt can be large. It therefore uses [`ClaudeBinaryStdinRunner`] -//! (prompt on stdin) to avoid the per-argument size limit (`E2BIG`). -//! -//! Cost note: as with the classifier agent-sdk backend, since 2026-06-15 a -//! headless `claude -p` run draws from the separate Agent SDK monthly credit -//! pool. Haiku keeps each backfill call cheap. - -use crate::classifier::agent_sdk::{ - claude_on_path, run_claude_json, ClaudeBinaryStdinRunner, CommandRunner, -}; -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; -use crate::dream::prompt::build_prompt; -use anyhow::Context; - -/// Default dream model: Haiku — cheap and subscription-friendly. Override with -/// `TJ_DREAM_MODEL` (shared with the API backend's override). -pub const DEFAULT_MODEL: &str = "claude-haiku-4-5"; - -pub struct ClaudeCliDreamBackend { - model: String, - runner: Box, -} - -impl ClaudeCliDreamBackend { - /// Build from environment. Returns `None` unless a `claude` binary is on - /// PATH, so the caller can fall back to the API backend. Model comes from - /// `TJ_DREAM_MODEL`, else Haiku. - pub fn from_env() -> Option { - if !claude_on_path() { - return None; - } - let model = std::env::var("TJ_DREAM_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); - Some(Self { - model, - runner: Box::new(ClaudeBinaryStdinRunner), - }) - } - - /// Test/dev constructor: inject a fake runner and explicit model so the - /// parse path is exercised without a live `claude` login. - pub fn with_runner(model: impl Into, runner: Box) -> Self { - Self { - model: model.into(), - runner, - } - } -} - -impl DreamBackend for ClaudeCliDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result> { - let prompt = build_prompt(input); - let verdict = run_claude_json(self.runner.as_ref(), &self.model, &prompt)?; - // The model returns a JSON array of events, sometimes fenced. - let json_str = verdict - .trim() - .trim_start_matches("```json") - .trim_start_matches("```") - .trim_end_matches("```") - .trim(); - serde_json::from_str(json_str) - .with_context(|| format!("dream JSON parse failed; got: {json_str}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dream::backend::BackfillInput; - use crate::event::EventType; - - /// Fake runner returning a canned `--output-format json` envelope whose - /// `result` is the model's reply text. - struct FakeRunner { - result_text: String, - } - - impl CommandRunner for FakeRunner { - fn run(&self, _model: &str, _prompt: &str) -> anyhow::Result { - Ok(serde_json::json!({ - "type": "result", - "is_error": false, - "result": self.result_text, - }) - .to_string()) - } - } - - fn input() -> BackfillInput { - BackfillInput { - tasks: vec![], - transcript: "user: решили взять Postgres вместо Mongo\nassistant: ок".into(), - } - } - - #[test] - fn parses_event_array_from_envelope() { - let runner = FakeRunner { - result_text: r#"[{"event_type":"decision","task_id":"tj-2","text":"Взяли Postgres вместо Mongo.","timestamp":"2026-06-08T10:00:00Z"}]"# - .into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].event_type, EventType::Decision); - assert_eq!(out[0].task_id, "tj-2"); - assert_eq!(out[0].text, "Взяли Postgres вместо Mongo."); - } - - #[test] - fn tolerates_code_fence_wrapped_array() { - let runner = FakeRunner { - result_text: "```json\n[{\"event_type\":\"finding\",\"task_id\":\"tj-1\",\"text\":\"x\",\"timestamp\":\"2026-06-08T10:00:00Z\"}]\n```" - .into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].event_type, EventType::Finding); - } - - #[test] - fn empty_array_yields_no_events() { - let runner = FakeRunner { - result_text: "[]".into(), - }; - let be = ClaudeCliDreamBackend::with_runner("claude-haiku-4-5", Box::new(runner)); - let out = be.backfill(&input()).unwrap(); - assert!(out.is_empty()); - } -} diff --git a/crates/tj-core/src/dream/backend.rs b/crates/tj-core/src/dream/backend.rs index 69d4ad5..1434ce5 100644 --- a/crates/tj-core/src/dream/backend.rs +++ b/crates/tj-core/src/dream/backend.rs @@ -39,10 +39,12 @@ pub trait DreamBackend { } /// Test backend that returns a canned list, ignoring the input. +#[cfg(test)] pub struct MockDreamBackend { pub events: Vec, } +#[cfg(test)] impl DreamBackend for MockDreamBackend { fn backfill(&self, _input: &BackfillInput) -> anyhow::Result> { Ok(self.events.clone()) diff --git a/crates/tj-core/src/dream/http.rs b/crates/tj-core/src/dream/http.rs deleted file mode 100644 index a1af1c2..0000000 --- a/crates/tj-core/src/dream/http.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Anthropic API HTTP client implementing DreamBackend. Mirrors -//! classifier::http but returns a list of missed events. - -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; -use crate::dream::prompt::build_prompt; -use anyhow::{anyhow, Context}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); -/// Stronger than the classifier default — dream sees the whole transcript. -pub const DEFAULT_MODEL: &str = "claude-sonnet-4-6"; -pub const DEFAULT_MAX_TOKENS: u32 = 2048; - -pub struct AnthropicDreamBackend { - pub api_key: String, - pub model: String, - pub base_url: String, - pub max_tokens: u32, - pub timeout: Duration, -} - -impl AnthropicDreamBackend { - pub fn from_env() -> anyhow::Result { - let api_key = - std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY env var not set")?; - let model = std::env::var("TJ_DREAM_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); - let max_tokens = std::env::var("TJ_DREAM_MAX_TOKENS") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(DEFAULT_MAX_TOKENS); - Ok(Self { - api_key, - model, - base_url: "https://api.anthropic.com".into(), - max_tokens, - timeout: DEFAULT_TIMEOUT, - }) - } -} - -#[derive(Serialize)] -struct MessagesRequest<'a> { - model: &'a str, - max_tokens: u32, - messages: Vec>, -} -#[derive(Serialize)] -struct MessageIn<'a> { - role: &'a str, - content: &'a str, -} -#[derive(Deserialize)] -struct MessagesResponse { - content: Vec, -} -#[derive(Deserialize)] -struct ContentBlock { - #[serde(rename = "type")] - kind: String, - #[serde(default)] - text: String, -} - -impl DreamBackend for AnthropicDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result> { - let prompt = build_prompt(input); - let body = MessagesRequest { - model: &self.model, - max_tokens: self.max_tokens, - messages: vec![MessageIn { - role: "user", - content: &prompt, - }], - }; - let url = format!("{}/v1/messages", self.base_url); - let resp: MessagesResponse = ureq::post(&url) - .timeout(self.timeout) - .set("x-api-key", &self.api_key) - .set("anthropic-version", "2023-06-01") - .set("content-type", "application/json") - .send_json(serde_json::to_value(&body)?) - .context("Anthropic API request failed")? - .into_json() - .context("decode Anthropic response")?; - - let text = resp - .content - .iter() - .find(|b| b.kind == "text") - .map(|b| b.text.clone()) - .ok_or_else(|| anyhow!("no text content in response"))?; - - let json_str = text - .trim() - .trim_start_matches("```json") - .trim_start_matches("```") - .trim_end_matches("```") - .trim(); - let out: Vec = serde_json::from_str(json_str) - .with_context(|| format!("dream JSON parse failed; got: {json_str}"))?; - Ok(out) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dream::backend::BackfillInput; - - #[test] - fn backend_parses_event_array() { - let mut server = mockito::Server::new(); - let url = server.url(); - let body = serde_json::json!({ - "content": [ - { "type": "text", "text": "[{\"event_type\":\"finding\",\"task_id\":\"tj-2\",\"text\":\"Found the bug.\",\"timestamp\":\"2026-06-08T10:00:00Z\"}]" } - ] - }); - let _m = server - .mock("POST", "/v1/messages") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(body.to_string()) - .create(); - - let be = AnthropicDreamBackend { - api_key: "k".into(), - model: "m".into(), - base_url: url, - max_tokens: 256, - timeout: Duration::from_secs(5), - }; - let input = BackfillInput { - tasks: vec![], - transcript: "t".into(), - }; - let out = be.backfill(&input).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].text, "Found the bug."); - assert_eq!(out[0].task_id, "tj-2"); - } -} diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index 3924953..b153520 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -4,10 +4,8 @@ //! significant typed events the realtime classifier missed. Additive — //! the JSONL source of truth is never mutated. -pub mod agent_sdk; pub mod backend; pub mod backfill; -pub mod http; pub mod llm_backend; pub mod prompt; pub mod scope; diff --git a/crates/tj-core/src/session/discovery.rs b/crates/tj-core/src/session/discovery.rs index 669896c..e9a26a0 100644 --- a/crates/tj-core/src/session/discovery.rs +++ b/crates/tj-core/src/session/discovery.rs @@ -37,7 +37,8 @@ pub fn encode_project_path(path: &str) -> String { } /// Find the project directory for a given filesystem path. -/// Tries exact match first, then prefix match for worktree variants. +/// Tries an exact match first, then a case-insensitive one (WSL paths can +/// differ in case). Worktrees are not matched to their main checkout. pub fn find_project_dir(project_path: &Path) -> anyhow::Result> { let projects = projects_dir()?; if !projects.exists() { @@ -98,35 +99,6 @@ pub fn list_sessions(project_dir: &Path) -> anyhow::Result> { Ok(sessions.into_iter().map(|(p, _)| p).collect()) } -/// List all project directories in Claude Code config. -pub fn list_all_projects() -> anyhow::Result> { - let projects = projects_dir()?; - if !projects.exists() { - return Ok(vec![]); - } - - let mut result = Vec::new(); - for entry in std::fs::read_dir(&projects)? { - let entry = entry?; - if entry.path().is_dir() { - let name = entry.file_name().to_string_lossy().to_string(); - // Decode the project name back to a readable path. - let decoded = decode_project_path(&name); - result.push((decoded, entry.path())); - } - } - result.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(result) -} - -/// Decode an encoded project directory name back to a readable path. -/// This is approximate — we can't distinguish `-` from original `/`. -fn decode_project_path(encoded: &str) -> String { - // Common pattern: leading `--` means the path started with a path separator. - // Replace double dashes carefully. - encoded.to_string() -} - fn dirs_home() -> anyhow::Result { directories::BaseDirs::new() .map(|d| d.home_dir().to_path_buf()) @@ -229,40 +201,6 @@ mod tests { assert!(result.is_err()); } - // --- list_all_projects() --- - - #[test] - fn list_all_projects_with_temp_dir() { - let dir = tempfile::tempdir().unwrap(); - // Override CLAUDE_CONFIG_DIR for this test. - let config_dir = dir.path(); - let projects = config_dir.join("projects"); - std::fs::create_dir_all(&projects).unwrap(); - - // Create project directories. - std::fs::create_dir(projects.join("-home-user-project-alpha")).unwrap(); - std::fs::create_dir(projects.join("-home-user-project-beta")).unwrap(); - // Create a file (should be skipped — not a directory). - std::fs::write(projects.join("not-a-dir.txt"), "").unwrap(); - - // We can't easily test list_all_projects() because it uses projects_dir() - // which reads CLAUDE_CONFIG_DIR. Instead, test the directory listing logic directly. - let mut result = Vec::new(); - for entry in std::fs::read_dir(&projects).unwrap() { - let entry = entry.unwrap(); - if entry.path().is_dir() { - let name = entry.file_name().to_string_lossy().to_string(); - let decoded = decode_project_path(&name); - result.push((decoded, entry.path())); - } - } - result.sort_by(|a, b| a.0.cmp(&b.0)); - - assert_eq!(result.len(), 2); - assert!(result[0].0.contains("alpha")); - assert!(result[1].0.contains("beta")); - } - // --- find_project_dir() with CLAUDE_CONFIG_DIR env override --- #[test] @@ -321,15 +259,6 @@ mod tests { assert!(result.unwrap().is_none()); } - // --- decode_project_path --- - - #[test] - fn decode_project_path_returns_same_string() { - // Current implementation is identity — just verify it doesn't panic. - let decoded = decode_project_path("-home-user-project"); - assert_eq!(decoded, "-home-user-project"); - } - // --- claude_config_dir --- /// Both `CLAUDE_CONFIG_DIR` cases are combined into one test so the From 219a58accbad9523af88fbc724bf2b6699b33f3a Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:40:35 +0400 Subject: [PATCH 020/104] fix(hooks): process pending entries oldest first --- crates/tj-cli/src/main.rs | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d6997ff..6505e47 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -351,11 +351,11 @@ fn project_pending_entries( dir: &std::path::Path, project_hash: &str, ) -> Result> { - let mut out = Vec::new(); if !dir.exists() { - return Ok(out); + return Ok(Vec::new()); } + let mut out = Vec::new(); for entry in std::fs::read_dir(dir)? { let path = entry?.path(); let Some(stem) = path @@ -366,16 +366,19 @@ fn project_pending_entries( continue; }; let stem = stem.strip_suffix(".dead").unwrap_or(stem); - let ours = match stem.split_once('.') { - Some((prefix, _)) => prefix == project_hash, - None => !legacy_pending_is_foreign(&path, project_hash), + let (ulid, ours) = match stem.split_once('.') { + Some((prefix, ulid)) => (ulid, prefix == project_hash), + None => (stem, !legacy_pending_is_foreign(&path, project_hash)), }; if ours { - out.push(path); + out.push((ulid.to_string(), path)); } } - Ok(out) + // ULIDs sort by creation time: oldest first, so a user prompt is + // classified before the assistant turn that answered it. + out.sort(); + Ok(out.into_iter().map(|(_, path)| path).collect()) } /// A legacy (un-prefixed) entry belongs to another project only when its @@ -6295,6 +6298,33 @@ mod inline_tests { .is_empty()); } + #[test] + fn pending_entries_come_back_oldest_first() { + let dir = tempfile::tempdir().unwrap(); + // ULID order == queue order; legacy un-prefixed names interleave by + // their ULID, not by the hash prefix of the new names. + let mut expected = Vec::new(); + for i in 0..20u32 { + let ulid = format!("01JA{i:022}"); + let name = if i % 5 == 0 { + format!("{ulid}.json") + } else { + format!("h.{ulid}.json") + }; + expected.push(name); + } + for name in expected.iter().rev() { + std::fs::write(dir.path().join(name), "{}").unwrap(); + } + + let got: Vec = project_pending_entries(dir.path(), "h") + .unwrap() + .iter() + .map(|p| p.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(got, expected); + } + #[test] fn is_rewind_prompt_simple() { assert!(is_rewind_prompt("/rewind")); From bee8212a052865652824d76a84bde8c00ec01675 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:40:46 +0400 Subject: [PATCH 021/104] fix(cli): events list prints snake_case event type names --- crates/tj-cli/src/main.rs | 6 +++++- crates/tj-cli/tests/cli.rs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 9887b58..c1ec0a6 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1269,7 +1269,11 @@ fn real_main() -> Result<()> { .and_then(|v| v.as_str()) .map(|s| s.to_string()) .unwrap_or_else(|| e.text.clone()); - println!("{} [{:?}] {}", e.timestamp, e.event_type, title); + let etype = serde_json::to_value(e.event_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "?".into()); + println!("{} [{etype}] {}", e.timestamp, title); } } }, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ca80903..1d3b4e4 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2289,6 +2289,29 @@ fn events_list_skips_malformed_jsonl_lines_with_a_warning() { .stderr(contains("skipping malformed JSONL line 2")); } +#[test] +fn events_list_prints_snake_case_event_types() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Typed listing"]) + .assert() + .success(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["events", "list"]) + .assert() + .success() + .stdout(contains("[open]").and(contains("[Open]").not())); +} + #[test] fn rebuild_state_creates_sqlite_with_one_task() { let dir = assert_fs::TempDir::new().unwrap(); From d4c60b1e0a89cb0a2073b057d40b67211bc3cb04 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:41:03 +0400 Subject: [PATCH 022/104] fix(core): dedup dream proposals against each other and across sessions of a run --- crates/tj-core/src/dream/backfill.rs | 36 ++++++++++++++++++------ crates/tj-core/src/dream/mod.rs | 41 ++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/crates/tj-core/src/dream/backfill.rs b/crates/tj-core/src/dream/backfill.rs index 952db1a..a249313 100644 --- a/crates/tj-core/src/dream/backfill.rs +++ b/crates/tj-core/src/dream/backfill.rs @@ -32,16 +32,20 @@ pub fn similarity(a: &str, b: &str) -> f64 { } } -/// Drop proposed events that are near-duplicates of `existing` texts. +/// Drop proposed events that are near-duplicates of `existing` texts or of +/// an earlier proposal in the same batch (two chunks restating one event). pub fn dedup_guard(proposed: Vec, existing: &[String]) -> Vec { - proposed - .into_iter() - .filter(|p| { - !existing - .iter() - .any(|e| similarity(&p.text, e) >= DUP_THRESHOLD) - }) - .collect() + let mut kept: Vec = Vec::new(); + for p in proposed { + let dup = existing + .iter() + .chain(kept.iter().map(|k| &k.text)) + .any(|e| similarity(&p.text, e) >= DUP_THRESHOLD); + if !dup { + kept.push(p); + } + } + kept } /// Build a journal Event from a proposed backfill event, stamping dream @@ -90,6 +94,20 @@ mod tests { assert!(kept[0].text.contains("TTL")); } + #[test] + fn drops_near_duplicates_among_proposals() { + // Two chunks of one session restating the same decision. + let proposed = vec![ + ev("We decided to use SQLite instead of Postgres"), + ev("Decided to use SQLite instead of Postgres"), + ev("The cache layer needs a TTL of 60 seconds"), + ]; + let kept = dedup_guard(proposed, &[]); + assert_eq!(kept.len(), 2); + assert!(kept[0].text.starts_with("We decided")); + assert!(kept[1].text.contains("TTL")); + } + #[test] fn identical_text_is_similarity_one() { assert!((similarity("hello world", "hello world") - 1.0).abs() < f64::EPSILON); diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index b153520..3eb4e47 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -37,6 +37,9 @@ pub fn run_dream( run_id: &str, ) -> anyhow::Result { let mut report = DreamReport::default(); + // Texts appended earlier in this run: a later session of the same task + // must not re-add them (its `existing_events` predate the run). + let mut appended: Vec = Vec::new(); for (session_id, input) in sessions { report.sessions_processed += 1; if opts.dry_run { @@ -48,6 +51,7 @@ pub fn run_dream( .tasks .iter() .flat_map(|t| t.existing_events.clone()) + .chain(appended.iter().cloned()) .collect(); let kept = crate::dream::backfill::dedup_guard(proposed, &existing); let mut writer = crate::storage::JsonlWriter::open(events_path)?; @@ -56,6 +60,7 @@ pub fn run_dream( writer.append(&e)?; crate::db::upsert_task_from_event(conn, &e, &opts.project_hash)?; crate::db::index_event(conn, &e)?; + appended.push(b.text.clone()); report.events_backfilled += 1; } writer.flush_durable()?; @@ -138,6 +143,42 @@ mod tests { assert!(!body.contains("\"text\":\"Already known fact.\",\"refs\"")); } + #[test] + fn run_dream_does_not_repeat_an_event_across_sessions() { + // Two sessions of one task both surface the same decision: the + // second must not append it again. + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![BackfillEvent { + event_type: EventType::Decision, + task_id: "tj-1".into(), + text: "Chose SQLite over Postgres.".into(), + timestamp: "2026-06-08T10:00:00Z".into(), + }], + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + let mut second = task_input(); + second.0 = "sess-2".into(); + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input(), second], + "run-1", + ) + .unwrap(); + + assert_eq!(report.sessions_processed, 2); + assert_eq!(report.events_backfilled, 1); + } + #[test] fn dry_run_writes_nothing_and_skips_backend() { let d = TempDir::new().unwrap(); From c531a0a115ba2960db7447a6fb909bbd73feb3d0 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:41:53 +0400 Subject: [PATCH 023/104] fix(dream): drop proposed events whose task id is not a session candidate --- crates/tj-cli/src/main.rs | 9 +++++- crates/tj-core/src/dream/mod.rs | 54 ++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 571a696..c5a7fb9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -4567,10 +4567,17 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l &project_hash, &chrono::Utc::now().to_rfc3339(), )?; - println!( + let mut summary = format!( "dream: {} session(s) processed, {} event(s) backfilled", report.sessions_processed, report.events_backfilled ); + if report.events_dropped_unknown_task > 0 { + summary.push_str(&format!( + ", {} dropped (unknown task id)", + report.events_dropped_unknown_task + )); + } + println!("{summary}"); Ok(()) } diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index 3eb4e47..14b7405 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -23,6 +23,9 @@ pub struct DreamOptions { pub struct DreamReport { pub sessions_processed: usize, pub events_backfilled: usize, + /// Proposals dropped because their `task_id` was not one of the + /// session's candidate tasks (the model invented or misspelled it). + pub events_dropped_unknown_task: usize, } /// Run one dream Pass A over the given sessions, using the supplied @@ -45,7 +48,14 @@ pub fn run_dream( if opts.dry_run { continue; } - let proposed = backend.backfill(&input)?; + let mut proposed = backend.backfill(&input)?; + + // Only the session's candidate tasks may receive events; anything + // else would be an orphan event for a nonexistent task. + let before = proposed.len(); + proposed.retain(|p| input.tasks.iter().any(|t| t.task_id == p.task_id)); + report.events_dropped_unknown_task += before - proposed.len(); + // Flatten existing texts across candidate tasks for the guard. let existing: Vec = input .tasks @@ -143,6 +153,48 @@ mod tests { assert!(!body.contains("\"text\":\"Already known fact.\",\"refs\"")); } + #[test] + fn run_dream_drops_events_for_tasks_outside_the_candidates() { + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![ + BackfillEvent { + event_type: EventType::Finding, + task_id: "tj-ghost".into(), // not a candidate → orphan + text: "Invented task id.".into(), + timestamp: "2026-06-08T10:00:00Z".into(), + }, + BackfillEvent { + event_type: EventType::Finding, + task_id: "tj-1".into(), + text: "A real finding.".into(), + timestamp: "2026-06-08T10:01:00Z".into(), + }, + ], + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input()], + "run-1", + ) + .unwrap(); + + assert_eq!(report.events_backfilled, 1); + assert_eq!(report.events_dropped_unknown_task, 1); + let body = std::fs::read_to_string(&events_path).unwrap(); + assert!(!body.contains("tj-ghost")); + } + #[test] fn run_dream_does_not_repeat_an_event_across_sessions() { // Two sessions of one task both surface the same decision: the From fdddb72a04c1d7f3969060a21a65ba926fa428d9 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:42:26 +0400 Subject: [PATCH 024/104] fix(cli): md and html export derive task title and status like the SQLite projection --- crates/tj-cli/src/main.rs | 72 ++++++++++++++++-------------------- crates/tj-cli/tests/cli.rs | 75 +++++++++++++++++++++++++++++++++++--- 2 files changed, 102 insertions(+), 45 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index c1ec0a6..47f6181 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -270,6 +270,36 @@ time { font-family: ui-monospace, monospace; color: var(--muted); margin-right: .suggested::after { content: " ?"; color: var(--muted); } "#; +/// Title and status of one task for the md/html export, folded over its +/// events with the same rules as the SQLite projection +/// (`tj_core::db::upsert_task_from_event`): the first `open` sets the title, +/// a later `rename` replaces it, and the last `close`/`reopen` decides status. +fn export_title_and_status(task_events: &[&tj_core::event::Event]) -> (String, &'static str) { + use tj_core::event::EventType; + + let mut title: Option = None; + let mut status = "open"; + + for e in task_events { + let named = || { + e.meta + .get("title") + .and_then(|v| v.as_str()) + .unwrap_or(&e.text) + .to_string() + }; + match e.event_type { + EventType::Open if title.is_none() => title = Some(named()), + EventType::Rename if title.is_some() => title = Some(named()), + EventType::Close => status = "closed", + EventType::Reopen => status = "open", + _ => {} + } + } + + (title.unwrap_or_else(|| "(untitled)".into()), status) +} + fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { use std::collections::BTreeMap; @@ -292,23 +322,7 @@ fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { out.push_str("
"); for (task_id, task_events) in &tasks { - let title = task_events - .iter() - .find(|e| e.event_type == tj_core::event::EventType::Open) - .and_then(|e| { - e.meta - .get("title") - .and_then(|v| v.as_str()) - .map(String::from) - .or_else(|| Some(e.text.clone())) - }) - .unwrap_or_else(|| "(untitled)".into()); - - let closed = task_events - .last() - .map(|e| e.event_type == tj_core::event::EventType::Close) - .unwrap_or(false); - let status = if closed { "closed" } else { "open" }; + let (title, status) = export_title_and_status(task_events); let created = task_events .first() @@ -3153,29 +3167,7 @@ runs in the background and won't block you; it only fills gaps and never closes } for (task_id, task_events) in &tasks { - // Derive title from the first open event's meta, or text. - let title = task_events - .iter() - .find(|e| e.event_type == tj_core::event::EventType::Open) - .and_then(|e| { - e.meta - .get("title") - .and_then(|v| v.as_str()) - .map(String::from) - .or_else(|| Some(e.text.clone())) - }) - .unwrap_or_else(|| "(untitled)".into()); - - // Determine status: closed if last event is close, else open. - let status = if task_events - .last() - .map(|e| e.event_type == tj_core::event::EventType::Close) - .unwrap_or(false) - { - "closed" - } else { - "open" - }; + let (title, status) = export_title_and_status(task_events); // Created timestamp from first event. let created = task_events diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 1d3b4e4..d97698a 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -531,8 +531,8 @@ fn export_html_emits_self_contained_document() { assert!(!html.contains("https://"), "external https url leaked"); } -/// Append a line that is not a valid event to the only JSONL log under `xdg`. -fn append_malformed_jsonl_line(xdg: &std::path::Path) { +/// Append a raw line to the only JSONL event log under `xdg`. +fn append_jsonl_line(xdg: &std::path::Path, line: &str) { use std::io::Write; let events = xdg.join("task-journal").join("events"); @@ -543,7 +543,72 @@ fn append_malformed_jsonl_line(xdg: &std::path::Path) { .find(|p| p.extension().and_then(|s| s.to_str()) == Some("jsonl")) .expect("events log present"); let mut f = std::fs::OpenOptions::new().append(true).open(log).unwrap(); - writeln!(f, "{{not json").unwrap(); + writeln!(f, "{line}").unwrap(); +} + +#[test] +fn export_md_and_html_follow_close_reopen_and_rename_like_the_projection() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Original title"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + // Closed, then renamed: the projection says closed + new title, while the + // last event alone (a rename) would read as open with the old title. + for (etype, text) in [ + (tj_core::event::EventType::Close, "done"), + (tj_core::event::EventType::Reopen, "one more fix"), + (tj_core::event::EventType::Close, "done again"), + (tj_core::event::EventType::Rename, "Renamed title"), + ] { + let e = tj_core::event::Event::new( + task_id.clone(), + etype, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + text.to_string(), + ); + append_jsonl_line(xdg.path(), &serde_json::to_string(&e).unwrap()); + } + + let export = |format: &str| { + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["export", "--format", format]) + .output() + .unwrap(); + String::from_utf8(out.stdout).unwrap() + }; + + let md = export("md"); + assert!( + md.contains(&format!("## [{task_id}] Renamed title")), + "md title must follow the rename: {md}" + ); + assert!(md.contains("**Status**: closed"), "md status: {md}"); + + let html = export("html"); + assert!( + html.contains(&format!("{task_id}Renamed title")), + "html title must follow the rename: {html}" + ); + assert!(html.contains("status: closed"), "html status: {html}"); } #[test] @@ -558,7 +623,7 @@ fn export_skips_malformed_jsonl_lines_with_a_warning() { .args(["create", "Survives a bad line"]) .assert() .success(); - append_malformed_jsonl_line(xdg.path()); + append_jsonl_line(xdg.path(), "{not json"); Command::cargo_bin("task-journal") .unwrap() @@ -2276,7 +2341,7 @@ fn events_list_skips_malformed_jsonl_lines_with_a_warning() { .args(["create", "Listed despite a bad line"]) .assert() .success(); - append_malformed_jsonl_line(xdg.path()); + append_jsonl_line(xdg.path(), "{not json"); Command::cargo_bin("task-journal") .unwrap() From 31f2444b94ee984f328e18d5efbd6998d5c77443 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:42:32 +0400 Subject: [PATCH 025/104] fix(dream): skip an unreadable session transcript instead of aborting the run --- crates/tj-cli/src/main.rs | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index c5a7fb9..31ef4c6 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -6145,7 +6145,17 @@ fn build_dream_inputs( .and_then(|s| s.to_str()) .unwrap_or("") .to_string(); - let parsed = parse_session(path)?; + // One unreadable transcript must not abort mining the rest. + let parsed = match parse_session(path) { + Ok(p) => p, + Err(e) => { + eprintln!( + "dream: skipping unreadable session {}: {e:#}", + path.display() + ); + continue; + } + }; let candidates = candidate_tasks_for_session( events_path, @@ -6316,6 +6326,36 @@ mod inline_tests { )); } + #[test] + fn build_dream_inputs_skips_an_unreadable_session() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("h.jsonl"); + let mut ev = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "task".into(), + ); + ev.meta = serde_json::json!({"session_id": "good"}); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + writer.append(&ev).unwrap(); + writer.flush_durable().unwrap(); + + // Invalid UTF-8 makes parse_session fail for this one file. + let bad = dir.path().join("bad.jsonl"); + std::fs::write(&bad, [0xff, 0xfe, b'\n']).unwrap(); + let good = dir.path().join("good.jsonl"); + std::fs::write(&good, + "{\"type\":\"user\",\"uuid\":\"u1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{\"content\":\"hi\"}}\n").unwrap(); + + let inputs = build_dream_inputs(&events_path, &[bad, good], None).unwrap(); + + assert_eq!(inputs.len(), 1); + assert_eq!(inputs[0].0, "good"); + } + #[test] fn persist_pending_v2_includes_session_id_when_present() { let dir = tempfile::tempdir().unwrap(); From f014386c1f62db711d2fb1da51d26be6e013ad79 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:43:15 +0400 Subject: [PATCH 026/104] fix(cli): migrate-project re-keys embeddings and dream_state too --- crates/tj-cli/src/main.rs | 14 +++---- crates/tj-cli/tests/cli.rs | 77 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 47f6181..38b8934 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -204,14 +204,12 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool let new_state_path = state_dir.join(format!("{to_hash}.sqlite")); if new_state_path.exists() { let conn = tj_core::db::open(&new_state_path)?; - conn.execute( - "UPDATE tasks SET project_hash = ?1 WHERE project_hash = ?2", - rusqlite::params![to_hash, from_hash], - )?; - conn.execute( - "UPDATE index_state SET project_hash = ?1 WHERE project_hash = ?2", - rusqlite::params![to_hash, from_hash], - )?; + for table in ["tasks", "index_state", "embeddings", "dream_state"] { + conn.execute( + &format!("UPDATE {table} SET project_hash = ?1 WHERE project_hash = ?2"), + rusqlite::params![to_hash, from_hash], + )?; + } } if moved.is_empty() { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index d97698a..ffc4480 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -724,6 +724,83 @@ fn migrate_project_refuses_overwrite_without_force() { .stderr(contains("destination already exists")); } +/// Two project dirs that each hash to themselves, plus their hashes. +fn two_projects() -> (assert_fs::TempDir, assert_fs::TempDir, String, String) { + let proj_a = assert_fs::TempDir::new().unwrap(); + let proj_b = assert_fs::TempDir::new().unwrap(); + std::fs::create_dir(proj_a.path().join(".git")).unwrap(); + std::fs::create_dir(proj_b.path().join(".git")).unwrap(); + let from_hash = tj_core::project_hash::from_path(proj_a.path()).unwrap(); + let to_hash = tj_core::project_hash::from_path(proj_b.path()).unwrap(); + + (proj_a, proj_b, from_hash, to_hash) +} + +fn migrate(xdg: &std::path::Path, from: &std::path::Path, to: &std::path::Path, force: bool) { + let mut args = vec![ + "migrate-project", + "--from", + from.to_str().unwrap(), + "--to", + to.to_str().unwrap(), + ]; + if force { + args.push("--force"); + } + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg) + .args(args) + .assert() + .success(); +} + +#[test] +fn migrate_project_rekeys_embeddings_and_dream_state() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Re-key everything"]) + .assert() + .success(); + { + let conn = tj_core::db::open(state.join(format!("{from_hash}.sqlite"))).unwrap(); + conn.execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES (?1, 't', 't')", + [&from_hash], + ) + .unwrap(); + conn.execute( + "INSERT INTO embeddings(event_id, task_id, project_hash, model, dim, vec, created_at) + VALUES ('e1', 'tj-1', ?1, 'hash', 1, x'00000000', 't')", + [&from_hash], + ) + .unwrap(); + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + let conn = rusqlite::Connection::open(state.join(format!("{to_hash}.sqlite"))).unwrap(); + for table in ["dream_state", "embeddings"] { + let rekeyed: i64 = conn + .query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE project_hash = ?1"), + [&to_hash], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + rekeyed, 1, + "{table} row not re-keyed to the new project_hash" + ); + } +} + #[test] fn close_unknown_task_id_returns_error() { let dir = assert_fs::TempDir::new().unwrap(); From 59303383fe4d8f0a9e796fb212a78a776d28c62c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:43:21 +0400 Subject: [PATCH 027/104] fix(hooks): cap PostToolUse chunks and limit their heuristic to test results --- crates/tj-cli/src/main.rs | 13 ++++ crates/tj-cli/tests/cli.rs | 80 ++++++++++++++++++++++ crates/tj-core/src/classifier/agent_sdk.rs | 1 + crates/tj-core/src/classifier/heuristic.rs | 31 +++++++++ crates/tj-core/src/classifier/http.rs | 2 + crates/tj-core/src/classifier/hybrid.rs | 1 + crates/tj-core/src/classifier/mock.rs | 1 + crates/tj-core/src/classifier/mod.rs | 6 ++ crates/tj-core/src/classifier/prompt.rs | 6 ++ 9 files changed, 141 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 6505e47..be21794 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2745,6 +2745,14 @@ runs in the background and won't block you; it only fills gaps and never closes return Ok(()); } + // A tool call arrives as its full input + response JSON; cap it + // so one big file read or command output can't flood the queue. + let text: String = if kind == "PostToolUse" { + text.chars().take(POST_TOOL_USE_TEXT_MAX).collect() + } else { + text + }; + // v0.6.2 fork-bomb fix. The real-classifier path used to run // `claude -p` synchronously inside the hook, blocking each // UserPromptSubmit/PostToolUse/Stop for 5-30s. Symptoms: @@ -2882,6 +2890,7 @@ runs in the background and won't block you; it only fills gaps and never closes text: text.clone(), author_hint: author_hint.into(), recent_tasks: recent, + tool_output: kind == "PostToolUse", }; let out = match classifier.classify(&input) { Ok(o) => o, @@ -5400,6 +5409,9 @@ fn spawn_classify_worker(backend: &str) -> anyhow::Result<()> { Ok(()) } +/// Characters of a PostToolUse chunk (tool input + response) that get queued. +const POST_TOOL_USE_TEXT_MAX: usize = 2000; + /// How long an empty (pid not yet written) worker lockfile counts as held. const LOCK_PID_GRACE: std::time::Duration = std::time::Duration::from_secs(5); @@ -5630,6 +5642,7 @@ fn process_pending_entry( text: text.clone(), author_hint: author_hint.into(), recent_tasks: recent, + tool_output: kind == "PostToolUse", }; let out = match classifier.classify(&input) { Ok(o) => o, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 2e3a014..dc2fd8d 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -4162,6 +4162,86 @@ fn post_tool_use_backlog_never_exits_two() { .stdout(contains("pending queue").not()); } +#[test] +fn post_tool_use_queues_truncated_text() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": "Read", + "tool_input": {"file_path": "big.rs"}, + "tool_response": {"content": "ж".repeat(10_000)}, + }) + .to_string(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .current_dir(&workdir) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload) + .assert() + .success(); + + let pending = dir.path().join("task-journal").join("pending"); + let entries: Vec<_> = std::fs::read_dir(&pending).unwrap().collect(); + assert_eq!(entries.len(), 1); + let body = std::fs::read_to_string(entries[0].as_ref().unwrap().path()).unwrap(); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + let chars = v["text"].as_str().unwrap().chars().count(); + assert!( + chars <= 2000, + "PostToolUse text must be capped, got {chars}" + ); + assert!( + chars > 100, + "the start of the tool call is kept, got {chars}" + ); +} + +#[test] +fn post_tool_use_output_wording_is_not_a_constraint() { + // "must be" / "requires" inside tool output used to become constraint + // events whose text was raw tool JSON. + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Tool output noise"]) + .assert() + .success(); + + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": "Read", + "tool_input": {"file_path": "schema.json"}, + "tool_response": {"content": "the id field must be unique and requires an index"}, + }) + .to_string(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_INGEST_SYNC", "1") + .current_dir(&workdir) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload) + .assert() + .success(); + + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + let journal = project_events(dir.path(), &hash); + assert!( + !journal.contains("\"constraint\""), + "tool output must not become a constraint: {journal}" + ); +} + // --------------------------------------------------------------------------- // v0.10.3 — search/pack quality fixes (user feedback) // --------------------------------------------------------------------------- diff --git a/crates/tj-core/src/classifier/agent_sdk.rs b/crates/tj-core/src/classifier/agent_sdk.rs index 777a54c..fdd11b8 100644 --- a/crates/tj-core/src/classifier/agent_sdk.rs +++ b/crates/tj-core/src/classifier/agent_sdk.rs @@ -397,6 +397,7 @@ mod tests { text: "We adopted Rust for the journal core.".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, } } diff --git a/crates/tj-core/src/classifier/heuristic.rs b/crates/tj-core/src/classifier/heuristic.rs index db99a70..6861f5e 100644 --- a/crates/tj-core/src/classifier/heuristic.rs +++ b/crates/tj-core/src/classifier/heuristic.rs @@ -31,6 +31,11 @@ pub fn try_heuristic(input: &ClassifyInput) -> Option { let task_id_guess = input.recent_tasks.first().map(|t| t.task_id.clone()); for (patterns, etype, confidence) in RULES { + // Tool output is JSON and logs full of "must be" / "requires" / + // "rejected": only a test result is a reliable signal there. + if input.tool_output && *etype != EventType::Evidence { + continue; + } for p in *patterns { if lower.contains(p) { return Some(ClassifyOutput { @@ -189,6 +194,7 @@ mod tests { last_events: vec![], constraints: vec![], }], + tool_output: false, } } @@ -255,6 +261,31 @@ mod tests { assert_eq!(out.event_type, EventType::Decision); } + /// Tool output is JSON and logs full of "must be" / "requires" / "rate + /// limit": only a test result is a reliable signal there. + #[test] + fn tool_output_only_matches_test_results() { + let tool = |text: &str| ClassifyInput { + tool_output: true, + ..input(text) + }; + + assert!(try_heuristic(&tool( + r#"Read: {"file_path":"a.rs"} → {"content":"the id must be unique; requires auth"}"# + )) + .is_none()); + assert!(try_heuristic(&tool( + r#"Bash: {"command":"x"} → {"stdout":"we'll use the cached build, rejected 2 files"}"# + )) + .is_none()); + + let out = try_heuristic(&tool( + r#"Bash: {"command":"cargo test"} → {"stdout":"all 12 tests pass"}"#, + )) + .unwrap(); + assert_eq!(out.event_type, EventType::Evidence); + } + #[test] fn suggested_text_is_truncated() { let long = "we'll use ".to_string() + &"x".repeat(500); diff --git a/crates/tj-core/src/classifier/http.rs b/crates/tj-core/src/classifier/http.rs index c828687..27d572b 100644 --- a/crates/tj-core/src/classifier/http.rs +++ b/crates/tj-core/src/classifier/http.rs @@ -130,6 +130,7 @@ mod tests { text: "We adopted Rust.".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }) .unwrap(); @@ -163,6 +164,7 @@ mod tests { text: "x".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }); let elapsed = start.elapsed(); diff --git a/crates/tj-core/src/classifier/hybrid.rs b/crates/tj-core/src/classifier/hybrid.rs index 3ae5e09..f2375c2 100644 --- a/crates/tj-core/src/classifier/hybrid.rs +++ b/crates/tj-core/src/classifier/hybrid.rs @@ -141,6 +141,7 @@ mod tests { last_events: vec![], constraints: vec![], }], + tool_output: false, } } diff --git a/crates/tj-core/src/classifier/mock.rs b/crates/tj-core/src/classifier/mock.rs index 8086cb3..66bf9e4 100644 --- a/crates/tj-core/src/classifier/mock.rs +++ b/crates/tj-core/src/classifier/mock.rs @@ -34,6 +34,7 @@ mod tests { text: "ignored".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }) .unwrap(); assert_eq!(out.event_type, EventType::Decision); diff --git a/crates/tj-core/src/classifier/mod.rs b/crates/tj-core/src/classifier/mod.rs index 4748d17..b682465 100644 --- a/crates/tj-core/src/classifier/mod.rs +++ b/crates/tj-core/src/classifier/mod.rs @@ -9,6 +9,11 @@ pub struct ClassifyInput { pub text: String, pub author_hint: String, pub recent_tasks: Vec, + /// The chunk is a tool call's input + output (PostToolUse). The + /// heuristic stage then only trusts test results; the LLM prompt is + /// unchanged. + #[serde(skip)] + pub tool_output: bool, } #[derive(Debug, Clone, Serialize)] @@ -133,6 +138,7 @@ mod tests { text: "Adopted Rust for the journal".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }; let s = serde_json::to_string(&i).unwrap(); assert!(s.contains("Adopted Rust")); diff --git a/crates/tj-core/src/classifier/prompt.rs b/crates/tj-core/src/classifier/prompt.rs index 5f0b75e..07fbda9 100644 --- a/crates/tj-core/src/classifier/prompt.rs +++ b/crates/tj-core/src/classifier/prompt.rs @@ -122,6 +122,7 @@ mod tests { last_events: vec!["[hypothesis] PKCE vs implicit".into()], constraints: vec![], }], + tool_output: false, }; let p = build(&input); assert!(p.contains("We adopted PKCE.")); @@ -145,6 +146,7 @@ mod tests { constraints: vec![], }) .collect(), + tool_output: false, }; let p = build(&input); assert!( @@ -160,6 +162,7 @@ mod tests { text: "anything".into(), author_hint: "assistant".into(), recent_tasks: vec![], + tool_output: false, }; let p = build(&input); assert!(p.contains("## Examples"), "Examples section missing"); @@ -188,6 +191,7 @@ mod tests { last_events: vec!["[decision] use ureq".into()], constraints: vec!["must stay sync (no tokio)".into()], }], + tool_output: false, }; let p = build(&input); assert!(p.contains("Known constraints for tj-9:")); @@ -207,6 +211,7 @@ mod tests { last_events: vec!["[finding] y".into()], constraints: vec![], }], + tool_output: false, }; let p = build(&input); assert!(!p.contains("Known constraints")); @@ -219,6 +224,7 @@ mod tests { text: "Hello".into(), author_hint: "user".into(), recent_tasks: vec![], + tool_output: false, }; let p = build(&input); assert!(p.contains("(no active tasks)")); From 66244a2472418efc4a736e1825ab64108659b158 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:43:31 +0400 Subject: [PATCH 028/104] perf(dream): read the events log once per run instead of once per session --- crates/tj-cli/src/main.rs | 72 ++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 31ef4c6..a8bce97 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -6071,26 +6071,18 @@ fn task_matches_session( }) } -/// Read the project's events from `events_path`, group by `task_id`, and -/// return candidate task contexts for sessions whose events match this -/// session (precise session_id, or legacy time-window). Each context -/// carries the task title and up to the last ~20 event texts (dedup -/// context for the backend). -fn candidate_tasks_for_session( +/// Read the project's events from `events_path`, grouped by `task_id`. +/// Read once per run and shared by every session's candidate lookup. +fn events_by_task( events_path: &std::path::Path, - session_id: &str, - first_ts: Option<&str>, - last_ts: Option<&str>, -) -> anyhow::Result> { - use std::collections::BTreeMap; - use tj_core::dream::backend::BackfillTaskContext; - use tj_core::event::{Event, EventType}; +) -> anyhow::Result>> { + use tj_core::event::Event; + let mut by_task = std::collections::BTreeMap::new(); if !events_path.exists() { - return Ok(Vec::new()); + return Ok(by_task); } let body = std::fs::read_to_string(events_path)?; - let mut by_task: BTreeMap> = BTreeMap::new(); for line in body.lines() { if line.trim().is_empty() { continue; @@ -6099,10 +6091,25 @@ fn candidate_tasks_for_session( by_task.entry(e.task_id.clone()).or_default().push(e); } } + Ok(by_task) +} + +/// Candidate task contexts for the tasks whose events match this session +/// (precise session_id, or legacy time-window). Each context carries the +/// task title and up to the last ~20 event texts (dedup context for the +/// backend). +fn candidate_tasks_for_session( + by_task: &std::collections::BTreeMap>, + session_id: &str, + first_ts: Option<&str>, + last_ts: Option<&str>, +) -> Vec { + use tj_core::dream::backend::BackfillTaskContext; + use tj_core::event::EventType; let mut out = Vec::new(); for (task_id, events) in by_task { - if !task_matches_session(&events, session_id, first_ts, last_ts) { + if !task_matches_session(events, session_id, first_ts, last_ts) { continue; } // Title from the Open event when present, else the first event's text. @@ -6120,12 +6127,12 @@ fn candidate_tasks_for_session( .map(|e| e.text.clone()) .collect(); out.push(BackfillTaskContext { - task_id, + task_id: task_id.clone(), title, existing_events, }); } - Ok(out) + out } /// Assemble per-session `(session_id, BackfillInput)` from the in-scope @@ -6138,6 +6145,7 @@ fn build_dream_inputs( use tj_core::dream::backend::BackfillInput; use tj_core::session::parser::parse_session; + let by_task = events_by_task(events_path)?; let mut out = Vec::new(); for path in sessions { let session_id = path @@ -6158,11 +6166,11 @@ fn build_dream_inputs( }; let candidates = candidate_tasks_for_session( - events_path, + &by_task, &session_id, parsed.first_timestamp.as_deref(), parsed.last_timestamp.as_deref(), - )?; + ); let tasks: Vec<_> = candidates .into_iter() .filter(|t| task_filter.is_none_or(|f| f == t.task_id)) @@ -6326,6 +6334,30 @@ mod inline_tests { )); } + #[test] + fn candidate_tasks_come_from_events_loaded_once() { + // The events log is grouped once per run and reused for every + // session, so candidate lookup takes the grouped map, not a path. + use tj_core::event::{Author, Event, EventType, Source}; + let mut tagged = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "Task one".into(), + ); + tagged.meta = serde_json::json!({"session_id": "sess-1"}); + let by_task = std::collections::BTreeMap::from([("tj-1".to_string(), vec![tagged])]); + + let hit = candidate_tasks_for_session(&by_task, "sess-1", None, None); + let miss = candidate_tasks_for_session(&by_task, "sess-2", None, None); + + assert_eq!(hit.len(), 1); + assert_eq!(hit[0].task_id, "tj-1"); + assert_eq!(hit[0].title, "Task one"); + assert!(miss.is_empty()); + } + #[test] fn build_dream_inputs_skips_an_unreadable_session() { use tj_core::event::{Author, Event, EventType, Source}; From 92c4993118d72f401669df6b235ecb6cdb6f9270 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:44:43 +0400 Subject: [PATCH 029/104] fix(hooks): stamp session_id on rewind, auto-open and sync-classified events --- crates/tj-cli/src/main.rs | 21 ++++++++-- crates/tj-cli/tests/cli.rs | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index be21794..183984a 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2738,6 +2738,7 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(1.0); event.status = tj_core::event::EventStatus::Confirmed; + tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; @@ -2833,8 +2834,13 @@ runs in the background and won't block you; it only fills gaps and never closes if auto_open_disabled || !kind.contains("UserPrompt") { return Ok(()); } - let Some(new_task) = - auto_open_task_from_prompt(&events_path, &project_hash, &conn, &text)? + let Some(new_task) = auto_open_task_from_prompt( + &events_path, + &project_hash, + &conn, + &text, + live_session_id.as_deref(), + )? else { // Prompt was only machine noise — nothing worth a task. return Ok(()); @@ -2966,6 +2972,7 @@ runs in the background and won't block you; it only fills gaps and never closes event.confidence = Some(confidence); event.status = tj_core::classifier::decide_status(confidence); event.evidence_strength = evidence_strength; + tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -5199,6 +5206,7 @@ fn auto_open_task_from_prompt( project_hash: &str, conn: &rusqlite::Connection, prompt: &str, + session_id: Option<&str>, ) -> anyhow::Result> { // Title/goal must read like a human wrote them on purpose. When the // prompt is only machine noise — session-start scrollback @@ -5220,6 +5228,7 @@ fn auto_open_task_from_prompt( title.clone(), ); event.meta = serde_json::json!({ "title": title, "auto_opened": true }); + tj_core::session_id::stamp_session_id(&mut event.meta, session_id); let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; writer.append(&event)?; @@ -5592,7 +5601,13 @@ fn process_pending_entry( std::fs::remove_file(path)?; return Ok(()); } - let Some(new_task) = auto_open_task_from_prompt(events_path, project_hash, &conn, &text)? + let Some(new_task) = auto_open_task_from_prompt( + events_path, + project_hash, + &conn, + &text, + chunk_session_id.as_deref(), + )? else { // Prompt was only machine noise — drop the entry silently. std::fs::remove_file(path)?; diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index dc2fd8d..3cef2d1 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -3721,6 +3721,92 @@ fn rewind_prompt_appends_correction_event() { .stdout(contains("[correction]").and(contains("/rewind"))); } +/// `meta.session_id` of the journal's events of `etype`. +fn session_ids_of(journal: &str, etype: &str) -> Vec> { + journal + .lines() + .map(|l| serde_json::from_str::(l).unwrap()) + .filter(|e| e["type"] == etype) + .map(|e| e["meta"]["session_id"].as_str().map(str::to_string)) + .collect() +} + +#[test] +fn hook_written_events_carry_the_session_id() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + let hook = |prompt: &str| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_INGEST_SYNC", "1") + .env_remove("CLAUDE_CODE_SESSION_ID") + .current_dir(&workdir) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin( + serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": "sess-hook", + "prompt": prompt, + }) + .to_string(), + ) + .assert() + .success(); + }; + + // No open task: the prompt auto-opens one and is classified inline. + hook("We decided to use postgres for the journal store going forward"); + hook("/rewind go back to plan A"); + + let journal = project_events(dir.path(), &hash); + let want = vec![Some("sess-hook".to_string())]; + assert_eq!(session_ids_of(&journal, "open"), want, "auto-open event"); + assert_eq!( + session_ids_of(&journal, "decision"), + want, + "sync-classified" + ); + assert_eq!(session_ids_of(&journal, "correction"), want, "/rewind"); +} + +#[test] +fn worker_auto_open_carries_the_chunk_session_id() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + + let pending = dir.path().join("task-journal").join("pending"); + let entry = write_pending_entry( + &pending, + &format!("{hash}.01JA0000000000000000000000.json"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use postgres for the journal store going forward", + ); + let mut v: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&entry).unwrap()).unwrap(); + v["session_id"] = serde_json::json!("sess-chunk"); + std::fs::write(&entry, v.to_string()).unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + let journal = project_events(dir.path(), &hash); + let want = vec![Some("sess-chunk".to_string())]; + assert_eq!(session_ids_of(&journal, "open"), want, "auto-open event"); + assert_eq!(session_ids_of(&journal, "decision"), want, "classified"); +} + #[test] fn install_hooks_wires_precompact_event() { let dir = assert_fs::TempDir::new().unwrap(); From 8ab801d9765c580617724d769aaff2f51ae06182 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:45:16 +0400 Subject: [PATCH 030/104] fix(cli): migrate-project checkpoints the WAL and moves SQLite sidecars --- crates/tj-cli/src/main.rs | 21 ++++++++++++++- crates/tj-cli/tests/cli.rs | 53 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 38b8934..8f606a0 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -157,7 +157,8 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool let state_dir = tj_core::paths::state_dir()?; let metrics_dir = tj_core::paths::metrics_dir()?; - // (source, destination) tuples to attempt to rename. + // (source, destination) tuples to attempt to rename. The SQLite runs in + // WAL mode, so its `-wal` / `-shm` sidecars travel with it. let pairs = [ ( events_dir.join(format!("{from_hash}.jsonl")), @@ -167,6 +168,14 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool state_dir.join(format!("{from_hash}.sqlite")), state_dir.join(format!("{to_hash}.sqlite")), ), + ( + state_dir.join(format!("{from_hash}.sqlite-wal")), + state_dir.join(format!("{to_hash}.sqlite-wal")), + ), + ( + state_dir.join(format!("{from_hash}.sqlite-shm")), + state_dir.join(format!("{to_hash}.sqlite-shm")), + ), ( metrics_dir.join(format!("{from_hash}.jsonl")), metrics_dir.join(format!("{to_hash}.jsonl")), @@ -185,6 +194,16 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool } } + // Fold uncheckpointed writes into the main file before it moves. When + // no one else holds the DB, closing this connection also deletes the + // sidecars; any that survive are moved with it below. + let src_state_path = state_dir.join(format!("{from_hash}.sqlite")); + if src_state_path.exists() { + let conn = rusqlite::Connection::open(&src_state_path)?; + conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .with_context(|| format!("checkpoint WAL of {src_state_path:?}"))?; + } + let mut moved: Vec = Vec::new(); for (src, dst) in &pairs { if !src.exists() { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ffc4480..e6ffd11 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -801,6 +801,59 @@ fn migrate_project_rekeys_embeddings_and_dream_state() { } } +/// A write still sitting in `.sqlite-wal` (another process holds the +/// DB open, so nothing checkpointed it) must reach the new project, and no +/// sidecar may stay behind under the old hash. Unix-only: Windows refuses to +/// rename a file another handle keeps open. +#[cfg(unix)] +#[test] +fn migrate_project_carries_uncheckpointed_wal_writes() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "WAL survives migration"]) + .assert() + .success(); + + let src = state.join(format!("{from_hash}.sqlite")); + let holder = tj_core::db::open(&src).unwrap(); + holder + .execute_batch("PRAGMA wal_autocheckpoint=0;") + .unwrap(); + holder + .execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES (?1, 't', 't')", + [&from_hash], + ) + .unwrap(); + // Keep the connection open for the rest of the test so its WAL is never + // checkpointed or deleted on close. + std::mem::forget(holder); + let wal = |hash: &str| state.join(format!("{hash}.sqlite-wal")); + assert!(wal(&from_hash).exists(), "precondition: source WAL present"); + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + for suffix in ["-wal", "-shm"] { + let left = state.join(format!("{from_hash}.sqlite{suffix}")); + assert!(!left.exists(), "sidecar left behind: {left:?}"); + } + let conn = rusqlite::Connection::open(state.join(format!("{to_hash}.sqlite"))).unwrap(); + let rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dream_state WHERE project_hash = ?1", + [&to_hash], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(rows, 1, "the WAL-only write was lost in migration"); +} + #[test] fn close_unknown_task_id_returns_error() { let dir = assert_fs::TempDir::new().unwrap(); From 958fac5f0149ecf8d652f3e66734aa576d183c33 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:46:11 +0400 Subject: [PATCH 031/104] fix(cli): migrate-project --force no longer deletes the destination before the move --- crates/tj-cli/src/main.rs | 6 ++-- crates/tj-cli/tests/cli.rs | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 8f606a0..e16fac6 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -212,9 +212,9 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool if let Some(parent) = dst.parent() { std::fs::create_dir_all(parent)?; } - if dst.exists() && force { - std::fs::remove_file(dst).with_context(|| format!("remove existing {dst:?}"))?; - } + // No remove-then-rename: rename replaces an existing destination + // atomically (POSIX rename, MOVEFILE_REPLACE_EXISTING on Windows), so + // a failed move under --force leaves the destination intact. std::fs::rename(src, dst).with_context(|| format!("rename {src:?} -> {dst:?}"))?; moved.push(dst.display().to_string()); } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index e6ffd11..3eaa8f4 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -854,6 +854,65 @@ fn migrate_project_carries_uncheckpointed_wal_writes() { assert_eq!(rows, 1, "the WAL-only write was lost in migration"); } +#[test] +fn migrate_project_force_keeps_destination_when_the_move_fails() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let events = xdg.path().join("task-journal").join("events"); + + // A directory cannot be renamed over a file, so the move must fail. + std::fs::create_dir_all(events.join(format!("{from_hash}.jsonl"))).unwrap(); + let dst = events.join(format!("{to_hash}.jsonl")); + std::fs::write(&dst, "destination data\n").unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + "--force", + ]) + .assert() + .failure(); + + assert_eq!( + std::fs::read_to_string(&dst).unwrap(), + "destination data\n", + "a failed --force move must leave the destination intact" + ); +} + +#[test] +fn migrate_project_force_replaces_existing_destination() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, _, _) = two_projects(); + + for (proj, title) in [(&proj_a, "Source task"), (&proj_b, "Overwritten task")] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", title]) + .assert() + .success(); + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), true); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_b.path()) + .args(["events", "list"]) + .assert() + .success() + .stdout(contains("Source task").and(contains("Overwritten task").not())); +} + #[test] fn close_unknown_task_id_returns_error() { let dir = assert_fs::TempDir::new().unwrap(); From c706d5f436fdd893202042d7987f25d2a51aa57d Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:46:19 +0400 Subject: [PATCH 032/104] fix(hooks): keep model-switch constraints out of the reminder and classifier context --- crates/tj-cli/src/main.rs | 83 ++++++++++++++++++++++++++++++---- crates/tj-cli/tests/cli.rs | 11 +++++ crates/tj-core/src/reminder.rs | 44 ++++++++++++++++-- 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 183984a..bffa271 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2466,7 +2466,10 @@ runs in the background and won't block you; it only fills gaps and never closes .get("source") .and_then(|v| v.as_str()) .unwrap_or("user"); - let text = format!("Model switched ({switch_source}): {from_model} → {to_model}"); + let text = format!( + "{}{switch_source}): {from_model} → {to_model}", + tj_core::reminder::MODEL_SWITCH_TEXT_PREFIX + ); let mut event = tj_core::event::Event::new( &tc.task_id, tj_core::event::EventType::Constraint, @@ -2476,6 +2479,10 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(0.9); event.status = tj_core::event::EventStatus::Confirmed; + // Kept in the journal, but left out of the constraint lists + // (resume reminder, classifier context) — see + // MODEL_SWITCH_TEXT_PREFIX. + event.meta = serde_json::json!({ "kind": "model_switch" }); tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -4099,17 +4106,22 @@ fn recent_task_contexts( "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id WHERE ei.task_id = ?1 AND ei.type = 'constraint' + AND COALESCE(sf.text, '') NOT LIKE ?3 ORDER BY ei.timestamp DESC LIMIT ?2", )?; + let model_switch = format!("{}%", tj_core::reminder::MODEL_SWITCH_TEXT_PREFIX); let constraints: Vec = c_stmt - .query_map(rusqlite::params![task_id, CONSTRAINT_CONTEXT_LIMIT], |r| { - let txt: Option = r.get(0)?; - Ok(txt - .unwrap_or_default() - .chars() - .take(120) - .collect::()) - })? + .query_map( + rusqlite::params![task_id, CONSTRAINT_CONTEXT_LIMIT, model_switch], + |r| { + let txt: Option = r.get(0)?; + Ok(txt + .unwrap_or_default() + .chars() + .take(120) + .collect::()) + }, + )? .collect::, _>>()? .into_iter() .filter(|s| !s.is_empty()) @@ -6493,6 +6505,59 @@ mod inline_tests { .any(|s| s.contains("constraint number 1"))); } + #[test] + fn recent_task_contexts_skips_model_switch_constraints() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = dir.path().join("h.sqlite"); + + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + let mut open = Event::new( + "tj-1", + EventType::Open, + Author::User, + Source::Cli, + "task one".into(), + ); + open.meta = serde_json::json!({ "title": "task one" }); + open.timestamp = "2026-01-01T00:00:00Z".into(); + writer.append(&open).unwrap(); + for i in 0..8 { + // Five real constraints, then three newer model switches. + let text = if i < 5 { + format!("constraint number {i}") + } else { + format!("Model switched (auto): opus → haiku {i}") + }; + let mut cons = Event::new( + "tj-1", + EventType::Constraint, + Author::Agent, + Source::Hook, + text, + ); + cons.timestamp = format!("2026-01-01T00:00:1{i}Z"); + writer.append(&cons).unwrap(); + } + writer.flush_durable().unwrap(); + + let conn = tj_core::db::open(&state_path).unwrap(); + tj_core::db::ingest_new_events(&conn, &events_path, "h").unwrap(); + + let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); + assert_eq!(ctx.constraints.len(), 5, "{:?}", ctx.constraints); + assert!( + ctx.constraints + .iter() + .all(|s| s.starts_with("constraint number")), + "{:?}", + ctx.constraints + ); + } + #[test] fn topic_is_fts_safe_basic() { assert!(topic_is_fts_safe("oauth")); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 3cef2d1..36949f5 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1109,6 +1109,17 @@ fn post_model_switch_records_a_constraint_on_the_active_task() { .stdout(contains( "Model switched (auto): claude-opus-5 → claude-haiku-4-5", )); + + // Marked so readers of the journal can tell it from a real constraint. + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + let journal = project_events(dir.path(), &hash); + let switch: serde_json::Value = journal + .lines() + .map(|l| serde_json::from_str::(l).unwrap()) + .find(|e| e["type"] == "constraint") + .expect("model switch recorded"); + assert_eq!(switch["meta"]["kind"], "model_switch", "{switch}"); + assert_eq!(switch["meta"]["session_id"], "s-switch", "{switch}"); } /// No open task → a model switch is not a reason to start one. diff --git a/crates/tj-core/src/reminder.rs b/crates/tj-core/src/reminder.rs index 19e4d6c..da2bd41 100644 --- a/crates/tj-core/src/reminder.rs +++ b/crates/tj-core/src/reminder.rs @@ -5,6 +5,12 @@ use rusqlite::Connection; pub const MAX_CONSTRAINTS: usize = 3; +/// Text prefix of the `constraint` events ingest-hook writes on a model +/// switch (they also carry `meta.kind = "model_switch"`). SQLite does not +/// index `meta`, so constraint lists filter on this prefix to keep a run of +/// switches from crowding out the task's real constraints. +pub const MODEL_SWITCH_TEXT_PREFIX: &str = "Model switched ("; + /// Most-recent OPEN task → "title + goal + up to MAX_CONSTRAINTS newest /// constraint texts". `None` when there is no open task. Read-only. pub fn active_task_reminder(conn: &Connection, label: &str) -> anyhow::Result> { @@ -30,12 +36,15 @@ pub fn active_task_reminder(conn: &Connection, label: &str) -> anyhow::Result = stmt - .query_map(rusqlite::params![task_id, MAX_CONSTRAINTS as i64], |r| { - r.get::<_, Option>(0) - })? + .query_map( + rusqlite::params![task_id, MAX_CONSTRAINTS as i64, model_switch], + |r| r.get::<_, Option>(0), + )? .filter_map(|r| r.ok().flatten()) .filter(|t| !t.trim().is_empty()) .collect(); @@ -126,6 +135,35 @@ mod tests { assert!(!r.contains("OLDEST"), "oldest constraint leaked: {r}"); } + #[test] + fn reminder_skips_model_switch_constraints() { + // Model switches are recorded as constraints, but a run of them must + // not push the task's real constraints out of the 3 slots. + let events = vec![ + open_event("tj-1", "Build the widget"), + constraint_event("tj-1", "API key rotates daily", "2026-06-01T00:00:00Z"), + constraint_event("tj-1", "Must support offline mode", "2026-06-02T00:00:00Z"), + constraint_event("tj-1", "Ship before Friday", "2026-06-03T00:00:00Z"), + constraint_event( + "tj-1", + "Model switched (auto): opus → haiku", + "2026-06-04T00:00:00Z", + ), + constraint_event( + "tj-1", + "Model switched (user): haiku → opus", + "2026-06-05T00:00:00Z", + ), + ]; + let (_d, conn) = seed(&events); + + let r = active_task_reminder(&conn, "Active task").unwrap().unwrap(); + assert!(r.contains("API key rotates daily"), "got: {r}"); + assert!(r.contains("Must support offline mode"), "got: {r}"); + assert!(r.contains("Ship before Friday"), "got: {r}"); + assert!(!r.contains("Model switched"), "got: {r}"); + } + #[test] fn reminder_none_when_no_open_task() { let (_d, conn) = seed(&[]); From f31bb228fc969eddafb3c4d070d34d591f56e20a Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:47:21 +0400 Subject: [PATCH 033/104] fix(cli): search and rejected --all-projects warn about projects they skip --- crates/tj-cli/src/main.rs | 38 ++++++++++++++++++++++++++++++++----- crates/tj-cli/tests/cli.rs | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index e16fac6..bfafcff 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3260,7 +3260,10 @@ runs in the background and won't block you; it only fills gaps and never closes let path = state_dir.join(format!("{hash}.sqlite")); let conn = match rusqlite::Connection::open(&path) { Ok(c) => c, - Err(_) => continue, + Err(e) => { + warn_skipped_project(&hash, e); + continue; + } }; let ids = match run_search( &conn, @@ -3270,7 +3273,10 @@ runs in the background and won't block you; it only fills gaps and never closes limit, ) { Ok(v) => v, - Err(_) => continue, + Err(e) => { + warn_skipped_project(&hash, e); + continue; + } }; for id in ids { println!("{hash}\t{id}"); @@ -3701,6 +3707,12 @@ fn run_search( Ok(ids_like) } +/// Cross-project reads keep going past a project they cannot read, but say +/// so on stderr instead of dropping it silently. +fn warn_skipped_project(hash: &str, err: impl std::fmt::Display) { + eprintln!("warning: skipping project {hash}: {err}"); +} + fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option) -> Result<()> { let cutoff: Option = since.map(|d| { (chrono::Utc::now() - chrono::Duration::days(d)) @@ -3733,11 +3745,21 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option = Vec::new(); + // Only the --all-projects sweep reports a skipped project: the current + // project of a fresh clone legitimately has no tables yet. + let skip = |hash: &str, e: rusqlite::Error| { + if all_projects { + warn_skipped_project(hash, e); + } + }; for hash in hashes { let path = state_dir.join(format!("{hash}.sqlite")); let conn = match rusqlite::Connection::open(&path) { Ok(c) => c, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; let use_fts = topic_is_fts_safe(topic); @@ -3763,7 +3785,10 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option s, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; let bind_q = if use_fts { topic.to_string() @@ -3780,7 +3805,10 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option r, - Err(_) => continue, + Err(e) => { + skip(&hash, e); + continue; + } }; for row in rows.flatten() { hits.push(row); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 3eaa8f4..46c18ef 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -956,6 +956,45 @@ fn search_all_projects_finds_match_in_other_project_hash() { .stdout(contains("aaaa1111").and(contains("bbbb2222"))); } +#[test] +fn all_projects_search_and_rejected_warn_about_unreadable_projects() { + let dir = assert_fs::TempDir::new().unwrap(); + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + + let good = "aaaa1111aaaa1111"; + let conn = tj_core::db::open(state.join(format!("{good}.sqlite"))).unwrap(); + let mut e = tj_core::event::Event::new( + "tj-good1".to_string(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + "Marker in a healthy project".to_string(), + ); + e.meta = serde_json::json!({"title": "Healthy"}); + tj_core::db::upsert_task_from_event(&conn, &e, good).unwrap(); + tj_core::db::index_event(&conn, &e).unwrap(); + drop(conn); + std::fs::write(state.join("cccc3333cccc3333.sqlite"), b"not a database").unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["search", "Marker", "--all-projects"]) + .assert() + .success() + .stdout(contains(good)) + .stderr(contains("warning: skipping project cccc3333cccc3333")); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["rejected", "Marker", "--all-projects"]) + .assert() + .success() + .stderr(contains("warning: skipping project cccc3333cccc3333")); +} + #[test] fn search_command_finds_task_by_event_text() { let dir = assert_fs::TempDir::new().unwrap(); From 2a63059e4187dce762aa8e582248402092df34a8 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:47:51 +0400 Subject: [PATCH 034/104] fix(tui): chat view title no longer panics on short or non-ASCII session ids --- crates/tj-cli/src/tui/chat_view.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/tui/chat_view.rs b/crates/tj-cli/src/tui/chat_view.rs index 07adeb7..59dde03 100644 --- a/crates/tj-cli/src/tui/chat_view.rs +++ b/crates/tj-cli/src/tui/chat_view.rs @@ -118,7 +118,8 @@ impl ChatView { .unwrap_or(&clean); truncate(line.trim(), 60) } else { - format!("Session {}", &session.session_id[..8]) + let head: String = session.session_id.chars().take(8).collect(); + format!("Session {head}") }; ChatView { @@ -347,6 +348,31 @@ fn format_ts(ts: &str) -> String { mod tests { use super::*; + // --- from_session() title fallback --- + + fn session_with_id(id: &str) -> ParsedSession { + ParsedSession { + session_id: id.into(), + file_path: String::new(), + entries: vec![], + first_timestamp: None, + last_timestamp: None, + } + } + + #[test] + fn title_fallback_survives_short_session_id() { + let view = ChatView::from_session(&session_with_id("abc")); + assert_eq!(view.title, "Session abc"); + } + + #[test] + fn title_fallback_cuts_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let view = ChatView::from_session(&session_with_id("aééééé")); + assert_eq!(view.title, "Session aééééé"); + } + // --- word_wrap() tests --- #[test] From 0c739560809feacd94e1b35e51665e661f51b753 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:48:31 +0400 Subject: [PATCH 035/104] fix(dream): advance the watermark only past cleanly mined sessions on unscoped runs --- crates/tj-cli/src/main.rs | 52 ++++++-- crates/tj-cli/tests/dream.rs | 154 ++++++++++++++++++++++++ crates/tj-core/src/dream/backend.rs | 27 ++++- crates/tj-core/src/dream/llm_backend.rs | 34 ++++-- crates/tj-core/src/dream/mod.rs | 44 ++++++- crates/tj-core/src/dream/scope.rs | 40 ++++++ 6 files changed, 319 insertions(+), 32 deletions(-) create mode 100644 crates/tj-cli/tests/dream.rs diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index a8bce97..dcedef9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -4524,11 +4524,13 @@ fn run_dream_op( Some(tj_core::dream::scope::SessionFile { path: p, mtime }) }) .collect(); + let mtimes: std::collections::HashMap = + scoped.iter().map(|s| (s.path.clone(), s.mtime)).collect(); let in_scope = tj_core::dream::scope::in_scope(scoped, since_time, limit); // 2. Assemble (session_id, BackfillInput) per session. let run_id = ulid::Ulid::new().to_string(); - let sessions = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; + let (sessions, unreadable) = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; let opts = tj_core::dream::DreamOptions { project_hash: project_hash.clone(), @@ -4561,12 +4563,25 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l &run_id, )?; - // 4. Advance watermark to now (only reached on success). - tj_core::dream::state::set_last_dream_at( - &conn, - &project_hash, - &chrono::Utc::now().to_rfc3339(), - )?; + // 4. Advance the watermark — only on an unscoped run (--task / --limit / + // --since skip sessions they never looked at), and only to the newest + // session such that it and every older in-scope one were mined cleanly. + if since.is_none() && task.is_none() && limit.is_none() { + let mined: Vec<(std::time::SystemTime, bool)> = in_scope + .iter() + .map(|p| { + let id = p.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + let clean = !unreadable.iter().any(|u| u == id) + && !report.failed_sessions.iter().any(|f| f == id); + (mtimes[p], clean) + }) + .collect(); + if let Some(t) = tj_core::dream::scope::next_watermark(&mined) { + let at = chrono::DateTime::::from(t).to_rfc3339(); + tj_core::dream::state::set_last_dream_at(&conn, &project_hash, &at)?; + } + } + let mut summary = format!( "dream: {} session(s) processed, {} event(s) backfilled", report.sessions_processed, report.events_backfilled @@ -4577,6 +4592,12 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l report.events_dropped_unknown_task )); } + if !report.failed_sessions.is_empty() { + summary.push_str(&format!( + ", {} only partly mined (retried next run)", + report.failed_sessions.len() + )); + } println!("{summary}"); Ok(()) } @@ -4709,7 +4730,7 @@ fn task_sessions( }) .collect(); let in_scope = tj_core::dream::scope::in_scope(scoped, None, None); - build_dream_inputs(events_path, &in_scope, Some(task_id)) + Ok(build_dream_inputs(events_path, &in_scope, Some(task_id))?.0) } /// Enrich a single task from every session that touched it. Unlike `dream`, @@ -6135,18 +6156,23 @@ fn candidate_tasks_for_session( out } +/// Per-session `(session_id, BackfillInput)` pairs fed to `run_dream`. +type DreamInputs = Vec<(String, tj_core::dream::backend::BackfillInput)>; + /// Assemble per-session `(session_id, BackfillInput)` from the in-scope -/// session transcripts and the project's existing events. +/// session transcripts and the project's existing events. Also returns the +/// ids of sessions skipped as unreadable. fn build_dream_inputs( events_path: &std::path::Path, sessions: &[std::path::PathBuf], task_filter: Option<&str>, -) -> anyhow::Result> { +) -> anyhow::Result<(DreamInputs, Vec)> { use tj_core::dream::backend::BackfillInput; use tj_core::session::parser::parse_session; let by_task = events_by_task(events_path)?; let mut out = Vec::new(); + let mut unreadable = Vec::new(); for path in sessions { let session_id = path .file_stem() @@ -6161,6 +6187,7 @@ fn build_dream_inputs( "dream: skipping unreadable session {}: {e:#}", path.display() ); + unreadable.push(session_id); continue; } }; @@ -6182,7 +6209,7 @@ fn build_dream_inputs( let transcript = flatten_transcript(&parsed); out.push((session_id, BackfillInput { tasks, transcript })); } - Ok(out) + Ok((out, unreadable)) } #[cfg(test)] @@ -6382,10 +6409,11 @@ mod inline_tests { std::fs::write(&good, "{\"type\":\"user\",\"uuid\":\"u1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{\"content\":\"hi\"}}\n").unwrap(); - let inputs = build_dream_inputs(&events_path, &[bad, good], None).unwrap(); + let (inputs, unreadable) = build_dream_inputs(&events_path, &[bad, good], None).unwrap(); assert_eq!(inputs.len(), 1); assert_eq!(inputs[0].0, "good"); + assert_eq!(unreadable, vec!["bad".to_string()]); } #[test] diff --git a/crates/tj-cli/tests/dream.rs b/crates/tj-cli/tests/dream.rs new file mode 100644 index 0000000..639ac5a --- /dev/null +++ b/crates/tj-cli/tests/dream.rs @@ -0,0 +1,154 @@ +//! End-to-end checks for the offline mining commands (`dream`, `backfill`) +//! against a sandboxed Claude Code projects dir and a mocked LLM backend. +//! Unix-only: the fixtures rely on the OS reporting a canonical cwd. +#![cfg(unix)] + +use assert_cmd::Command; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +struct Fixture { + xdg: assert_fs::TempDir, + proj: assert_fs::TempDir, + claude: assert_fs::TempDir, + /// `/projects/` — where transcripts live. + sessions_dir: PathBuf, + events_path: PathBuf, +} + +/// A project with an (empty) Claude Code sessions dir and events log dir. +fn fixture() -> Fixture { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + // The CLI resolves the project from its cwd, which the OS reports + // canonicalised (macOS `/var` → `/private/var`). + let path = proj.path().canonicalize().unwrap(); + let sessions_dir = + claude + .path() + .join("projects") + .join(tj_core::session::discovery::encode_project_path( + &path.to_string_lossy(), + )); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let hash = tj_core::project_hash::from_path(&path).unwrap(); + let events_path = xdg + .path() + .join("task-journal") + .join("events") + .join(format!("{hash}.jsonl")); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + Fixture { + xdg, + proj, + claude, + sessions_dir, + events_path, + } +} + +/// Write a two-turn transcript containing `marker`, aged `ago_secs`. +fn write_session(fx: &Fixture, id: &str, marker: &str, ago_secs: u64) { + let p = fx.sessions_dir.join(format!("{id}.jsonl")); + let body = format!( + "{{\"type\":\"user\",\"uuid\":\"u1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{{\"content\":\"{marker} first question\"}}}}\n\ + {{\"type\":\"assistant\",\"uuid\":\"a1\",\"timestamp\":\"2026-01-01T00:00:01Z\",\"message\":{{\"content\":[{{\"type\":\"text\",\"text\":\"an answer\"}}]}}}}\n\ + {{\"type\":\"user\",\"uuid\":\"u2\",\"timestamp\":\"2026-01-01T00:00:02Z\",\"message\":{{\"content\":\"second question\"}}}}\n" + ); + std::fs::write(&p, body).unwrap(); + let f = std::fs::File::options().write(true).open(&p).unwrap(); + f.set_modified(SystemTime::now() - Duration::from_secs(ago_secs)) + .unwrap(); +} + +/// Tie task `tj-dream` to each session via `meta.session_id`. +fn write_task_events(fx: &Fixture, session_ids: &[&str]) { + use tj_core::event::{Author, Event, EventType, Source}; + let mut w = tj_core::storage::JsonlWriter::open(&fx.events_path).unwrap(); + for sid in session_ids { + let mut e = Event::new( + "tj-dream", + EventType::Open, + Author::User, + Source::Cli, + "Dream task".into(), + ); + e.meta = serde_json::json!({ "session_id": sid }); + w.append(&e).unwrap(); + } + w.flush_durable().unwrap(); +} + +/// Mock OpenAI-compatible server: transcripts with OKSESSION succeed (no +/// events), transcripts with FAILME get a 500. +fn mock_llm() -> (mockito::ServerGuard, Vec) { + let mut server = mockito::Server::new(); + let ok = server + .mock("POST", "/v1/chat/completions") + .match_body(mockito::Matcher::Regex("OKSESSION".into())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"choices":[{"message":{"content":"[]"}}]}"#) + .create(); + let fail = server + .mock("POST", "/v1/chat/completions") + .match_body(mockito::Matcher::Regex("FAILME".into())) + .with_status(500) + .create(); + (server, vec![ok, fail]) +} + +fn dream(fx: &Fixture, url: &str, extra: &[&str]) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .current_dir(fx.proj.path()) + .env("XDG_DATA_HOME", fx.xdg.path()) + .env("CLAUDE_CONFIG_DIR", fx.claude.path()) + .env("TJ_OLLAMA_URL", url) + .args(["dream", "--backend", "ollama"]) + .args(extra) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn dream_scoped_run_does_not_advance_the_watermark() { + let fx = fixture(); + write_session(&fx, "sess-old", "OKSESSION", 7200); + write_session(&fx, "sess-new", "OKSESSION", 3600); + write_task_events(&fx, &["sess-old", "sess-new"]); + let (server, _mocks) = mock_llm(); + + let scoped = dream(&fx, &server.url(), &["--limit", "1"]); + assert!(scoped.contains("1 session(s) processed"), "{scoped}"); + + // The --limit run skipped sess-old, so an unscoped run still sees both. + let full = dream(&fx, &server.url(), &[]); + assert!(full.contains("2 session(s) processed"), "{full}"); + + // Everything mined cleanly → watermark reached the newest session. + let again = dream(&fx, &server.url(), &[]); + assert!(again.contains("0 session(s) processed"), "{again}"); +} + +#[test] +fn dream_session_with_a_failed_chunk_is_mined_again() { + let fx = fixture(); + write_session(&fx, "sess-old", "OKSESSION", 7200); + write_session(&fx, "sess-new", "FAILME", 3600); + write_task_events(&fx, &["sess-old", "sess-new"]); + let (server, _mocks) = mock_llm(); + + let first = dream(&fx, &server.url(), &[]); + assert!(first.contains("2 session(s) processed"), "{first}"); + + // sess-old was clean so the watermark moved to it; sess-new failed and + // must stay in scope. + let second = dream(&fx, &server.url(), &[]); + assert!(second.contains("1 session(s) processed"), "{second}"); +} diff --git a/crates/tj-core/src/dream/backend.rs b/crates/tj-core/src/dream/backend.rs index 1434ce5..464fa78 100644 --- a/crates/tj-core/src/dream/backend.rs +++ b/crates/tj-core/src/dream/backend.rs @@ -33,21 +33,34 @@ pub struct BackfillTaskContext { pub existing_events: Vec, } +/// What one session's backfill call produced. +#[derive(Debug, Default)] +pub struct BackfillOutput { + pub events: Vec, + /// Transcript chunks whose call failed or whose reply was unusable. + /// Non-zero means the session was only partly mined. + pub failed_chunks: usize, +} + pub trait DreamBackend { /// Return the events the realtime classifier missed for this session. - fn backfill(&self, input: &BackfillInput) -> anyhow::Result>; + fn backfill(&self, input: &BackfillInput) -> anyhow::Result; } /// Test backend that returns a canned list, ignoring the input. #[cfg(test)] pub struct MockDreamBackend { pub events: Vec, + pub failed_chunks: usize, } #[cfg(test)] impl DreamBackend for MockDreamBackend { - fn backfill(&self, _input: &BackfillInput) -> anyhow::Result> { - Ok(self.events.clone()) + fn backfill(&self, _input: &BackfillInput) -> anyhow::Result { + Ok(BackfillOutput { + events: self.events.clone(), + failed_chunks: self.failed_chunks, + }) } } @@ -64,14 +77,16 @@ mod tests { text: "Chose A over B.".into(), timestamp: "2026-06-08T10:00:00Z".into(), }], + failed_chunks: 0, }; let input = BackfillInput { tasks: vec![], transcript: "x".into(), }; let out = be.backfill(&input).unwrap(); - assert_eq!(out.len(), 1); - assert_eq!(out[0].task_id, "tj-1"); - assert_eq!(out[0].event_type, EventType::Decision); + assert_eq!(out.events.len(), 1); + assert_eq!(out.events[0].task_id, "tj-1"); + assert_eq!(out.events[0].event_type, EventType::Decision); + assert_eq!(out.failed_chunks, 0); } } diff --git a/crates/tj-core/src/dream/llm_backend.rs b/crates/tj-core/src/dream/llm_backend.rs index 777ab4c..7b6e110 100644 --- a/crates/tj-core/src/dream/llm_backend.rs +++ b/crates/tj-core/src/dream/llm_backend.rs @@ -5,7 +5,7 @@ use anyhow::Context; -use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; +use crate::dream::backend::{BackfillEvent, BackfillInput, BackfillOutput, DreamBackend}; use crate::llm::LlmBackend; /// Adapts any [`LlmBackend`] into a [`DreamBackend`]: build the dream prompt, @@ -44,8 +44,8 @@ impl LlmDreamBackend { const TRANSCRIPT_CHAR_BUDGET: usize = 150_000; impl DreamBackend for LlmDreamBackend { - fn backfill(&self, input: &BackfillInput) -> anyhow::Result> { - let mut out = Vec::new(); + fn backfill(&self, input: &BackfillInput) -> anyhow::Result { + let mut out = BackfillOutput::default(); for chunk in chunk_transcript(&input.transcript, TRANSCRIPT_CHAR_BUDGET) { let chunk_input = BackfillInput { tasks: input.tasks.clone(), @@ -57,17 +57,22 @@ impl DreamBackend for LlmDreamBackend { // (model continued the transcript dialogue) — is skipped, never // aborting the finalize. A genuinely broken backend still surfaces // at the judge step, which has its own (small, always-sized) call. + // Skips are counted so `dream` keeps the session in scope. match self.llm.complete_usage(&prompt, 1024) { Ok((text, usage)) => { self.usage.lock().unwrap().add(usage); match parse_backfill_json(&text) { - Ok(evs) => out.extend(evs), + Ok(evs) => out.events.extend(evs), Err(e) => { + out.failed_chunks += 1; tracing::warn!(error = %e, "dream backfill: skipping unparseable chunk") } } } - Err(e) => tracing::warn!(error = %e, "dream backfill: skipping failed chunk"), + Err(e) => { + out.failed_chunks += 1; + tracing::warn!(error = %e, "dream backfill: skipping failed chunk") + } } } Ok(out) @@ -176,8 +181,11 @@ mod tests { tasks: vec![], transcript: "user: hi\nassistant: hello".into(), }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + // Reported, not silently swallowed, so the caller won't advance the + // dream watermark past this session. + assert_eq!(out.failed_chunks, 1); } #[test] @@ -200,8 +208,9 @@ mod tests { tasks: vec![], transcript: "user: hi\nassistant: hello".into(), }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + assert_eq!(out.failed_chunks, 1); } #[test] @@ -251,8 +260,9 @@ mod tests { tasks: vec![], transcript, }; - let evs = b.backfill(&input).unwrap(); - assert!(evs.is_empty()); + let out = b.backfill(&input).unwrap(); + assert!(out.events.is_empty()); + assert_eq!(out.failed_chunks, 0); } #[test] @@ -275,7 +285,7 @@ mod tests { tasks: vec![], transcript: "x".into(), }; - let evs = b.backfill(&input).unwrap(); + let evs = b.backfill(&input).unwrap().events; assert_eq!(evs.len(), 1); assert_eq!(evs[0].text, "found it"); assert_eq!(b.backend_name(), "fake"); diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index 14b7405..ba9e292 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -26,6 +26,9 @@ pub struct DreamReport { /// Proposals dropped because their `task_id` was not one of the /// session's candidate tasks (the model invented or misspelled it). pub events_dropped_unknown_task: usize, + /// Sessions with at least one failed transcript chunk — only partly + /// mined, so the watermark must not move past them. + pub failed_sessions: Vec, } /// Run one dream Pass A over the given sessions, using the supplied @@ -48,7 +51,11 @@ pub fn run_dream( if opts.dry_run { continue; } - let mut proposed = backend.backfill(&input)?; + let out = backend.backfill(&input)?; + if out.failed_chunks > 0 { + report.failed_sessions.push(session_id.clone()); + } + let mut proposed = out.events; // Only the session's candidate tasks may receive events; anything // else would be an orphan event for a nonexistent task. @@ -130,6 +137,7 @@ mod tests { timestamp: "2026-06-08T10:01:00Z".into(), }, ], + failed_chunks: 0, }; let opts = DreamOptions { project_hash: "ph".into(), @@ -173,6 +181,7 @@ mod tests { timestamp: "2026-06-08T10:01:00Z".into(), }, ], + failed_chunks: 0, }; let opts = DreamOptions { project_hash: "ph".into(), @@ -209,6 +218,7 @@ mod tests { text: "Chose SQLite over Postgres.".into(), timestamp: "2026-06-08T10:00:00Z".into(), }], + failed_chunks: 0, }; let opts = DreamOptions { project_hash: "ph".into(), @@ -231,12 +241,42 @@ mod tests { assert_eq!(report.events_backfilled, 1); } + #[test] + fn run_dream_reports_sessions_with_failed_chunks() { + let d = TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + let events_path = d.path().join("events.jsonl"); + let backend = MockDreamBackend { + events: vec![], + failed_chunks: 1, + }; + let opts = DreamOptions { + project_hash: "ph".into(), + dry_run: false, + }; + + let report = run_dream( + &conn, + &events_path, + &opts, + &backend, + vec![task_input()], + "run-1", + ) + .unwrap(); + + assert_eq!(report.failed_sessions, vec!["sess-1".to_string()]); + } + #[test] fn dry_run_writes_nothing_and_skips_backend() { let d = TempDir::new().unwrap(); let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); let events_path = d.path().join("events.jsonl"); - let backend = MockDreamBackend { events: vec![] }; + let backend = MockDreamBackend { + events: vec![], + failed_chunks: 0, + }; let opts = DreamOptions { project_hash: "ph".into(), dry_run: true, diff --git a/crates/tj-core/src/dream/scope.rs b/crates/tj-core/src/dream/scope.rs index b758116..adf56fb 100644 --- a/crates/tj-core/src/dream/scope.rs +++ b/crates/tj-core/src/dream/scope.rs @@ -31,6 +31,25 @@ pub fn in_scope( out } +/// Where an unscoped run may move the watermark, given every in-scope +/// session as `(mtime, mined_cleanly)`: the newest clean mtime older than +/// every failed or skipped session, so the next run (which keeps only +/// mtimes strictly after the watermark) still sees those. `None` = leave +/// the watermark where it is. +pub fn next_watermark(sessions: &[(SystemTime, bool)]) -> Option { + let first_bad = sessions + .iter() + .filter(|(_, clean)| !clean) + .map(|(t, _)| *t) + .min(); + + sessions + .iter() + .filter(|(t, clean)| *clean && first_bad.is_none_or(|bad| *t < bad)) + .map(|(t, _)| *t) + .max() +} + #[cfg(test)] mod tests { use super::*; @@ -64,6 +83,27 @@ mod tests { ); } + #[test] + fn watermark_advances_to_newest_when_all_clean() { + let s = [(at(300), true), (at(100), true), (at(200), true)]; + assert_eq!(next_watermark(&s), Some(at(300))); + } + + #[test] + fn watermark_never_jumps_over_a_failed_session() { + // 200 failed → stop at 100, so 200 and 300 are mined again. + let s = [(at(100), true), (at(200), false), (at(300), true)]; + assert_eq!(next_watermark(&s), Some(at(100))); + // Oldest failed → nothing to advance to. + let s = [(at(100), false), (at(200), true)]; + assert_eq!(next_watermark(&s), None); + // A clean session sharing the failed one's mtime can't be the + // watermark either (`in_scope` keeps only mtime > watermark). + let s = [(at(100), true), (at(200), true), (at(200), false)]; + assert_eq!(next_watermark(&s), Some(at(100))); + assert_eq!(next_watermark(&[]), None); + } + #[test] fn none_since_keeps_all_limit_caps() { let s = vec![ From 74d42798b06e3c8bb5d999d91d59cf6781480d13 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:48:55 +0400 Subject: [PATCH 036/104] fix(tui): task detail reads the project given to ui --project, not cwd --- crates/tj-cli/src/tui/app.rs | 43 +++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/tj-cli/src/tui/app.rs b/crates/tj-cli/src/tui/app.rs index 0845924..55e9ee3 100644 --- a/crates/tj-cli/src/tui/app.rs +++ b/crates/tj-cli/src/tui/app.rs @@ -16,7 +16,7 @@ use crossterm::{ }; use ratatui::{backend::CrosstermBackend, Terminal}; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use tj_core::session::{discovery, parser}; use super::chat_view::ChatView; @@ -42,6 +42,8 @@ pub struct App { pub session_list: Option, pub chat_view: Option, pub should_quit: bool, + /// The project this UI browses (`ui --project`, else cwd). + project_path: PathBuf, } impl App { @@ -68,6 +70,7 @@ impl App { session_list: None, chat_view: None, should_quit: false, + project_path: project_path.to_path_buf(), }) } @@ -108,6 +111,7 @@ impl App { session_list: Some(SessionList::new(items, project_str)), chat_view: None, should_quit: false, + project_path: project_path.to_path_buf(), }) } @@ -226,8 +230,7 @@ impl App { // The state SQLite already exists (App::new opened it). // Re-resolve through paths to avoid storing the connection // on App and dealing with !Send across the render loop. - let cwd = std::env::current_dir()?; - let project_hash = tj_core::project_hash::from_path(&cwd)?; + let project_hash = tj_core::project_hash::from_path(&self.project_path)?; let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; // Full mode: show the complete reasoning chain — every event, @@ -325,3 +328,37 @@ impl App { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn task_detail_reads_the_project_passed_to_ui_not_cwd() { + let data = tempfile::TempDir::new().unwrap(); + let proj = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(proj.path().join(".git")).unwrap(); + // The only test in this binary that resolves the data dir. + std::env::set_var("TASK_JOURNAL_DATA_DIR", data.path()); + + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + let state = tj_core::paths::state_dir() + .unwrap() + .join(format!("{hash}.sqlite")); + let conn = tj_core::db::open(&state).unwrap(); + let open = tj_core::event::Event::new( + "tj-ui1".to_string(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + "Picked via --project".to_string(), + ); + tj_core::db::upsert_task_from_event(&conn, &open, &hash).unwrap(); + tj_core::db::index_event(&conn, &open).unwrap(); + drop(conn); + + let app = App::new(proj.path()).unwrap(); + let pack = app.assemble_pack("tj-ui1").unwrap(); + assert!(pack.contains("Picked via --project"), "{pack}"); + } +} From 3c8e89a90587d46fe06f39790f8c8438b1a9d70e Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:49:38 +0400 Subject: [PATCH 037/104] fix(tui): ignore key release events so keys do not fire twice --- crates/tj-cli/src/tui/app.rs | 54 +++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/crates/tj-cli/src/tui/app.rs b/crates/tj-cli/src/tui/app.rs index 55e9ee3..00966d7 100644 --- a/crates/tj-cli/src/tui/app.rs +++ b/crates/tj-cli/src/tui/app.rs @@ -10,7 +10,7 @@ use anyhow::Result; use crossterm::{ - event::{self, Event, KeyCode, KeyModifiers}, + event::{self, Event, KeyCode, KeyEvent, KeyEventKind, KeyModifiers}, execute, terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, }; @@ -158,18 +158,7 @@ impl App { if event::poll(std::time::Duration::from_millis(100))? { if let Event::Key(key) = event::read()? { - if key.modifiers.contains(KeyModifiers::CONTROL) - && key.code == KeyCode::Char('c') - { - self.should_quit = true; - } - - match &self.screen { - Screen::TaskList => self.handle_task_list_input(key.code), - Screen::TaskDetail => self.handle_task_detail_input(key.code), - Screen::SessionList => self.handle_session_list_input(key.code), - Screen::Chat => self.handle_chat_input(key.code), - } + self.handle_key(key); } } @@ -180,6 +169,25 @@ impl App { Ok(()) } + fn handle_key(&mut self, key: KeyEvent) { + // Terminals that report key releases (Windows) would otherwise fire + // every key twice. + if key.kind != KeyEventKind::Press { + return; + } + + if key.modifiers.contains(KeyModifiers::CONTROL) && key.code == KeyCode::Char('c') { + self.should_quit = true; + } + + match &self.screen { + Screen::TaskList => self.handle_task_list_input(key.code), + Screen::TaskDetail => self.handle_task_detail_input(key.code), + Screen::SessionList => self.handle_session_list_input(key.code), + Screen::Chat => self.handle_chat_input(key.code), + } + } + fn handle_task_list_input(&mut self, key: KeyCode) { let Some(ref mut tl) = self.task_list else { return; @@ -361,4 +369,24 @@ mod tests { let pack = app.assemble_pack("tj-ui1").unwrap(); assert!(pack.contains("Picked via --project"), "{pack}"); } + + #[test] + fn key_release_events_are_ignored() { + let mut app = App { + screen: Screen::TaskList, + task_list: Some(TaskList::new(vec![], "proj".into())), + task_detail: None, + session_list: None, + chat_view: None, + should_quit: false, + project_path: PathBuf::from("proj"), + }; + let q = |kind| KeyEvent::new_with_kind(KeyCode::Char('q'), KeyModifiers::NONE, kind); + + app.handle_key(q(KeyEventKind::Release)); + assert!(!app.should_quit, "a key release must not act as a press"); + + app.handle_key(q(KeyEventKind::Press)); + assert!(app.should_quit); + } } From 04dd0bcbf788bf369f3e72d29776321ca5828853 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:49:46 +0400 Subject: [PATCH 038/104] fix(backfill): detect imported sessions by meta.session_id and index them into SQLite --- crates/tj-cli/src/main.rs | 33 ++++++++++++------- crates/tj-cli/tests/dream.rs | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index dcedef9..58766c2 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3349,17 +3349,21 @@ runs in the background and won't block you; it only fills gaps and never closes project_path.display() ); - // Check which sessions are already imported (idempotent). - let already_imported = if events_path.exists() { - let content = std::fs::read_to_string(&events_path).unwrap_or_default(); - sessions - .iter() - .filter_map(|p| p.file_stem().and_then(|s| s.to_str()).map(String::from)) - .filter(|sid| content.contains(sid)) - .collect::>() - } else { - std::collections::HashSet::new() - }; + // Check which sessions are already imported (idempotent): a session + // counts once some event is tagged with it in `meta.session_id` — + // not merely mentioned in some unrelated event's text. + let already_imported: std::collections::HashSet = + std::fs::read_to_string(&events_path) + .unwrap_or_default() + .lines() + .filter_map(|l| serde_json::from_str::(l).ok()) + .filter_map(|e| { + e.meta + .get("session_id") + .and_then(|v| v.as_str()) + .map(String::from) + }) + .collect(); let mut total_tasks = 0; let mut total_events = 0; @@ -3452,6 +3456,13 @@ runs in the background and won't block you; it only fills gaps and never closes ); eprintln!("Run without --dry-run to import."); } else { + // Index the appended events so search / pack see them now. + if total_tasks > 0 { + let state_path = + tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } eprintln!("\nImported {total_tasks} task(s) with {total_events} event(s)."); } } diff --git a/crates/tj-cli/tests/dream.rs b/crates/tj-cli/tests/dream.rs index 639ac5a..3f13fe4 100644 --- a/crates/tj-cli/tests/dream.rs +++ b/crates/tj-cli/tests/dream.rs @@ -14,6 +14,7 @@ struct Fixture { /// `/projects/` — where transcripts live. sessions_dir: PathBuf, events_path: PathBuf, + state_path: PathBuf, } /// A project with an (empty) Claude Code sessions dir and events log dir. @@ -39,12 +40,18 @@ fn fixture() -> Fixture { .join("events") .join(format!("{hash}.jsonl")); std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = xdg + .path() + .join("task-journal") + .join("state") + .join(format!("{hash}.sqlite")); Fixture { xdg, proj, claude, sessions_dir, events_path, + state_path, } } @@ -152,3 +159,60 @@ fn dream_session_with_a_failed_chunk_is_mined_again() { let second = dream(&fx, &server.url(), &[]); assert!(second.contains("1 session(s) processed"), "{second}"); } + +fn backfill(fx: &Fixture) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .current_dir(fx.proj.path()) + .env("XDG_DATA_HOME", fx.xdg.path()) + .env("CLAUDE_CONFIG_DIR", fx.claude.path()) + .args(["backfill"]) + .assert() + .success() + .get_output() + .stderr + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn backfill_imports_a_session_only_mentioned_in_unrelated_text() { + use tj_core::event::{Author, Event, EventType, Source}; + let fx = fixture(); + write_session(&fx, "sess-mention", "hello", 60); + // Another task's text names the session id; it was never imported. + let e = Event::new( + "tj-other", + EventType::Finding, + Author::User, + Source::Cli, + "see sess-mention for the repro".into(), + ); + let mut w = tj_core::storage::JsonlWriter::open(&fx.events_path).unwrap(); + w.append(&e).unwrap(); + w.flush_durable().unwrap(); + + let first = backfill(&fx); + assert!(first.contains("Imported 1 task(s)"), "{first}"); + + // Now it really was imported (its events carry meta.session_id). + let second = backfill(&fx); + assert!(second.contains("already imported"), "{second}"); + assert!(second.contains("Imported 0 task(s)"), "{second}"); +} + +#[test] +fn backfill_indexes_imported_tasks_into_sqlite() { + let fx = fixture(); + write_session(&fx, "sess-index", "hello", 60); + + let out = backfill(&fx); + assert!(out.contains("Imported 1 task(s)"), "{out}"); + + // search / pack read SQLite, so the task must be there right away. + let conn = rusqlite::Connection::open(&fx.state_path).unwrap(); + let tasks: i64 = conn + .query_row("SELECT COUNT(*) FROM tasks", [], |r| r.get(0)) + .unwrap(); + assert_eq!(tasks, 1); +} From 6c7e44e9a05be7c38476ea9e8b77a8f21a573d0f Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:50:27 +0400 Subject: [PATCH 039/104] fix(tui): run every terminal restore step and report the first error --- crates/tj-cli/src/tui/app.rs | 53 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/crates/tj-cli/src/tui/app.rs b/crates/tj-cli/src/tui/app.rs index 00966d7..8518e64 100644 --- a/crates/tj-cli/src/tui/app.rs +++ b/crates/tj-cli/src/tui/app.rs @@ -124,9 +124,14 @@ impl App { let result = self.main_loop(&mut terminal); - disable_raw_mode()?; - execute!(terminal.backend_mut(), LeaveAlternateScreen)?; - terminal.show_cursor()?; + run_restore_steps( + &mut terminal, + &[ + |_| disable_raw_mode(), + |t| execute!(t.backend_mut(), LeaveAlternateScreen), + |t| t.show_cursor(), + ], + )?; result } @@ -337,10 +342,52 @@ impl App { } } +/// Terminal restore sequence on exit. Every step runs even when an earlier +/// one fails — a half-restored terminal (raw mode, alternate screen, hidden +/// cursor) is worse than a late error — and the first error is reported. +fn run_restore_steps(target: &mut T, steps: &[fn(&mut T) -> io::Result<()>]) -> io::Result<()> { + let mut first = Ok(()); + + for step in steps { + let result = step(target); + if first.is_ok() { + first = result; + } + } + + first +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn restore_runs_every_step_and_reports_the_first_error() { + let mut log: Vec<&str> = Vec::new(); + let err = run_restore_steps( + &mut log, + &[ + |log| { + log.push("raw mode"); + Err(io::Error::other("raw mode failed")) + }, + |log| { + log.push("alt screen"); + Err(io::Error::other("alt screen failed")) + }, + |log| { + log.push("cursor"); + Ok(()) + }, + ], + ) + .unwrap_err(); + + assert_eq!(log, ["raw mode", "alt screen", "cursor"]); + assert_eq!(err.to_string(), "raw mode failed"); + } + #[test] fn task_detail_reads_the_project_passed_to_ui_not_cwd() { let data = tempfile::TempDir::new().unwrap(); From 074d910f3fb735eb69fbddaac099fe85b33fb048 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:50:53 +0400 Subject: [PATCH 040/104] fix(hooks): pending retry classifies through the worker's classifier path --- crates/tj-cli/src/main.rs | 236 +++++++++++++++++++++++++------------ crates/tj-cli/tests/cli.rs | 88 +++++++++++++- 2 files changed, 244 insertions(+), 80 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index bffa271..c63c689 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -430,6 +430,7 @@ fn run_pending_list() -> Result<()> { } fn run_pending_retry( + backend: &str, mock_etype: Option<&str>, mock_tid: Option<&str>, mock_conf: Option, @@ -443,6 +444,23 @@ fn run_pending_retry( let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + // The CI-safe mock branch needs no classifier. Without a usable backend + // a retry can't do better than the attempt that queued the entry, so + // leave everything as it is rather than burn attempts toward `.dead`. + let classifier = match (mock_etype, mock_tid) { + (Some(_), Some(_)) => None, + _ => match retry_classifier(backend)? { + Some(c) => Some(c), + None => { + println!( + "pending retry: no classifier backend available for `{backend}` \ + (no `claude` on PATH, no ANTHROPIC_API_KEY) — entries left untouched" + ); + return Ok(()); + } + }, + }; + let mut succeeded = 0usize; let mut died = 0usize; let mut still_pending = 0usize; @@ -469,11 +487,14 @@ fn run_pending_retry( .and_then(|x| x.as_str()) .unwrap_or("") .to_string(); + let kind = v + .get("kind") + .and_then(|x| x.as_str()) + .unwrap_or("Stop") + .to_string(); - // The real retry path would call the classifier. The CI-safe - // mock branch lets tests drive a deterministic outcome. - let outcome: anyhow::Result<()> = match (mock_etype, mock_tid) { - (Some(etype), Some(tid)) => { + let outcome: anyhow::Result<()> = match (mock_etype, mock_tid, &classifier) { + (Some(etype), Some(tid), _) => { let mut event = tj_core::event::Event::new( tid, parse_event_type(etype)?, @@ -488,9 +509,18 @@ fn run_pending_retry( writer.flush_durable()?; Ok(()) } - _ => Err(anyhow::anyhow!( - "no real classifier wired in retry path yet — pass --mock-* for tests, or run install-hooks and let the hook drain the queue" - )), + (_, _, Some(classifier)) => match classify_chunk( + classifier.as_ref(), + &events_path, + &project_hash, + &kind, + &text, + None, + )? { + ChunkOutcome::Unplaced(err) => Err(anyhow::anyhow!(err)), + ChunkOutcome::Recorded | ChunkOutcome::Dropped => Ok(()), + }, + _ => unreachable!("a missing classifier returns before the loop"), }; match outcome { @@ -526,6 +556,24 @@ fn run_pending_retry( Ok(()) } +/// The classifier `pending retry` runs, or `None` when the backend has +/// nothing beyond what already failed: hybrid without an LLM fallback, or +/// agent-sdk / api without `claude` / a key. +fn retry_classifier( + backend: &str, +) -> anyhow::Result>> { + Ok(match backend { + "hybrid" | "" => { + let hybrid = tj_core::classifier::hybrid::HybridClassifier::from_env(); + hybrid + .has_llm_fallback() + .then(|| Box::new(hybrid) as Box) + } + "agent-sdk" | "api" => build_classifier(backend).ok(), + other => Some(build_classifier(other)?), + }) +} + fn run_doctor() -> Result { let mut issues: Vec = Vec::new(); let mut notes: Vec = Vec::new(); @@ -1140,7 +1188,12 @@ enum PendingCmd { List, /// Re-feed every pending entry through the classifier. Marks an /// entry as `.dead.json` after PENDING_MAX_ATTEMPTS failures. + /// With no usable backend, entries are left untouched. Retry { + /// Classifier backend: "hybrid", "agent-sdk", "api", or "heuristic". + /// Defaults to hybrid. + #[arg(long, default_value = "hybrid")] + backend: String, /// Test/dev override: bypass classifier and force this event /// type. Hidden from --help. #[arg(long, hide = true)] @@ -2173,11 +2226,13 @@ fn real_main() -> Result<()> { run_pending_list()?; } PendingCmd::Retry { + backend, mock_event_type, mock_task_id, mock_confidence, } => { run_pending_retry( + &backend, mock_event_type.as_deref(), mock_task_id.as_deref(), mock_confidence, @@ -2908,7 +2963,7 @@ runs in the background and won't block you; it only fills gaps and never closes let out = match classifier.classify(&input) { Ok(o) => o, Err(e) => { - persist_pending(&events_path, &project_hash, &text, &e.to_string())?; + persist_pending(&events_path, &project_hash, &kind, &text, &e.to_string())?; return Ok(()); } }; @@ -2939,6 +2994,7 @@ runs in the background and won't block you; it only fills gaps and never closes persist_pending( &events_path, &project_hash, + &kind, &text, &format!("task_id_guess `{tid}` not found"), )?; @@ -2948,6 +3004,7 @@ runs in the background and won't block you; it only fills gaps and never closes persist_pending( &events_path, &project_hash, + &kind, &text, &format!("task_id_guess `{tid}` is closed"), )?; @@ -5289,6 +5346,7 @@ fn auto_open_task_from_prompt( fn persist_pending( events_path: &std::path::Path, project_hash: &str, + kind: &str, text: &str, err: &str, ) -> anyhow::Result<()> { @@ -5300,7 +5358,8 @@ fn persist_pending( .join("pending"); std::fs::create_dir_all(&pending_dir)?; let id = ulid::Ulid::new().to_string(); - let payload = serde_json::json!({"text": text, "error": err, "queued_at": chrono::Utc::now().to_rfc3339()}); + // `kind` lets `pending retry` classify the chunk the way the hook would. + let payload = serde_json::json!({"kind": kind, "text": text, "error": err, "queued_at": chrono::Utc::now().to_rfc3339()}); std::fs::write( pending_dir.join(format!("{project_hash}.{id}.json")), serde_json::to_string_pretty(&payload)?, @@ -5593,47 +5652,39 @@ fn process_pending_entry( // Inherit the session id queued on the v2 chunk (additive; absent → None). let chunk_session_id = tj_core::session_id::session_id_from_payload(&v); - // Mirror the synchronous flow that used to live in IngestHook — - // see commit history of v0.6.1 for the original. Auto-open, run - // classifier, apply integrity safeguards, persist event, telemetry. - let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + let classifier = build_classifier(backend)?; + let outcome = classify_chunk( + classifier.as_ref(), + events_path, + project_hash, + &kind, + &text, + chunk_session_id.as_deref(), + )?; + if let ChunkOutcome::Unplaced(err) = outcome { + // Persist as legacy v1 pending entry so `pending retry` + // surfaces it; remove the v2 source. + persist_pending(events_path, project_hash, &kind, &text, &err)?; } - let mut recent = recent_task_contexts(&conn, 5)?; - if recent.is_empty() { - let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") - .ok() - .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) - .unwrap_or(false); - if auto_open_disabled || !kind.contains("UserPrompt") { - // Nothing to do — drop the entry silently. - std::fs::remove_file(path)?; - return Ok(()); - } - let Some(new_task) = auto_open_task_from_prompt( - events_path, - project_hash, - &conn, - &text, - chunk_session_id.as_deref(), - )? - else { - // Prompt was only machine noise — drop the entry silently. - std::fs::remove_file(path)?; - return Ok(()); - }; - recent.push(new_task); - } + std::fs::remove_file(path)?; + Ok(()) +} - let author_hint = if kind.contains("UserPrompt") { - "user" - } else { - "assistant" - }; +/// What became of one classified chunk. +enum ChunkOutcome { + /// An event was written to the journal. + Recorded, + /// Nothing worth recording: no task to attach to, machine noise, no + /// task guess, or a session-end "close". + Dropped, + /// The classifier failed or guessed a missing / closed task; the reason + /// goes back into `pending/` with the chunk. + Unplaced(String), +} +/// The classifier behind a `--backend` name. +fn build_classifier(backend: &str) -> anyhow::Result> { use tj_core::classifier::Classifier; let classifier: Box = match backend { "hybrid" | "" => Box::new(tj_core::classifier::hybrid::HybridClassifier::from_env()), @@ -5665,53 +5716,83 @@ fn process_pending_entry( "unknown backend: {other} (expected `hybrid`, `agent-sdk`, `api`, or `heuristic`)" ), }; + Ok(classifier) +} + +/// Classify one chunk against the project's open tasks and record the +/// event. Shared by classify-worker and `pending retry`, so both auto-open, +/// check attribution, stamp the session and write telemetry the same way. +fn classify_chunk( + classifier: &dyn tj_core::classifier::Classifier, + events_path: &std::path::Path, + project_hash: &str, + kind: &str, + text: &str, + session_id: Option<&str>, +) -> anyhow::Result { + // Mirror the synchronous flow that used to live in IngestHook — + // see commit history of v0.6.1 for the original. Auto-open, run + // classifier, apply integrity safeguards, persist event, telemetry. + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; + } + + let mut recent = recent_task_contexts(&conn, 5)?; + if recent.is_empty() { + let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") + .ok() + .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) + .unwrap_or(false); + if auto_open_disabled || !kind.contains("UserPrompt") { + // Nothing to do — drop the entry silently. + return Ok(ChunkOutcome::Dropped); + } + let Some(new_task) = + auto_open_task_from_prompt(events_path, project_hash, &conn, text, session_id)? + else { + // Prompt was only machine noise — drop the entry silently. + return Ok(ChunkOutcome::Dropped); + }; + recent.push(new_task); + } + + let author_hint = if kind.contains("UserPrompt") { + "user" + } else { + "assistant" + }; + let input = tj_core::classifier::ClassifyInput { - text: text.clone(), + text: text.to_string(), author_hint: author_hint.into(), recent_tasks: recent, tool_output: kind == "PostToolUse", }; let out = match classifier.classify(&input) { Ok(o) => o, - Err(e) => { - // Persist as legacy v1 pending entry so `pending retry` - // surfaces it; remove the v2 source. - persist_pending(events_path, project_hash, &text, &e.to_string())?; - std::fs::remove_file(path)?; - return Ok(()); - } + Err(e) => return Ok(ChunkOutcome::Unplaced(e.to_string())), }; let Some(tid) = out.task_id_guess else { - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Dropped); }; use tj_core::event::EventType; if matches!(out.event_type, EventType::Close) && kind == "Stop" { - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Dropped); } match tj_core::db::task_status(&conn, &tid)? { None => { - persist_pending( - events_path, - project_hash, - &text, - &format!("task_id_guess `{tid}` not found"), - )?; - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Unplaced(format!( + "task_id_guess `{tid}` not found" + ))) } Some(s) if s == "closed" => { - persist_pending( - events_path, - project_hash, - &text, - &format!("task_id_guess `{tid}` is closed"), - )?; - std::fs::remove_file(path)?; - return Ok(()); + return Ok(ChunkOutcome::Unplaced(format!( + "task_id_guess `{tid}` is closed" + ))) } _ => {} } @@ -5731,7 +5812,7 @@ fn process_pending_entry( event.confidence = Some(confidence); event.status = tj_core::classifier::decide_status(confidence); event.evidence_strength = evidence_strength; - tj_core::session_id::stamp_session_id(&mut event.meta, chunk_session_id.as_deref()); + tj_core::session_id::stamp_session_id(&mut event.meta, session_id); let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; writer.append(&event)?; @@ -5759,8 +5840,7 @@ fn process_pending_entry( }, ); - std::fs::remove_file(path)?; - Ok(()) + Ok(ChunkOutcome::Recorded) } fn drain_pending( @@ -6319,7 +6399,7 @@ mod inline_tests { std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); persist_pending_v2(&events_path, "PostToolUse", "txt", "h", "hybrid", None).unwrap(); - persist_pending(&events_path, "h", "txt", "err").unwrap(); + persist_pending(&events_path, "h", "Stop", "txt", "err").unwrap(); let pending = dir.path().join("pending"); let names: Vec = std::fs::read_dir(&pending) diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 36949f5..441adb5 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -361,13 +361,22 @@ fn pending_retry_marks_dead_after_max_attempts() { let proj = assert_fs::TempDir::new().unwrap(); // Already at attempts=2; one more failure should rename to *.dead.json. write_pending(xdg.path(), "tj-dying", "any text", 2); + // An open task, so the chunk reaches the classifier at all. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Dying host"]) + .assert() + .success(); Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", xdg.path()) .current_dir(proj.path()) - // No --mock-* flags → retry fails → attempts becomes 3 → dead. - .args(["pending", "retry"]) + // The heuristic can't place "any text" → a real classifier failure + // → attempts becomes 3 → dead. + .args(["pending", "retry", "--backend", "heuristic"]) .assert() .success() .stdout(contains("1 marked dead")); @@ -379,6 +388,81 @@ fn pending_retry_marks_dead_after_max_attempts() { assert!(dead.exists(), "dead file must exist: {dead:?}"); } +#[test] +fn pending_retry_without_a_backend_leaves_entries_untouched() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "tj-waiting", "any text", 2); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + // Hybrid with no LLM fallback can only repeat the heuristic that + // already failed — that is "no backend", not a failed attempt. + .env("TJ_HYBRID_LLM_ORDER", "none") + .env_remove("ANTHROPIC_API_KEY") + .current_dir(proj.path()) + .args(["pending", "retry"]) + .assert() + .success() + .stdout(contains("no classifier backend")); + + let pending_dir = xdg.path().join("task-journal").join("pending"); + let live = pending_dir.join("tj-waiting.json"); + assert!(live.exists(), "entry must stay queued"); + assert!(!pending_dir.join("tj-waiting.dead.json").exists()); + let v: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&live).unwrap()).unwrap(); + assert_eq!(v["attempts"], 2, "no attempt is burned without a backend"); +} + +#[test] +fn pending_retry_classifies_through_the_real_backend() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Retry host"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + write_pending( + xdg.path(), + "tj-retry-real", + "After review we decided to use postgres for the journal store", + 1, + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pending", "retry", "--backend", "heuristic"]) + .assert() + .success() + .stdout(contains("1 drained")); + + let pending_dir = xdg.path().join("task-journal").join("pending"); + assert!(!pending_dir.join("tj-retry-real.json").exists()); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("[decision]").and(contains("postgres"))); +} + #[test] fn export_sqlite_round_trips_through_pack() { // Setup A: write a project + task in xdg_a/proj_a. From 6ad8e6d3af7c6f3ade978773af92c3c9eafa180d Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:51:08 +0400 Subject: [PATCH 041/104] feat(cli): doctor reports whether codex is on PATH --- crates/tj-cli/src/main.rs | 11 ++++++++ crates/tj-cli/tests/cli.rs | 51 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index bfafcff..ce14dfd 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -14,6 +14,8 @@ struct DoctorReport { task_journal_version: &'static str, claude_in_path: bool, claude_version: Option, + /// Informational only: needed just for `--backend codex`. + codex_in_path: bool, data_dir: PathBuf, events_dir: PathBuf, state_dir: PathBuf, @@ -48,6 +50,14 @@ impl DoctorReport { "NOT FOUND in PATH".into() } ); + println!( + " codex binary {}", + if self.codex_in_path { + "found" + } else { + "not found in PATH (only needed for the codex backend)" + } + ); println!(" data dir {}", self.data_dir.display()); println!( " events dir {} ({})", @@ -618,6 +628,7 @@ fn run_doctor() -> Result { task_journal_version: env!("CARGO_PKG_VERSION"), claude_in_path, claude_version, + codex_in_path: tj_core::llm::codex_on_path(), data_dir, events_dir, state_dir, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 46c18ef..e1c85bf 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -255,6 +255,57 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +/// `doctor` with PATH limited to `bin`, so only what the test puts there +/// resolves. +fn doctor_with_path(xdg: &std::path::Path, bin: &std::path::Path, json: bool) -> String { + let mut args = vec!["doctor"]; + if json { + args.push("--json"); + } + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg) + .env("PATH", bin) + .args(args) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +#[test] +fn doctor_reports_missing_codex_and_claude_as_information_only() { + let xdg = assert_fs::TempDir::new().unwrap(); + let empty_bin = assert_fs::TempDir::new().unwrap(); + + let v: serde_json::Value = + serde_json::from_str(&doctor_with_path(xdg.path(), empty_bin.path(), true)).unwrap(); + assert_eq!(v["codex_in_path"], false); + assert_eq!(v["claude_in_path"], false); + assert_eq!(v["issues"].as_array().unwrap().len(), 0, "{v}"); + + let human = doctor_with_path(xdg.path(), empty_bin.path(), false); + assert!(human.contains("codex binary"), "{human}"); +} + +#[cfg(unix)] +#[test] +fn doctor_detects_codex_on_path() { + use std::os::unix::fs::PermissionsExt; + + let xdg = assert_fs::TempDir::new().unwrap(); + let bin = assert_fs::TempDir::new().unwrap(); + let codex = bin.path().join("codex"); + std::fs::write(&codex, "#!/bin/sh\necho codex-cli 9.9.9\n").unwrap(); + std::fs::set_permissions(&codex, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let v: serde_json::Value = + serde_json::from_str(&doctor_with_path(xdg.path(), bin.path(), true)).unwrap(); + assert_eq!(v["codex_in_path"], true); +} + fn write_pending(xdg: &std::path::Path, id: &str, text: &str, attempts: u32) { let dir = xdg.join("task-journal").join("pending"); std::fs::create_dir_all(&dir).unwrap(); From a502ca7919989cfc88ec83afff22f7c803212884 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:51:23 +0400 Subject: [PATCH 042/104] fix(cli): say that dream and backfill read Claude Code transcripts only --- crates/tj-cli/src/main.rs | 8 ++++++-- crates/tj-cli/tests/dream.rs | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 58766c2..069406f 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3319,7 +3319,8 @@ runs in the background and won't block you; it only fills gaps and never closes Some(d) => d, None => { eprintln!( - "No Claude Code sessions found for: {}", + "No Claude Code sessions found for: {} — backfill reads Claude Code \ +transcripts only (Codex sessions are not read yet)", project_path.display() ); eprintln!( @@ -4509,7 +4510,10 @@ fn run_dream_op( // 1. Resolve session files in scope. let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; let Some(project_dir) = project_dir else { - println!("dream: no Claude Code sessions found for this project"); + println!( + "dream: no Claude Code session directory for this project — dream mines \ +Claude Code transcripts only (Codex sessions are not read yet)" + ); return Ok(()); }; let session_paths = tj_core::session::discovery::list_sessions(&project_dir)?; diff --git a/crates/tj-cli/tests/dream.rs b/crates/tj-cli/tests/dream.rs index 3f13fe4..44ff3f3 100644 --- a/crates/tj-cli/tests/dream.rs +++ b/crates/tj-cli/tests/dream.rs @@ -216,3 +216,23 @@ fn backfill_indexes_imported_tasks_into_sqlite() { .unwrap(); assert_eq!(tasks, 1); } + +#[test] +fn mining_without_claude_code_sessions_says_codex_is_not_read() { + // A Codex-only project has no `~/.claude/projects/` dir: say why + // nothing was mined instead of a bare zero. + let fx = fixture(); + std::fs::remove_dir(&fx.sessions_dir).unwrap(); + + let out = dream(&fx, "http://127.0.0.1:9", &[]); + assert!( + out.contains("Claude Code") && out.contains("Codex"), + "{out}" + ); + + let err = backfill(&fx); + assert!( + err.contains("Claude Code") && err.contains("Codex"), + "{err}" + ); +} From db54119c373781f188f601d37dfb20b29934b033 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:56:22 +0400 Subject: [PATCH 043/104] fix(tui): cut session ids on a char boundary in the session list --- crates/tj-cli/src/tui/chat_view.rs | 3 +-- crates/tj-cli/src/tui/mod.rs | 5 +++++ crates/tj-cli/src/tui/session_list.rs | 25 +++++++++++++++++++++---- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/crates/tj-cli/src/tui/chat_view.rs b/crates/tj-cli/src/tui/chat_view.rs index 59dde03..2f2a67f 100644 --- a/crates/tj-cli/src/tui/chat_view.rs +++ b/crates/tj-cli/src/tui/chat_view.rs @@ -118,8 +118,7 @@ impl ChatView { .unwrap_or(&clean); truncate(line.trim(), 60) } else { - let head: String = session.session_id.chars().take(8).collect(); - format!("Session {head}") + format!("Session {}", super::short_session_id(&session.session_id)) }; ChatView { diff --git a/crates/tj-cli/src/tui/mod.rs b/crates/tj-cli/src/tui/mod.rs index f11dd56..69732c8 100644 --- a/crates/tj-cli/src/tui/mod.rs +++ b/crates/tj-cli/src/tui/mod.rs @@ -6,3 +6,8 @@ pub mod chat_view; pub mod session_list; pub mod task_detail; pub mod task_list; + +/// First 8 characters of a session id, cut on a char boundary. +fn short_session_id(id: &str) -> String { + id.chars().take(8).collect() +} diff --git a/crates/tj-cli/src/tui/session_list.rs b/crates/tj-cli/src/tui/session_list.rs index 21fb1a6..437b1df 100644 --- a/crates/tj-cli/src/tui/session_list.rs +++ b/crates/tj-cli/src/tui/session_list.rs @@ -268,7 +268,7 @@ impl SessionList { s.assistant_message_count() ); let duration = format_duration(&s.first_timestamp, &s.last_timestamp); - let id_short = &s.session_id[..8.min(s.session_id.len())]; + let id_short = super::short_session_id(&s.session_id); let line = Line::from(vec![ Span::styled(format!("{date} "), Style::default().fg(Color::DarkGray)), @@ -355,8 +355,7 @@ fn session_title(s: &ParsedSession) -> String { .unwrap_or(&clean); truncate_with_ellipsis(line.trim(), 80) } else { - let head: String = s.session_id.chars().take(8).collect(); - format!("Session {head}") + format!("Session {}", super::short_session_id(&s.session_id)) } } @@ -431,7 +430,25 @@ fn shorten_path(path: &str) -> String { #[cfg(test)] mod tests { - use super::truncate_with_ellipsis; + use super::{truncate_with_ellipsis, SessionList}; + use ratatui::{backend::TestBackend, Terminal}; + use tj_core::session::parser::ParsedSession; + + #[test] + fn render_list_cuts_non_ascii_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let session = ParsedSession { + session_id: "aééééé".into(), + file_path: String::new(), + entries: vec![], + first_timestamp: None, + last_timestamp: None, + }; + let list = SessionList::new(vec![session], String::new()); + let mut terminal = Terminal::new(TestBackend::new(80, 12)).unwrap(); + + terminal.draw(|frame| list.render(frame)).unwrap(); + } #[test] fn truncate_ascii_under_limit_returns_input_as_is() { From 9caa6fb72ea6abe26aedc56a4c891dff31b1dfc7 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:57:21 +0400 Subject: [PATCH 044/104] fix(cli): migrate-project re-keys the project's rows in global memory --- crates/tj-cli/src/main.rs | 9 ++++++++ crates/tj-cli/tests/cli.rs | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 3a58c2d..c2970da 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -241,6 +241,15 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool } } + // The global cross-project index keys its rows by project_hash too. + let memory_path = tj_core::paths::memory_db()?; + if memory_path.exists() { + tj_core::memory::open(&memory_path)?.execute( + "UPDATE global_memory SET project_hash = ?1 WHERE project_hash = ?2", + rusqlite::params![to_hash, from_hash], + )?; + } + if moved.is_empty() { println!("no on-disk data found for project_hash {from_hash} — nothing to migrate"); } else { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ec20e95..603bc92 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -936,6 +936,48 @@ fn migrate_project_rekeys_embeddings_and_dream_state() { } } +#[test] +fn migrate_project_rekeys_only_its_rows_in_global_memory() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Re-key global memory"]) + .assert() + .success(); + let memory_path = xdg.path().join("task-journal").join("memory.sqlite"); + { + let global = tj_core::memory::open(&memory_path).unwrap(); + for (event_id, hash) in [("e-mine", from_hash.as_str()), ("e-other", "otherhash")] { + global + .execute( + "INSERT INTO global_memory(event_id, project_hash, task_id, type, text, model, dim, vec, created_at) + VALUES (?1, ?2, 'tj-1', 'decision', 't', 'hash', 1, x'00000000', 't')", + [event_id, hash], + ) + .unwrap(); + } + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + let global = rusqlite::Connection::open(&memory_path).unwrap(); + let hash_of = |event_id: &str| -> String { + global + .query_row( + "SELECT project_hash FROM global_memory WHERE event_id = ?1", + [event_id], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!(hash_of("e-mine"), to_hash); + assert_eq!(hash_of("e-other"), "otherhash"); +} + /// A write still sitting in `.sqlite-wal` (another process holds the /// DB open, so nothing checkpointed it) must reach the new project, and no /// sidecar may stay behind under the old hash. Unix-only: Windows refuses to From f21d5ccce6eea0a76758e355a32a590159500f2d Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:58:24 +0400 Subject: [PATCH 045/104] fix(cli): backfill --limit skips already-imported sessions before cutting --- crates/tj-cli/src/main.rs | 45 ++++++++++++++++++++++-------------- crates/tj-cli/tests/dream.rs | 25 ++++++++++++++++---- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index c2970da..9ae5463 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3399,23 +3399,6 @@ transcripts only (Codex sessions are not read yet)", } }; - // List available sessions. - let mut sessions = discovery::list_sessions(&proj_dir)?; - if let Some(max) = limit { - sessions.truncate(max); - } - - if sessions.is_empty() { - eprintln!("No session JSONL files found in: {}", proj_dir.display()); - return Ok(()); - } - - eprintln!( - "Found {} session(s) for {}", - sessions.len(), - project_path.display() - ); - // Check which sessions are already imported (idempotent): a session // counts once some event is tagged with it in `meta.session_id` — // not merely mentioned in some unrelated event's text. @@ -3432,6 +3415,34 @@ transcripts only (Codex sessions are not read yet)", }) .collect(); + // List available sessions. --limit counts only sessions not yet + // imported, so a rerun reaches older ones; imported ones stay in + // the list and are reported as skipped below. + let mut sessions = discovery::list_sessions(&proj_dir)?; + if let Some(max) = limit { + let mut fresh = 0; + sessions.retain(|p| { + let id = p.file_stem().and_then(|s| s.to_str()).unwrap_or("?"); + if already_imported.contains(id) { + return true; + } + + fresh += 1; + fresh <= max + }); + } + + if sessions.is_empty() { + eprintln!("No session JSONL files found in: {}", proj_dir.display()); + return Ok(()); + } + + eprintln!( + "Found {} session(s) for {}", + sessions.len(), + project_path.display() + ); + let mut total_tasks = 0; let mut total_events = 0; diff --git a/crates/tj-cli/tests/dream.rs b/crates/tj-cli/tests/dream.rs index 44ff3f3..0924829 100644 --- a/crates/tj-cli/tests/dream.rs +++ b/crates/tj-cli/tests/dream.rs @@ -160,13 +160,14 @@ fn dream_session_with_a_failed_chunk_is_mined_again() { assert!(second.contains("1 session(s) processed"), "{second}"); } -fn backfill(fx: &Fixture) -> String { +fn backfill(fx: &Fixture, extra: &[&str]) -> String { let out = Command::cargo_bin("task-journal") .unwrap() .current_dir(fx.proj.path()) .env("XDG_DATA_HOME", fx.xdg.path()) .env("CLAUDE_CONFIG_DIR", fx.claude.path()) .args(["backfill"]) + .args(extra) .assert() .success() .get_output() @@ -192,21 +193,35 @@ fn backfill_imports_a_session_only_mentioned_in_unrelated_text() { w.append(&e).unwrap(); w.flush_durable().unwrap(); - let first = backfill(&fx); + let first = backfill(&fx, &[]); assert!(first.contains("Imported 1 task(s)"), "{first}"); // Now it really was imported (its events carry meta.session_id). - let second = backfill(&fx); + let second = backfill(&fx, &[]); assert!(second.contains("already imported"), "{second}"); assert!(second.contains("Imported 0 task(s)"), "{second}"); } +#[test] +fn backfill_limit_counts_only_sessions_not_yet_imported() { + let fx = fixture(); + write_session(&fx, "sess-old", "hello", 3600); + write_session(&fx, "sess-new", "hello", 60); + + let first = backfill(&fx, &["--limit", "1"]); + assert!(first.contains("Imported 1 task(s)"), "{first}"); + + // sess-new is imported now; the next --limit 1 run must reach sess-old. + let second = backfill(&fx, &["--limit", "1"]); + assert!(second.contains("Imported 1 task(s)"), "{second}"); +} + #[test] fn backfill_indexes_imported_tasks_into_sqlite() { let fx = fixture(); write_session(&fx, "sess-index", "hello", 60); - let out = backfill(&fx); + let out = backfill(&fx, &[]); assert!(out.contains("Imported 1 task(s)"), "{out}"); // search / pack read SQLite, so the task must be there right away. @@ -230,7 +245,7 @@ fn mining_without_claude_code_sessions_says_codex_is_not_read() { "{out}" ); - let err = backfill(&fx); + let err = backfill(&fx, &[]); assert!( err.contains("Claude Code") && err.contains("Codex"), "{err}" From d0c346a30919aed19203e296c40ba8d0999a8b19 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:58:45 +0400 Subject: [PATCH 046/104] fix(llm): name the binary that timed out instead of always claude -p --- crates/tj-core/src/classifier/agent_sdk.rs | 8 +++++--- crates/tj-core/src/llm.rs | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/tj-core/src/classifier/agent_sdk.rs b/crates/tj-core/src/classifier/agent_sdk.rs index 343278a..6ec870d 100644 --- a/crates/tj-core/src/classifier/agent_sdk.rs +++ b/crates/tj-core/src/classifier/agent_sdk.rs @@ -123,10 +123,12 @@ fn claude_timeout() -> std::time::Duration { /// Wait for `child` up to `timeout`, draining stdout/stderr concurrently so a /// full pipe can't deadlock the wait. On timeout the child is killed and an -/// error returned; otherwise the captured output is handed back. +/// error naming `what` (e.g. "claude -p") is returned; otherwise the captured +/// output is handed back. pub(crate) fn wait_with_timeout( mut child: std::process::Child, timeout: std::time::Duration, + what: &str, ) -> anyhow::Result { use std::io::Read; let mut out_pipe = child.stdout.take(); @@ -153,7 +155,7 @@ pub(crate) fn wait_with_timeout( if start.elapsed() >= timeout { let _ = child.kill(); let _ = child.wait(); - anyhow::bail!("`claude -p` timed out after {}s", timeout.as_secs()); + anyhow::bail!("`{what}` timed out after {}s", timeout.as_secs()); } std::thread::sleep(std::time::Duration::from_millis(150)); }; @@ -197,7 +199,7 @@ impl CommandRunner for ClaudeBinaryStdinRunner { .context("claude stdin was not captured")? .write_all(prompt.as_bytes()) .context("failed to write prompt to claude stdin")?; - let output = wait_with_timeout(child, claude_timeout())?; + let output = wait_with_timeout(child, claude_timeout(), "claude -p")?; if !output.status.success() { return Err(claude_exit_error( output.status, diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs index f96498b..7898fc1 100644 --- a/crates/tj-core/src/llm.rs +++ b/crates/tj-core/src/llm.rs @@ -243,7 +243,7 @@ impl LlmBackend for CodexCliBackend { .context("codex stdin was not captured")? .write_all(prompt.as_bytes()) .context("failed to write prompt to codex stdin")?; - crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout()) + crate::classifier::agent_sdk::wait_with_timeout(child, codex_timeout(), "codex exec") }; let output = run(); let answer = std::fs::read_to_string(&out_path).unwrap_or_default(); @@ -580,6 +580,24 @@ mod tests { assert!(left.is_empty(), "codex temp file leaked: {left:?}"); } + /// The timeout error names the binary that hung, not always `claude -p`. + #[cfg(unix)] + #[test] + fn codex_timeout_error_names_codex() { + let _l = ENV_LOCK.lock().unwrap(); + let dir = fake_codex_dir(); + let _path = path_with(dir.path()); + let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); + let _sleep = EnvGuard::set("FAKE_CODEX_SLEEP", "5"); + + let err = CodexCliBackend { model: None } + .complete_usage("classify me", 64) + .unwrap_err() + .to_string(); + + assert!(err.contains("`codex exec` timed out"), "{err}"); + } + #[test] fn unknown_backend_errors() { let _l = ENV_LOCK.lock().unwrap(); From ed0a5dcef73e0cd8b37ea9e2487d086b45ca6b56 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 12:59:11 +0400 Subject: [PATCH 047/104] fix(cli): rejected warns about rows it cannot read instead of dropping them silently --- crates/tj-cli/src/main.rs | 14 +++++++++-- crates/tj-cli/tests/cli.rs | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 9ae5463..d9b1786 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3863,8 +3863,18 @@ fn run_rejected(topic: &str, all_projects: bool, limit: usize, since: Option hits.push(row), + Err(e) => { + first_err.get_or_insert(e); + } + } + } + if let Some(e) = first_err { + eprintln!("warning: skipped unreadable rejection row(s) in project {hash}: {e}"); } } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 603bc92..29b9cab 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1172,6 +1172,54 @@ fn all_projects_search_and_rejected_warn_about_unreadable_projects() { .stderr(contains("warning: skipping project cccc3333cccc3333")); } +#[test] +fn rejected_warns_about_a_row_it_cannot_read_and_keeps_the_rest() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = assert_fs::TempDir::new().unwrap(); + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + let hash = "aaaa1111aaaa1111"; + let conn = tj_core::db::open(state.join(format!("{hash}.sqlite"))).unwrap(); + + for task_id in ["tj-good1", "tj-bad1"] { + let mut open = Event::new( + task_id, + EventType::Open, + Author::User, + Source::Cli, + "t".into(), + ); + open.meta = serde_json::json!({"title": task_id}); + let rejection = Event::new( + task_id, + EventType::Rejection, + Author::User, + Source::Cli, + format!("Marker rejected in {task_id}"), + ); + for e in [&open, &rejection] { + tj_core::db::upsert_task_from_event(&conn, e, hash).unwrap(); + tj_core::db::index_event(&conn, e).unwrap(); + } + } + // A BLOB title cannot be read as text, so this task's row fails to map. + conn.execute( + "UPDATE tasks SET title = x'00' WHERE task_id = 'tj-bad1'", + [], + ) + .unwrap(); + drop(conn); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["rejected", "Marker", "--all-projects"]) + .assert() + .success() + .stdout(contains("tj-good1")) + .stderr(contains("warning:").and(contains(hash))); +} + #[test] fn search_command_finds_task_by_event_text() { let dir = assert_fs::TempDir::new().unwrap(); From a51bde0b034fb7e3607af438b616f07e73d8c8c3 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:01:04 +0400 Subject: [PATCH 048/104] fix(core): extractor counts chars and only treats a real test runner step as a test command --- crates/tj-core/src/session/extractor.rs | 125 ++++++++++++++++++------ 1 file changed, 94 insertions(+), 31 deletions(-) diff --git a/crates/tj-core/src/session/extractor.rs b/crates/tj-core/src/session/extractor.rs index 368c49f..d940cdd 100644 --- a/crates/tj-core/src/session/extractor.rs +++ b/crates/tj-core/src/session/extractor.rs @@ -195,17 +195,15 @@ fn derive_title(session: &ParsedSession) -> String { .unwrap_or(&clean); let trimmed = first_line.trim(); // Skip empty or very short titles (likely slash commands). - if trimmed.len() > 5 { + if trimmed.chars().count() > 5 { return truncate(trimmed, 120); } } } } - format!( - "Session {}", - &session.session_id[..8.min(session.session_id.len())] - ) + let head: String = session.session_id.chars().take(8).collect(); + format!("Session {head}") } /// Strip XML/HTML-like tags from text (e.g. , ). @@ -322,17 +320,51 @@ fn classify_text_heuristic(task_id: &str, text: &str, timestamp: &str) -> Option None } -/// Check if a bash command is a test command. +const TEST_RUNNERS: [&str; 8] = [ + "cargo test", + "npm test", + "pytest", + "phpunit", + "jest", + "vitest", + "go test", + "make test", +]; + +/// True when a bash command actually runs a test runner: some `&&`/`;` step +/// must be it (after `VAR=value` assignments, an `npx` / `python -m` +/// launcher or a `vendor/bin/` path), so an `echo`/`grep` that merely +/// mentions "cargo test" doesn't count. fn is_test_command(cmd: &str) -> bool { let lower = cmd.to_lowercase(); - lower.contains("cargo test") - || lower.contains("npm test") - || lower.contains("pytest") - || lower.contains("phpunit") - || lower.contains("jest") - || lower.contains("vitest") - || lower.contains("go test") - || lower.contains("make test") + let is_env_assignment = |word: &&str| { + word.split_once('=').is_some_and(|(name, _)| { + !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + }) + }; + + lower.split("&&").flat_map(|s| s.split(';')).any(|step| { + let words: Vec<&str> = step + .split_whitespace() + .skip_while(is_env_assignment) + .collect(); + let words = match words.as_slice() { + ["npx", rest @ ..] => rest, + [python, "-m", rest @ ..] if python.starts_with("python") => rest, + all => all, + }; + let Some((bin, args)) = words.split_first() else { + return false; + }; + let bin = bin.rsplit('/').next().unwrap_or(bin); + + TEST_RUNNERS + .iter() + .any(|runner| match runner.split_once(' ') { + Some((runner_bin, sub)) => bin == runner_bin && args.first() == Some(&sub), + None => bin == *runner, + }) + }) } /// True when a bash command actually runs `git commit` (not `--amend`): some @@ -354,17 +386,11 @@ fn shorten_path(path: &str) -> String { } } -/// Truncate text to max_len, adding "…" if truncated. +/// Truncate text to max_len chars, adding "…" if truncated. fn truncate(text: &str, max_len: usize) -> String { - if text.len() <= max_len { - text.to_string() - } else { - let mut end = max_len; - // Don't cut in the middle of a UTF-8 char. - while end > 0 && !text.is_char_boundary(end) { - end -= 1; - } - format!("{}…", &text[..end]) + match text.char_indices().nth(max_len) { + Some((end, _)) => format!("{}…", &text[..end]), + None => text.to_string(), } } @@ -788,6 +814,35 @@ mod tests { assert!(title.contains("abcdefgh")); } + #[test] + fn derive_title_counts_chars_not_bytes() { + // 5 chars (9 bytes): as short as "/init", so it is skipped. + let session = ParsedSession { + session_id: "abcdefghij".into(), + file_path: "/tmp/s.jsonl".into(), + entries: vec![ + make_user_entry("u1", "t", "Да да"), + make_user_entry("u2", "t", "Implement the feature for user profiles"), + ], + first_timestamp: None, + last_timestamp: None, + }; + assert!(derive_title(&session).contains("Implement the feature")); + } + + #[test] + fn derive_title_fallback_cuts_session_id_on_char_boundary() { + // Byte 8 falls inside the fourth 'é'. + let session = ParsedSession { + session_id: "aééééé".into(), + file_path: "/tmp/s.jsonl".into(), + entries: vec![make_user_entry("u1", "t", "hi")], + first_timestamp: None, + last_timestamp: None, + }; + assert_eq!(derive_title(&session), "Session aééééé"); + } + #[test] fn derive_title_strips_xml_from_summary() { let session = ParsedSession { @@ -850,10 +905,21 @@ mod tests { assert!(is_test_command("go test ./...")); assert!(is_test_command("make test")); assert!(is_test_command("phpunit tests/Unit")); - assert!(is_test_command("echo 'cargo test'")); // matches because it contains "cargo test" assert!(!is_test_command("ls -la")); } + #[test] + fn is_test_command_requires_the_runner_to_be_the_command() { + assert!(is_test_command("cd crates && cargo test")); + assert!(is_test_command("cargo fmt; cargo test --workspace")); + assert!(is_test_command("RUST_LOG=debug cargo test")); + assert!(is_test_command("npx jest src/")); + assert!(is_test_command("./vendor/bin/phpunit tests/Unit")); + assert!(!is_test_command("echo 'cargo test'")); + assert!(!is_test_command("grep -rn \"pytest\" .")); + assert!(!is_test_command("cat jest.config.js")); + } + // --- shorten_path additional tests --- #[test] @@ -873,12 +939,9 @@ mod tests { #[test] fn test_truncate_multibyte_utf8() { - // Russian text: each char is 2 bytes. - let text = "Привет мир"; - let truncated = truncate(text, 6); - // 6 bytes = 3 cyrillic chars ("При") - assert!(truncated.ends_with('…')); - assert!(truncated.starts_with("При")); + // Russian text: each char is 2 bytes, but the limit counts chars. + assert_eq!(truncate("Привет мир", 6), "Привет…"); + assert_eq!(truncate("Привет", 6), "Привет"); } #[test] From c8f7762fdfe6b7dc129d5956add505ff7236b837 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:02:26 +0400 Subject: [PATCH 049/104] fix(cli): --enrich says it reads Claude Code transcripts only when the project has none --- crates/tj-cli/src/main.rs | 21 +++++++++++++++++-- crates/tj-cli/tests/cli.rs | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d9b1786..3b7b303 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -5092,6 +5092,23 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l ); } +/// The Claude Code session dir that `--enrich` reads. With `--enrich` and no +/// such dir, say why nothing gets enriched instead of staying silent. +fn complete_project_dir( + cwd: &std::path::Path, + enrich: bool, +) -> anyhow::Result> { + let dir = tj_core::session::discovery::find_project_dir(cwd)?; + if enrich && dir.is_none() { + eprintln!( + "complete: no Claude Code session directory for this project — --enrich reads \ +Claude Code transcripts only (Codex sessions are not read yet)" + ); + } + + Ok(dir) +} + /// `complete ` — finalize a single task. fn run_complete_single( task_id: &str, @@ -5110,7 +5127,7 @@ fn run_complete_single( if !tj_core::db::task_exists(&conn, task_id)? { anyhow::bail!("task not found: {task_id}"); } - let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; + let project_dir = complete_project_dir(&cwd, enrich)?; let ctx = ProjectCtx { conn: &conn, events_path: &events_path, @@ -5156,7 +5173,7 @@ fn run_complete_batch( return Ok(()); } - let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; + let project_dir = complete_project_dir(&cwd, enrich)?; // Show the numbered list with event/session counts so the user can judge // what to keep before anything is touched. diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 29b9cab..8631895 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -6984,6 +6984,48 @@ fn complete_command_runs_and_skips_cleanly_without_sessions() { .stdout(contains("session(s) to enrich")); } +#[test] +fn complete_enrich_without_claude_code_sessions_says_codex_is_not_read() { + // A Codex-only project has no `~/.claude/projects/` dir: --enrich + // must say why it read nothing. `anthropic` without a key = no backend, + // so no model is ever called. + let dir = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Enrich me", "--goal", "g"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + for args in [ + vec!["complete", task_id.as_str(), "--enrich"], + vec!["complete", "--enrich", "--yes"], + ] { + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", dir.path()) + .env("CLAUDE_CONFIG_DIR", claude.path()) + .env_remove("ANTHROPIC_API_KEY") + .args(args) + .args(["--backend", "anthropic"]) + .assert() + .success() + .stderr(contains("Claude Code").and(contains("Codex"))); + } +} + #[test] fn complete_unknown_task_errors() { // A non-existent id is a hard error, not a silent no-op. From 1e952f7e746c9eea2bcc94e1419c89da1838b142 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:03:15 +0400 Subject: [PATCH 050/104] docs: describe the dream backend env vars the code actually reads --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b4d7ef5..e565096 100644 --- a/README.md +++ b/README.md @@ -225,10 +225,13 @@ chain `agent-sdk → api`, using whichever backends are available. Reorder the c with `TJ_HYBRID_LLM_ORDER` (e.g. `api,agent-sdk` to prefer the API key). With no LLM backend available, the heuristic still runs and ambiguous chunks queue in `pending/`. -The offline **`dream`** backfill (re-mining old transcripts for missed reasoning) uses -the same `agent-sdk` path: it prefers the local `claude` CLI pinned to Haiku and only -falls back to the `api` backend when no `claude` is on PATH — so dreaming over your -history needs no API key either. Override its model with `TJ_DREAM_MODEL`. +The offline **`dream`** backfill (re-mining old transcripts for missed reasoning) does +not use this chain. It runs on one LLM backend picked with `--backend` or `TJ_BACKEND`: +`claude-p` (the default — the local `claude` CLI pinned to Haiku, no API key), `codex` +(local `codex exec`, model via `TJ_CODEX_MODEL` as above), `anthropic` (needs +`ANTHROPIC_API_KEY`), `openai` (needs `OPENAI_API_KEY`) or `ollama` (free, local). +There is no fallback: when the chosen backend is unavailable (say, no `claude` on +PATH), `dream` says so and stops. Model overrides per backend are in the table below. | Env var | Effect | Default | |---------|--------|---------| @@ -236,7 +239,10 @@ history needs no API key either. Override its model with `TJ_DREAM_MODEL`. | `TJ_AGENT_SDK_MODEL` | Override the model the `agent-sdk` backend passes to `claude --model`. | `claude-haiku-4-5` | | `TJ_HYBRID_LLM_ORDER` | Comma-separated fallback order for `--backend=hybrid`. | `agent-sdk,api` | | `TJ_CLASSIFIER_MODEL` | Override the Anthropic model used by the `api` backend. | `claude-haiku-4-5-20251001` | -| `TJ_DREAM_MODEL` | Override the model used by `dream` backfill (both agent-sdk and api). | `claude-haiku-4-5` (agent-sdk) | +| `TJ_BACKEND` | LLM backend for `dream` (and `complete`, `consolidate`): `claude-p`, `codex`, `anthropic`, `openai` or `ollama`. `--backend` overrides it. | `claude-p` | +| `TJ_CONSOLIDATE_MODEL` | Model for `dream` on the `claude-p` and `anthropic` backends. | `claude-haiku-4-5` (claude-p), `claude-haiku-4-5-20251001` (anthropic) | +| `TJ_OPENAI_MODEL` / `TJ_OPENAI_BASE_URL` | Model and endpoint for `dream` on the `openai` backend (any OpenAI-compatible API). | `gpt-4o-mini` / `https://api.openai.com` | +| `TJ_OLLAMA_MODEL` / `TJ_OLLAMA_URL` | Model and endpoint for `dream` on the `ollama` backend. | `llama3.1` / `http://localhost:11434` | | `TJ_AUTO_OPEN_TASKS` | Set to `0` / `false` to disable auto-opening a task from `UserPromptSubmit` when no open task exists. | `1` | ## Event types From 060b9dddf1ebac336a1cf976fcc27b7d130e815c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:08:15 +0400 Subject: [PATCH 051/104] test(cli): run every CLI test without real claude/codex, API keys or live session env --- crates/tj-cli/tests/cli.rs | 63 +++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ec20e95..f07ac73 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1,7 +1,51 @@ -use assert_cmd::Command; use predicates::prelude::*; use predicates::str::contains; +/// Every `Command::cargo_bin` in this file goes through this wrapper, not +/// `assert_cmd`'s, so a test can't reach a real model or the developer's live +/// session by accident: PATH loses every directory holding a `claude` or +/// `codex` binary, API keys and backend overrides are cleared, and the Claude +/// Code session / mod markers of the shell running `cargo test` are dropped. +/// A test that needs a model installs its own fake and sets PATH itself. +struct Command; + +impl Command { + fn cargo_bin(name: &str) -> Result { + let mut cmd = assert_cmd::Command::cargo_bin(name)?; + cmd.env("PATH", path_without_llm_clis()); + for var in [ + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "TJ_BACKEND", + "TJ_HYBRID_LLM_ORDER", + "TASK_JOURNAL_DATA_DIR", + "CLAUDE_CODE_SESSION_ID", + "TJ_IN_CLASSIFIER", + "TJ_MOD_ACTIVE", + ] { + cmd.env_remove(var); + } + Ok(cmd) + } +} + +/// The test process's PATH minus any directory that holds `claude` or `codex`. +/// Computed once: on WSL the PATH carries dozens of slow `/mnt/c` directories. +fn path_without_llm_clis() -> &'static std::ffi::OsStr { + static PATH: std::sync::OnceLock = std::sync::OnceLock::new(); + PATH.get_or_init(|| { + let path = std::env::var_os("PATH").unwrap_or_default(); + let keep = std::env::split_paths(&path).filter(|dir| { + ["claude", "codex"].iter().all(|bin| { + ["", ".exe", ".cmd"] + .iter() + .all(|ext| !dir.join(format!("{bin}{ext}")).exists()) + }) + }); + std::env::join_paths(keep).unwrap() + }) +} + #[test] fn pack_command_prints_markdown_for_existing_task() { let dir = assert_fs::TempDir::new().unwrap(); @@ -290,6 +334,23 @@ fn doctor_reports_missing_codex_and_claude_as_information_only() { assert!(human.contains("codex binary"), "{human}"); } +/// The suite must never reach a real model: a test command sees no `claude` +/// or `codex` unless the test installs its own fake on PATH. +#[test] +fn test_commands_see_no_real_claude_or_codex() { + let xdg = assert_fs::TempDir::new().unwrap(); + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args(["doctor", "--json"]) + .output() + .unwrap(); + + let v: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["claude_in_path"], false, "{v}"); + assert_eq!(v["codex_in_path"], false, "{v}"); +} + #[cfg(unix)] #[test] fn doctor_detects_codex_on_path() { From 44b75c9d454dc57d00cb2f6bb5551c69df455d6e Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:09:29 +0400 Subject: [PATCH 052/104] fix(hooks): ingest-hook no longer deletes legacy pending entries it did not record --- crates/tj-cli/src/main.rs | 40 ++++++++++++++--------- crates/tj-cli/tests/cli.rs | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 3a58c2d..885b0ae 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2819,7 +2819,7 @@ runs in the background and won't block you; it only fills gaps and never closes ); } - // Drain any pending entries first (Task 10 fills the real-classifier branch). + // Mock path only: drain legacy pending entries first. drain_pending( &events_path, &project_hash, @@ -5954,6 +5954,10 @@ fn classify_chunk( Ok(ChunkOutcome::Recorded) } +/// Mock-only drain: with the mock flags, turn this project's legacy (v1) +/// pending entries into events. Without them it does nothing — a v1 entry is +/// a classifier failure waiting for `pending retry`, and v2 entries belong to +/// classify-worker. An entry is removed only after its event is written. fn drain_pending( events_path: &std::path::Path, project_hash: &str, @@ -5961,6 +5965,9 @@ fn drain_pending( mock_tid: Option<&str>, mock_conf: Option, ) -> anyhow::Result<()> { + let (Some(t), Some(tid)) = (mock_etype, mock_tid) else { + return Ok(()); + }; let pending_dir = events_path .parent() .unwrap() @@ -5981,22 +5988,23 @@ fn drain_pending( .and_then(|x| x.as_str()) .unwrap_or("") .to_string(); - if !text.is_empty() { - if let (Some(t), Some(tid)) = (mock_etype, mock_tid) { - let mut event = tj_core::event::Event::new( - tid, - parse_event_type(t)?, - tj_core::event::Author::Classifier, - tj_core::event::Source::Hook, - text, - ); - event.confidence = mock_conf; - event.status = tj_core::classifier::decide_status(mock_conf.unwrap_or(1.0)); - let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - } + if text.is_empty() { + continue; } + + let mut event = tj_core::event::Event::new( + tid, + parse_event_type(t)?, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text, + ); + event.confidence = mock_conf; + event.status = tj_core::classifier::decide_status(mock_conf.unwrap_or(1.0)); + let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + std::fs::remove_file(&path)?; } Ok(()) diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index f07ac73..48db5d4 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2029,6 +2029,72 @@ fn ingest_hook_drains_pending_queue_via_mock() { ); } +/// A legacy entry is a classifier failure waiting for `pending retry`; a +/// live hook must not throw it away. +#[test] +fn ingest_hook_keeps_legacy_pending_entries_for_pending_retry() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "01failed", "We decided to adopt PKCE flow.", 1); + + for kind in ["UserPromptSubmit", "PostToolUse"] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .current_dir(proj.path()) + .args(["ingest-hook", "--kind", kind, "--text", "next chunk"]) + .assert() + .success(); + } + + let entry = xdg + .path() + .join("task-journal") + .join("pending") + .join("01failed.json"); + assert!( + entry.exists(), + "the failed chunk must wait for pending retry" + ); +} + +/// The mock drain only removes an entry it turned into an event. +#[test] +fn ingest_hook_mock_drain_keeps_entries_it_cannot_record() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "01empty", "", 0); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args([ + "ingest-hook", + "--kind", + "Stop", + "--text", + "Live chunk", + "--mock-event-type", + "decision", + "--mock-task-id", + "tj-mock", + ]) + .assert() + .success(); + + let entry = xdg + .path() + .join("task-journal") + .join("pending") + .join("01empty.json"); + assert!( + entry.exists(), + "an entry that wrote no event must stay queued" + ); +} + #[test] fn stats_command_shows_classifier_counts() { let dir = assert_fs::TempDir::new().unwrap(); From d9bd5f89b8196cb5611ab72185064c4f00aaf6b6 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:10:20 +0400 Subject: [PATCH 053/104] fix(core): terminate a torn last JSONL line before appending and write each event in one call --- crates/tj-core/src/storage.rs | 55 +++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/tj-core/src/storage.rs b/crates/tj-core/src/storage.rs index d57a48f..d1ad73a 100644 --- a/crates/tj-core/src/storage.rs +++ b/crates/tj-core/src/storage.rs @@ -2,7 +2,7 @@ use crate::event::Event; use anyhow::Context; use fd_lock::RwLock as FdLock; use std::fs::{File, OpenOptions}; -use std::io::Write; +use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; /// Append-only writer for the events JSONL log. Holds an advisory @@ -45,12 +45,22 @@ impl JsonlWriter { } pub fn append(&mut self, event: &Event) -> anyhow::Result<()> { - let line = serde_json::to_string(event).context("serialize event")?; + let mut line = serde_json::to_string(event).context("serialize event")?; + line.push('\n'); let mut guard = self.lock.write().context("acquire exclusive file lock")?; + + // A crash mid-append leaves a torn last line without its newline. + // Terminate it first, so only that line is lost and not this event, + // which would otherwise glue onto it and be skipped as malformed too. + if ends_without_newline(&mut guard).context("check the log's last byte")? { + guard + .write_all(b"\n") + .context("terminate a torn last line")?; + } + guard .write_all(line.as_bytes()) .context("write event line")?; - guard.write_all(b"\n").context("write newline")?; Ok(()) } @@ -68,6 +78,21 @@ impl JsonlWriter { } } +/// True when the file is non-empty and its last byte is not `\n`. Writes +/// still go to the end afterwards: the file is opened in append mode. +fn ends_without_newline(file: &mut File) -> std::io::Result { + let len = file.metadata()?.len(); + if len == 0 { + return Ok(false); + } + + file.seek(SeekFrom::Start(len - 1))?; + let mut last = [0u8; 1]; + file.read_exact(&mut last)?; + + Ok(last[0] != b'\n') +} + #[cfg(test)] mod tests { use super::*; @@ -124,6 +149,30 @@ mod tests { assert_eq!(body.lines().count(), 2); } + #[test] + fn append_after_a_torn_last_line_loses_only_that_line() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("events.jsonl"); + + // A crash mid-append left a whole event and then half of the next one. + let whole = serde_json::to_string(&sample_event("whole")).unwrap(); + std::fs::write(&path, format!("{whole}\n{{\"event_id\":\"01TORN")).unwrap(); + + let mut w = JsonlWriter::open(&path).unwrap(); + w.append(&sample_event("after crash")).unwrap(); + w.flush_durable().unwrap(); + drop(w); + + let body = std::fs::read_to_string(&path).unwrap(); + let parsed: Vec = body + .lines() + .filter_map(|l| serde_json::from_str(l).ok()) + .collect(); + let texts: Vec<&str> = parsed.iter().map(|e| e.text.as_str()).collect(); + assert_eq!(texts, ["whole", "after crash"], "file:\n{body}"); + assert!(body.ends_with('\n')); + } + #[test] fn concurrent_appends_do_not_interleave_bytes() { // Eight threads, each owning its own JsonlWriter (own File handle From 89745e2d07a7b3f118827c5c56b508b9764b9ac7 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:10:40 +0400 Subject: [PATCH 054/104] refactor(hooks): synchronous ingest picks its classifier through build_classifier --- crates/tj-cli/src/main.rs | 268 ++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 154 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 885b0ae..27ad67d 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2925,169 +2925,129 @@ runs in the background and won't block you; it only fills gaps and never closes "assistant" }; - let (etype, task_id, confidence, evidence_strength, suggested_text) = if let ( - Some(t), - Some(tid), - ) = - (mock_event_type.as_deref(), mock_task_id.as_deref()) - { - ( - parse_event_type(t)?, - tid.to_string(), - mock_confidence.unwrap_or(1.0), - None, - None, - ) - } else { - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } - let mut recent = recent_task_contexts(&conn, 5)?; - if recent.is_empty() { - // No open tasks. v0.5.0 Phase A: auto-open a new - // task from the user's prompt so subsequent - // events have somewhere to land. Without this - // every fresh session was a black hole — events - // dropped silently because there was nothing to - // classify against. Opt-out via - // TJ_AUTO_OPEN_TASKS=0; only fires for - // UserPromptSubmit (assistant tool calls - // shouldn't conjure tasks). - let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") - .ok() - .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) - .unwrap_or(false); - if auto_open_disabled || !kind.contains("UserPrompt") { - return Ok(()); - } - let Some(new_task) = auto_open_task_from_prompt( - &events_path, - &project_hash, - &conn, - &text, - live_session_id.as_deref(), - )? - else { - // Prompt was only machine noise — nothing worth a task. - return Ok(()); - }; - recent.push(new_task); - } - - use tj_core::classifier::Classifier; - let classifier: Box = match backend.as_str() { - // v0.8.0: hybrid is the new default. Heuristic - // pattern-matching first (free), Anthropic API - // fallback when uncertain (requires ANTHROPIC_API_KEY). - // No background spawn of `claude -p` — that subprocess - // now bills tokens separately from Pro/Max. - "hybrid" | "" => { - Box::new(tj_core::classifier::hybrid::HybridClassifier::from_env()) + let (etype, task_id, confidence, evidence_strength, suggested_text) = + if let (Some(t), Some(tid)) = (mock_event_type.as_deref(), mock_task_id.as_deref()) + { + ( + parse_event_type(t)?, + tid.to_string(), + mock_confidence.unwrap_or(1.0), + None, + None, + ) + } else { + let state_path = + tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } - "api" => Box::new(tj_core::classifier::http::AnthropicClassifier::from_env()?), - "agent-sdk" => Box::new( - tj_core::classifier::agent_sdk::ClaudeCliClassifier::from_env() - .ok_or_else(|| { - anyhow::anyhow!( - "agent-sdk backend selected but no `claude` binary on PATH — \ - install Claude Code (https://claude.com/claude-code) or pick another --backend" - ) - })?, - ), - "heuristic" => { - // Heuristic-only: no LLM at all. Trades coverage - // for absolute zero-cost / offline operation. - use tj_core::classifier::heuristic::try_heuristic; - use tj_core::classifier::{ClassifyInput, ClassifyOutput}; - struct HeuristicOnly; - impl Classifier for HeuristicOnly { - fn classify( - &self, - input: &ClassifyInput, - ) -> anyhow::Result { - try_heuristic(input).ok_or_else(|| { - anyhow::anyhow!( - "heuristic uncertain (heuristic-only mode has no LLM fallback)" - ) - }) - } + let mut recent = recent_task_contexts(&conn, 5)?; + if recent.is_empty() { + // No open tasks. v0.5.0 Phase A: auto-open a new + // task from the user's prompt so subsequent + // events have somewhere to land. Without this + // every fresh session was a black hole — events + // dropped silently because there was nothing to + // classify against. Opt-out via + // TJ_AUTO_OPEN_TASKS=0; only fires for + // UserPromptSubmit (assistant tool calls + // shouldn't conjure tasks). + let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") + .ok() + .map(|v| v == "0" || v.eq_ignore_ascii_case("false")) + .unwrap_or(false); + if auto_open_disabled || !kind.contains("UserPrompt") { + return Ok(()); } - Box::new(HeuristicOnly) - } - other => anyhow::bail!( - "unknown backend: {other} (expected `hybrid`, `agent-sdk`, `api`, or `heuristic`)" - ), - }; - let input = tj_core::classifier::ClassifyInput { - text: text.clone(), - author_hint: author_hint.into(), - recent_tasks: recent, - tool_output: kind == "PostToolUse", - }; - let out = match classifier.classify(&input) { - Ok(o) => o, - Err(e) => { - persist_pending(&events_path, &project_hash, &kind, &text, &e.to_string())?; - return Ok(()); - } - }; - - let Some(tid) = out.task_id_guess else { - return Ok(()); - }; - - // Journal-integrity safeguards. The classifier sometimes - // mis-attributes events to old or closed tasks (no fault - // of the model — its prompt only sees recent_tasks). We - // reject three patterns that produce confusing journals: - // - // 1. Stop-hook → Close event. The Stop hook fires at - // every Claude Code session end. Session ending - // != task done. Closes happen via explicit - // `task-journal close ` only. - // 2. task_id_guess pointing at a non-existent task — - // route to pending so the user can decide later. - // 3. task_id_guess pointing at a CLOSED task — same - // treatment; closed tasks must stay closed. - use tj_core::event::EventType; - if matches!(out.event_type, EventType::Close) && kind == "Stop" { - return Ok(()); - } - match tj_core::db::task_status(&conn, &tid)? { - None => { - persist_pending( + let Some(new_task) = auto_open_task_from_prompt( &events_path, &project_hash, - &kind, + &conn, &text, - &format!("task_id_guess `{tid}` not found"), - )?; - return Ok(()); + live_session_id.as_deref(), + )? + else { + // Prompt was only machine noise — nothing worth a task. + return Ok(()); + }; + recent.push(new_task); } - Some(s) if s == "closed" => { - persist_pending( - &events_path, - &project_hash, - &kind, - &text, - &format!("task_id_guess `{tid}` is closed"), - )?; + + let classifier = build_classifier(&backend)?; + let input = tj_core::classifier::ClassifyInput { + text: text.clone(), + author_hint: author_hint.into(), + recent_tasks: recent, + tool_output: kind == "PostToolUse", + }; + let out = match classifier.classify(&input) { + Ok(o) => o, + Err(e) => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &e.to_string(), + )?; + return Ok(()); + } + }; + + let Some(tid) = out.task_id_guess else { + return Ok(()); + }; + + // Journal-integrity safeguards. The classifier sometimes + // mis-attributes events to old or closed tasks (no fault + // of the model — its prompt only sees recent_tasks). We + // reject three patterns that produce confusing journals: + // + // 1. Stop-hook → Close event. The Stop hook fires at + // every Claude Code session end. Session ending + // != task done. Closes happen via explicit + // `task-journal close ` only. + // 2. task_id_guess pointing at a non-existent task — + // route to pending so the user can decide later. + // 3. task_id_guess pointing at a CLOSED task — same + // treatment; closed tasks must stay closed. + use tj_core::event::EventType; + if matches!(out.event_type, EventType::Close) && kind == "Stop" { return Ok(()); } - _ => {} - } + match tj_core::db::task_status(&conn, &tid)? { + None => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &format!("task_id_guess `{tid}` not found"), + )?; + return Ok(()); + } + Some(s) if s == "closed" => { + persist_pending( + &events_path, + &project_hash, + &kind, + &text, + &format!("task_id_guess `{tid}` is closed"), + )?; + return Ok(()); + } + _ => {} + } - ( - out.event_type, - tid, - out.confidence, - out.evidence_strength, - Some(out.suggested_text), - ) - }; + ( + out.event_type, + tid, + out.confidence, + out.evidence_strength, + Some(out.suggested_text), + ) + }; // Use classifier's suggested_text if available (it's more concise and specific), // fall back to raw hook text for mock/manual events. From 2057f39dd4ce480574788d7c9ad856ed34effa91 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:11:09 +0400 Subject: [PATCH 055/104] feat(core): fall back to CODEX_THREAD_ID and CODEX_SESSION_ID for the live session id --- crates/tj-core/src/session_id.rs | 77 ++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/crates/tj-core/src/session_id.rs b/crates/tj-core/src/session_id.rs index e561345..37e6703 100644 --- a/crates/tj-core/src/session_id.rs +++ b/crates/tj-core/src/session_id.rs @@ -1,4 +1,4 @@ -//! Live Claude Code session id helpers. +//! Live agent session id helpers (Claude Code and Codex). //! //! task-journal already *parses* session ids out of Claude Code //! transcripts (`session::parser`) — that is a passive, read-only @@ -9,8 +9,9 @@ //! without time-window heuristics. //! //! Source order: hook payload field `session_id` → `CLAUDE_CODE_SESSION_ID` -//! env var → `None`. `None` means standalone behaviour is unchanged — -//! nothing is added to `meta`. +//! env var → `CODEX_THREAD_ID` → `CODEX_SESSION_ID` (Codex exports both, +//! with the same value, to the processes it starts) → `None`. `None` means +//! standalone behaviour is unchanged — nothing is added to `meta`. use serde_json::Value; @@ -24,11 +25,17 @@ pub fn session_id_from_payload(payload: &Value) -> Option { .map(str::to_string) } -/// Read `CLAUDE_CODE_SESSION_ID` from the environment. Empty counts as absent. +/// Read the live session id from the environment: `CLAUDE_CODE_SESSION_ID`, +/// then Codex's `CODEX_THREAD_ID`, then `CODEX_SESSION_ID`. Empty counts as +/// absent. pub fn session_id_from_env() -> Option { - std::env::var("CLAUDE_CODE_SESSION_ID") - .ok() - .filter(|s| !s.is_empty()) + [ + "CLAUDE_CODE_SESSION_ID", + "CODEX_THREAD_ID", + "CODEX_SESSION_ID", + ] + .into_iter() + .find_map(|var| std::env::var(var).ok().filter(|s| !s.is_empty())) } /// Resolve the live session id: hook payload first, env var as fallback. @@ -115,8 +122,62 @@ mod tests { #[test] fn live_none_when_no_source() { let _g = ENV_LOCK.lock().unwrap(); - std::env::remove_var("CLAUDE_CODE_SESSION_ID"); + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } assert_eq!(live_session_id(None), None); assert_eq!(live_session_id(Some(&json!({}))), None); } + + const SESSION_ENV_VARS: [&str; 3] = [ + "CLAUDE_CODE_SESSION_ID", + "CODEX_THREAD_ID", + "CODEX_SESSION_ID", + ]; + + /// Run `f` with exactly the given session env vars set (the rest unset). + fn with_session_env(set: &[(&str, &str)], f: impl FnOnce()) { + let _g = ENV_LOCK.lock().unwrap(); + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } + for (k, v) in set { + std::env::set_var(k, v); + } + + f(); + + for var in SESSION_ENV_VARS { + std::env::remove_var(var); + } + } + + #[test] + fn env_falls_back_to_codex_thread_then_session_id() { + with_session_env( + &[ + ("CODEX_THREAD_ID", "thread"), + ("CODEX_SESSION_ID", "session"), + ], + || assert_eq!(session_id_from_env().as_deref(), Some("thread")), + ); + with_session_env(&[("CODEX_SESSION_ID", "session")], || { + assert_eq!(session_id_from_env().as_deref(), Some("session")) + }); + with_session_env( + &[("CODEX_THREAD_ID", ""), ("CODEX_SESSION_ID", "session")], + || assert_eq!(session_id_from_env().as_deref(), Some("session")), + ); + } + + #[test] + fn env_prefers_claude_code_over_codex() { + with_session_env( + &[ + ("CLAUDE_CODE_SESSION_ID", "claude"), + ("CODEX_THREAD_ID", "thread"), + ], + || assert_eq!(session_id_from_env().as_deref(), Some("claude")), + ); + } } From 85f0b00111cbf1c65699ab3d1481522835ac1f15 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:13:51 +0400 Subject: [PATCH 056/104] fix(hooks): push-recall shows each hit once per session and skips the session's own entries --- crates/tj-cli/src/main.rs | 93 ++++++++++++++++++++++++++++++- crates/tj-cli/tests/cli.rs | 103 +++++++++++++++++++++++++++++++++++ crates/tj-core/src/recall.rs | 2 + 3 files changed, 196 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 27ad67d..f27fef9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2393,6 +2393,12 @@ fn real_main() -> Result<()> { &text, tj_core::recall::DEFAULT_MAX_HITS, ) { + let hits = fresh_recall_hits( + hits, + live_session_id.as_deref(), + &events_path, + &project_hash, + ); if !hits.is_empty() { let mut ctx = String::new(); for h in &hits { @@ -2425,8 +2431,12 @@ fn real_main() -> Result<()> { // mcp__ tools) — gated MCP-only, falls through to the queue path so // event capture is unaffected. Disabled by TJ_PUSH_RECALL=0. if kind == "PostToolUse" && std::env::var("TJ_PUSH_RECALL").as_deref() != Ok("0") { - if let Some(envelope) = push_recall_envelope(&payload, &events_path, &project_hash) - { + if let Some(envelope) = push_recall_envelope( + &payload, + &events_path, + &project_hash, + live_session_id.as_deref(), + ) { println!("{}", serde_json::to_string(&envelope)?); } } @@ -5998,6 +6008,7 @@ fn push_recall_envelope( payload: &serde_json::Value, events_path: &std::path::Path, project_hash: &str, + session_id: Option<&str>, ) -> Option { // MCP-only gate: Claude Code prefixes MCP tools `mcp____`. let tool_name = payload.get("tool_name").and_then(|v| v.as_str())?; @@ -6028,6 +6039,7 @@ fn push_recall_envelope( let hits = tj_core::recall::relevant_recall(&conn, &query_text, tj_core::recall::DEFAULT_MAX_HITS) .ok()?; + let hits = fresh_recall_hits(hits, session_id, events_path, project_hash); if hits.is_empty() { return None; } @@ -6040,6 +6052,83 @@ fn push_recall_envelope( })) } +/// Most ` ` lines the shown-recall log keeps. +const RECALL_SHOWN_CAP: usize = 2000; + +/// The recall hits still worth pushing to `session_id`: each one at most once +/// per session (remembered in `/.recall-shown`, newest +/// [`RECALL_SHOWN_CAP`] lines), and never an event the session wrote on its +/// current task — the agent just wrote it. Without a session id every hit +/// passes, as before. Best-effort: an unreadable log never hides a hit. +fn fresh_recall_hits( + hits: Vec, + session_id: Option<&str>, + events_path: &std::path::Path, + project_hash: &str, +) -> Vec { + let Some(sid) = session_id else { + return hits; + }; + if hits.is_empty() { + return hits; + } + let Ok(log) = + tj_core::paths::state_dir().map(|d| d.join(format!("{project_hash}.recall-shown"))) + else { + return hits; + }; + + let own = session_task_event_ids(events_path, sid); + let body = std::fs::read_to_string(&log).unwrap_or_default(); + let shown: std::collections::HashSet<&str> = body.lines().collect(); + let fresh: Vec<_> = hits + .into_iter() + .filter(|h| !own.contains(&h.event_id)) + .filter(|h| !shown.contains(format!("{sid} {}", h.event_id).as_str())) + .collect(); + if fresh.is_empty() { + return fresh; + } + + // ponytail: read-modify-write without a lock; two parallel hooks can + // re-show a hit once. Add a file lock if that ever shows up in practice. + let mut lines: Vec = body.lines().map(str::to_string).collect(); + lines.extend(fresh.iter().map(|h| format!("{sid} {}", h.event_id))); + let keep = &lines[lines.len().saturating_sub(RECALL_SHOWN_CAP)..]; + let _ = std::fs::write(&log, keep.join("\n") + "\n"); + + fresh +} + +/// Ids of the events session `sid` wrote on its current task — the task of +/// its latest event — read from the journal by `meta.session_id`. +fn session_task_event_ids( + events_path: &std::path::Path, + sid: &str, +) -> std::collections::HashSet { + let body = std::fs::read_to_string(events_path).unwrap_or_default(); + let mine: Vec<(String, String)> = body + .lines() + .filter(|l| l.contains(sid)) + .filter_map(|l| serde_json::from_str::(l).ok()) + .filter(|e| e["meta"]["session_id"].as_str() == Some(sid)) + .filter_map(|e| { + Some(( + e["event_id"].as_str()?.into(), + e["task_id"].as_str()?.into(), + )) + }) + .collect(); + + let Some((_, current)) = mine.last() else { + return Default::default(); + }; + mine.iter() + .filter(|(_, task)| task == current) + .map(|(id, _)| id.clone()) + .collect() +} + /// One ⚠ line per recall hit (mirrors the close-gate / SessionStart convention). fn render_recall_banner(hits: &[tj_core::recall::RecallHit]) -> String { let mut s = String::from("\u{26a0} Task Journal recall — you may be repeating a prior path:"); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 48db5d4..a8a1680 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5543,6 +5543,109 @@ fn post_tool_use_emits_recall_additional_context() { ); } +/// One real (non-mock) PostToolUse hook run; `session: None` sends no +/// session id at all. Returns stdout. +fn post_tool_use(dir: &std::path::Path, session: Option<&str>, tool: &str, input: &str) -> String { + let mut payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": tool, + "tool_input": { "command": input }, + "tool_response": { "output": "" } + }); + if let Some(sid) = session { + payload["session_id"] = sid.into(); + } + + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap() +} + +/// After every tool call the same hits used to come back, flooding the +/// agent's context. A session sees each hit once, on either recall path. +#[test] +fn push_recall_shows_each_hit_once_per_session() { + let (dir, _task_id) = seed_axum_rejection(); + let axum = "let's switch the server to axum"; + + let first = post_tool_use(dir.path(), Some("s-a"), "Bash", axum); + assert!(first.contains("⚠ recall"), "{first}"); + + let again = post_tool_use(dir.path(), Some("s-a"), "Bash", axum); + assert!(!again.contains("recall"), "{again}"); + let via_mcp = post_tool_use(dir.path(), Some("s-a"), "mcp__x__do", axum); + assert!(!via_mcp.contains("recall"), "{via_mcp}"); + + let other = post_tool_use(dir.path(), Some("s-b"), "Bash", axum); + assert!(other.contains("⚠ recall"), "{other}"); +} + +#[test] +fn push_recall_without_a_session_id_repeats_as_before() { + let (dir, _task_id) = seed_axum_rejection(); + let axum = "let's switch the server to axum"; + + for _ in 0..2 { + let out = post_tool_use(dir.path(), None, "Bash", axum); + assert!(out.contains("⚠ recall"), "{out}"); + } +} + +/// The agent just wrote that rejection on its own task in this session — +/// echoing it back is noise. Another session still gets it. +#[test] +fn push_recall_skips_what_this_session_wrote_on_its_task() { + let dir = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["create", "Own session host"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + let payload = serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": "s-own", + "prompt": "Tried switching the server to axum but it broke rmcp stdio." + }); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args([ + "ingest-hook", + "--mock-event-type", + "rejection", + "--mock-task-id", + &task_id, + ]) + .write_stdin(payload.to_string()) + .assert() + .success(); + let axum = "let's switch the server to axum"; + + let own = post_tool_use(dir.path(), Some("s-own"), "Bash", axum); + assert!(!own.contains("recall"), "{own}"); + + let other = post_tool_use(dir.path(), Some("s-other"), "Bash", axum); + assert!(other.contains("⚠ recall"), "{other}"); +} + #[test] fn post_tool_use_no_recall_when_no_match() { let (dir, task_id) = seed_axum_rejection(); diff --git a/crates/tj-core/src/recall.rs b/crates/tj-core/src/recall.rs index 68179d2..eea0347 100644 --- a/crates/tj-core/src/recall.rs +++ b/crates/tj-core/src/recall.rs @@ -8,6 +8,7 @@ use crate::event::EventType; /// One recalled high-signal event that matched the current context. #[derive(Debug, Clone, PartialEq)] pub struct RecallHit { + pub event_id: String, pub task_id: String, pub event_type: EventType, // Rejection | Decision pub text: String, @@ -164,6 +165,7 @@ pub fn relevant_recall( .filter_map(|(eid, score)| { meta.remove(&eid) .map(|(task_id, event_type, text)| RecallHit { + event_id: eid, task_id, event_type, text, From 6b560f729c8389d5a12915d00b00e78fd4300ced Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:14:06 +0400 Subject: [PATCH 057/104] fix(core): run each schema migration in an immediate transaction and retry the WAL switch on a busy fresh DB --- crates/tj-core/src/db.rs | 105 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index e116ce0..5573195 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -211,9 +211,27 @@ fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { if applied.contains(&migration.version) { continue; } - conn.execute_batch(migration.sql) + + // One IMMEDIATE transaction per migration: it takes the write lock up + // front, so a second process opening the same fresh DB waits here and + // then sees the version as applied instead of re-running its ALTERs. + // The migration and its row commit together — a failure halfway rolls + // the whole migration back instead of leaving a partial schema. + let tx = + rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate) + .with_context(|| format!("begin schema migration v{:03}", migration.version))?; + let already_applied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = ?1)", + rusqlite::params![migration.version], + |r| r.get(0), + )?; + if already_applied { + continue; + } + + tx.execute_batch(migration.sql) .with_context(|| format!("apply schema migration v{:03}", migration.version))?; - conn.execute( + tx.execute( "INSERT INTO schema_migrations(version, applied_at) VALUES (?1, ?2)", rusqlite::params![ migration.version, @@ -226,6 +244,8 @@ fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { migration.version ) })?; + tx.commit() + .with_context(|| format!("commit schema migration v{:03}", migration.version))?; } Ok(()) } @@ -962,13 +982,33 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { Ok(()) } +/// Switch the DB to WAL. On a fresh file that needs an exclusive lock, and +/// SQLite answers SQLITE_BUSY at once — no busy handler, to avoid a deadlock — +/// when another connection is switching the same file: two processes opening +/// a fresh DB together. Retry briefly; once the file is in WAL it is a no-op. +fn enable_wal(conn: &Connection) -> anyhow::Result<()> { + let mut attempts = 0; + loop { + match conn.execute_batch("PRAGMA journal_mode=WAL;") { + Err(rusqlite::Error::SqliteFailure(e, _)) + if e.code == rusqlite::ErrorCode::DatabaseBusy && attempts < 100 => + { + attempts += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + } + res => return res.context("set WAL journal mode"), + } + } +} + pub fn open(path: impl AsRef) -> anyhow::Result { if let Some(parent) = path.as_ref().parent() { std::fs::create_dir_all(parent).with_context(|| format!("create dir {parent:?}"))?; } let conn = Connection::open(&path).with_context(|| format!("open SQLite at {:?}", path.as_ref()))?; - conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?; + enable_wal(&conn)?; + conn.execute_batch("PRAGMA foreign_keys=ON;")?; apply_migrations(&conn).context("apply schema migrations")?; Ok(conn) } @@ -1489,6 +1529,65 @@ mod tests { ); } + #[test] + fn a_migration_failing_halfway_leaves_no_partial_schema() { + let d = TempDir::new().unwrap(); + let conn = Connection::open(d.path().join("state.sqlite")).unwrap(); + conn.execute_batch( + "CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL); + INSERT INTO schema_migrations VALUES (1, 'x'), (2, 'x');", + ) + .unwrap(); + conn.execute_batch(MIGRATION_001).unwrap(); + conn.execute_batch(MIGRATION_002).unwrap(); + // v003 adds goal, then outcome: make its second ALTER fail. + conn.execute_batch("ALTER TABLE tasks ADD COLUMN outcome TEXT;") + .unwrap(); + + assert!(apply_migrations(&conn).is_err()); + + let goal_cols: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('tasks') WHERE name = 'goal'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(goal_cols, 0, "v003's first ALTER must roll back"); + let v3: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 3", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(v3, 0); + } + + #[test] + fn concurrent_opens_of_a_fresh_db_both_succeed() { + let d = TempDir::new().unwrap(); + + for round in 0..20 { + let path = d.path().join(format!("state-{round}.sqlite")); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let (path, barrier) = (path.clone(), barrier.clone()); + std::thread::spawn(move || { + barrier.wait(); + open(&path).map(|_| ()) + }) + }) + .collect(); + + for h in handles { + let res = h.join().unwrap(); + assert!(res.is_ok(), "round {round}: {:#}", res.unwrap_err()); + } + } + } + fn make_text_event(text: &str) -> crate::event::Event { crate::event::Event::new( "tj-x", From 0969add5a2b569e1509689d5ed61076f1d1d8408 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:16:13 +0400 Subject: [PATCH 058/104] feat(hooks): classic capture hooks and nudge stand down when TJ_MOD_ACTIVE is set --- crates/tj-cli/src/main.rs | 22 ++++++++ crates/tj-cli/tests/cli.rs | 109 +++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index f27fef9..24e422c 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2338,6 +2338,18 @@ fn real_main() -> Result<()> { _ => parse_hook_stdin()?, }; + // The Claude Code mod captures in-process and marks the hooks it + // starts; the classic capture stands down instead of doing the + // same work twice. Resume packs and model switches stay ours. + if mod_active() + && matches!( + kind.as_str(), + "UserPromptSubmit" | "PostToolUse" | "Stop" | "PreCompact" | "SessionEnd" + ) + { + return Ok(()); + } + // Emergency capture kill-switch: a `.capture-disabled` marker in the // data dir no-ops realtime capture (the read-only SessionStart // resume still runs). Because the hook re-invokes this binary on @@ -4306,6 +4318,12 @@ fn count_session_events_tail(path: &std::path::Path, sid: &str, tail_lines: usiz .count() } +/// True when the Claude Code mod (`plugin/hooks/register.ts`) runs in this +/// session: it sets `TJ_MOD_ACTIVE` for every hook it starts. Codex never does. +fn mod_active() -> bool { + std::env::var("TJ_MOD_ACTIVE").is_ok_and(|v| !v.is_empty() && v != "0") +} + /// Adaptive UserPromptSubmit nudge (caveman pattern, non-blocking, free): always /// emit the base "record as you go" reminder, and — when the session has done /// substantial work but logged little — escalate. All signals are cheap (a file @@ -4316,6 +4334,10 @@ fn run_nudge() -> anyhow::Result<()> { if std::env::var(tj_core::classifier::agent_sdk::IN_CLASSIFIER_ENV).is_ok() { return Ok(()); } + // The Claude Code mod nudges by itself (after N turns without an entry). + if mod_active() { + return Ok(()); + } let mut ctx = NUDGE_BASE.to_string(); let escalation = (|| -> Option { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a8a1680..940bf00 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2095,6 +2095,115 @@ fn ingest_hook_mock_drain_keeps_entries_it_cannot_record() { ); } +/// Run one `ingest-hook` payload of `kind` with `TJ_MOD_ACTIVE` set to +/// `mod_value` (None = unset), in a fresh project with an open task and a +/// transcript. True when the hook left any trace: stdout, a pending entry, +/// or a new journal line. +fn hook_leaves_a_trace(kind: &str, mod_value: Option<&str>) -> bool { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Mod host"]) + .assert() + .success(); + let transcript = proj.path().join("session.jsonl"); + std::fs::write( + &transcript, + r#"{"type":"user","uuid":"u1","timestamp":"2099-01-01T00:00:00.000Z","sessionId":"s1","message":{"content":"the refund flow needs idempotency keys"}}"#, + ) + .unwrap(); + let payload = serde_json::json!({ + "hook_event_name": kind, + "session_id": "s1", + "transcript_path": transcript.to_str().unwrap(), + "reason": "clear", + "prompt": "We decided to adopt the PKCE flow.", + "tool_name": "Bash", + "tool_input": { "command": "cargo test" }, + "tool_response": { "output": "ok" }, + "from_model": "claude-opus", + "to_model": "claude-sonnet", + }); + let journal = xdg.path().join("task-journal").join("events"); + let journal_len = || -> u64 { + std::fs::read_dir(&journal) + .unwrap() + .map(|e| e.unwrap().metadata().unwrap().len()) + .sum() + }; + let before = journal_len(); + + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .current_dir(proj.path()); + if let Some(v) = mod_value { + cmd.env("TJ_MOD_ACTIVE", v); + } + let out = cmd + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + + let pending = std::fs::read_dir(xdg.path().join("task-journal").join("pending")) + .map(|d| d.count()) + .unwrap_or(0); + !out.is_empty() || pending > 0 || journal_len() != before +} + +/// The Claude Code mod captures in-process and sets TJ_MOD_ACTIVE for the +/// hooks it starts; the classic capture must then stand down. +#[test] +fn mod_active_silences_the_classic_capture_hooks() { + for kind in [ + "UserPromptSubmit", + "PostToolUse", + "Stop", + "PreCompact", + "SessionEnd", + ] { + assert!(!hook_leaves_a_trace(kind, Some("1")), "{kind}, mod on"); + assert!(hook_leaves_a_trace(kind, Some("0")), "{kind}, mod \"0\""); + assert!(hook_leaves_a_trace(kind, None), "{kind}, mod unset"); + } +} + +/// Resume packs and the model-switch constraint are not the mod's job. +#[test] +fn mod_active_keeps_session_start_and_model_switch() { + for kind in ["SessionStart", "PostModelSwitch"] { + assert!(hook_leaves_a_trace(kind, Some("1")), "{kind}, mod on"); + } +} + +#[test] +fn mod_active_silences_the_nudge() { + for (mod_value, silent) in [(Some("1"), true), (Some("0"), false), (None, false)] { + let xdg = assert_fs::TempDir::new().unwrap(); + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", xdg.path()); + if let Some(v) = mod_value { + cmd.env("TJ_MOD_ACTIVE", v); + } + let out = cmd + .arg("nudge") + .write_stdin("{}") + .assert() + .success() + .get_output() + .stdout + .clone(); + assert_eq!(out.is_empty(), silent, "TJ_MOD_ACTIVE={mod_value:?}"); + } +} + #[test] fn stats_command_shows_classifier_counts() { let dir = assert_fs::TempDir::new().unwrap(); From f9d4ea793ac429b266d3f1a27f4c8dc1c0da6f54 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:16:22 +0400 Subject: [PATCH 059/104] feat(core): index meta.session_id on events and add active_task_for_session --- crates/tj-core/src/db.rs | 162 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 3 deletions(-) diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 5573195..f4b4d6d 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -150,6 +150,16 @@ CREATE INDEX IF NOT EXISTS idx_emb_project_tier ON embeddings(project_hash, tier ALTER TABLE events_index ADD COLUMN memory_tier TEXT NOT NULL DEFAULT 'episodic'; "#; +/// v0.30.0 per-session queries — `session_id` projects an event's +/// `meta.session_id`. Clearing `index_state` makes the next +/// `ingest_new_events` replay the whole log (every projection is an +/// idempotent upsert), which fills the column for events indexed before. +const MIGRATION_009: &str = r#" +ALTER TABLE events_index ADD COLUMN session_id TEXT; +CREATE INDEX IF NOT EXISTS idx_events_session_time ON events_index(session_id, timestamp); +DELETE FROM index_state; +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -185,6 +195,10 @@ const MIGRATIONS: &[Migration] = &[ version: 8, sql: MIGRATION_008, }, + Migration { + version: 9, + sql: MIGRATION_009, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -499,6 +513,27 @@ pub fn task_id_by_external(conn: &Connection, reference: &str) -> anyhow::Result Ok(id) } +/// The open task of `project_hash` whose most recent event carries +/// `meta.session_id == session_id` — the task a live agent session is +/// working on. `None` when the session has no event on an open task. +pub fn active_task_for_session( + conn: &Connection, + project_hash: &str, + session_id: &str, +) -> anyhow::Result> { + let id: Option = conn + .query_row( + "SELECT ei.task_id FROM events_index ei + JOIN tasks t ON t.task_id = ei.task_id + WHERE ei.session_id = ?2 AND t.project_hash = ?1 AND t.status = 'open' + ORDER BY ei.timestamp DESC LIMIT 1", + rusqlite::params![project_hash, session_id], + |r| r.get::<_, String>(0), + ) + .optional()?; + Ok(id) +} + /// Read-only metadata bundle used by pack rendering (and TUI list /// teasers in v0.4.0+). Returns `None` for unknown tasks. #[derive(Debug, Clone, Default)] @@ -915,12 +950,13 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { } else { Some(serde_json::to_string(&artifacts)?) }; + let session_id = event.meta.get("session_id").and_then(|v| v.as_str()); conn.execute( - "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", rusqlite::params![ event.event_id, event.task_id, type_str, - event.timestamp, event.confidence, status_str, artifacts_json + event.timestamp, event.confidence, status_str, artifacts_json, session_id ], )?; // search_fts has no PK; clear then insert to keep idempotent across rebuild_state replays. @@ -2314,6 +2350,126 @@ mod tests { assert_eq!(task_id_by_external(&conn, "loom:t-xy").unwrap(), None); } + /// An event of `task_id` stamped with `session` at a fixed `timestamp`. + fn session_event(task_id: &str, session: &str, timestamp: &str) -> crate::event::Event { + let mut e = make_text_event("work"); + e.task_id = task_id.into(); + e.timestamp = timestamp.into(); + e.meta = serde_json::json!({"session_id": session}); + e + } + + fn indexed_session_id(conn: &Connection, event_id: &str) -> Option { + conn.query_row( + "SELECT session_id FROM events_index WHERE event_id = ?1", + rusqlite::params![event_id], + |r| r.get(0), + ) + .unwrap() + } + + #[test] + fn index_event_stores_the_session_id_from_meta() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + + let stamped = session_event("tj-s", "sess-1", "2026-01-01T00:00:00.000Z"); + let plain = make_text_event("no session"); + index_event(&conn, &stamped).unwrap(); + index_event(&conn, &plain).unwrap(); + + assert_eq!( + indexed_session_id(&conn, &stamped.event_id).as_deref(), + Some("sess-1") + ); + assert_eq!(indexed_session_id(&conn, &plain.event_id), None); + } + + #[test] + fn active_task_for_session_picks_the_open_task_with_its_latest_event() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let ph = "feedfacefeedface"; + + for id in ["tj-a", "tj-b", "tj-c"] { + upsert_task_from_event(&conn, &make_open_event(id, id), ph).unwrap(); + } + upsert_task_from_event(&conn, &make_open_event("tj-other", "o"), "otherproject0000") + .unwrap(); + let events = [ + session_event("tj-a", "s1", "2026-01-01T00:00:01.000Z"), + session_event("tj-b", "s1", "2026-01-01T00:00:02.000Z"), + session_event("tj-a", "s2", "2026-01-01T00:00:03.000Z"), + session_event("tj-c", "s1", "2026-01-01T00:00:04.000Z"), + session_event("tj-other", "s1", "2026-01-01T00:00:05.000Z"), + ]; + for e in &events { + upsert_task_from_event(&conn, e, ph).unwrap(); + index_event(&conn, e).unwrap(); + } + let mut close = make_text_event("done"); + close.task_id = "tj-c".into(); + close.event_type = crate::event::EventType::Close; + upsert_task_from_event(&conn, &close, ph).unwrap(); + + // tj-c has s1's latest event but is closed; tj-other is another project. + assert_eq!( + active_task_for_session(&conn, ph, "s1").unwrap().as_deref(), + Some("tj-b") + ); + assert_eq!( + active_task_for_session(&conn, ph, "s2").unwrap().as_deref(), + Some("tj-a") + ); + assert_eq!(active_task_for_session(&conn, ph, "s3").unwrap(), None); + } + + #[test] + fn upgrading_an_existing_db_backfills_session_ids_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-up", "Upgrade"); + let stamped = session_event("tj-up", "sess-old", "2026-01-01T00:00:01.000Z"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + write_event_line(&mut f, &stamped); + drop(f); + + // A pre-session_id database that has already indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + // Written straight to SQLite by older versions: the replay must keep them. + set_task_goal(&conn, "tj-up", "legacy goal").unwrap(); + add_task_external(&conn, "tj-up", "loom:t-legacy").unwrap(); + conn.execute_batch( + "DROP INDEX IF EXISTS idx_events_session_time; + ALTER TABLE events_index DROP COLUMN session_id; + DELETE FROM schema_migrations WHERE version = 9;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + assert_eq!( + indexed_session_id(&conn, &stamped.event_id).as_deref(), + Some("sess-old") + ); + assert_eq!( + active_task_for_session(&conn, ph, "sess-old") + .unwrap() + .as_deref(), + Some("tj-up") + ); + let meta = task_metadata(&conn, "tj-up").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("legacy goal")); + assert_eq!(meta.external.as_deref(), Some("loom:t-legacy")); + } + #[test] fn open_event_meta_parent_id_is_persisted() { let d = tempfile::TempDir::new().unwrap(); From 48341c1d90bc9e3e59251949f3e3e1d6ddec2871 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:23:19 +0400 Subject: [PATCH 060/104] fix(core): record task goal and external refs in the journal so a state rebuild restores them --- crates/tj-cli/src/main.rs | 101 +++++++++++++++---------- crates/tj-cli/tests/cli.rs | 61 +++++++++++++++ crates/tj-core/src/db.rs | 147 +++++++++++++++++++++++++++++++++++- crates/tj-core/src/event.rs | 6 ++ crates/tj-mcp/src/main.rs | 90 ++++++++++++++++------ 5 files changed, 338 insertions(+), 67 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 3a58c2d..955a3bb 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1339,24 +1339,17 @@ fn real_main() -> Result<()> { if let Some(ref parent_id) = parent { meta["parent_id"] = serde_json::Value::String(parent_id.clone()); } + // The goal rides in the open event, so ingest (and a rebuild + // from the JSONL) restores it. + if let Some(g) = goal { + meta["goal"] = serde_json::Value::String(g); + } event.meta = meta; let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; - // If --goal was provided, ingest the open event into SQLite - // (so the row exists) and write the goal column. Skipping - // this when --goal is absent keeps the SQLite hot path - // exclusive to ingest-hook / pack callers. - if let Some(g) = goal { - let state_path = - tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::set_task_goal(&conn, &task_id, &g)?; - } - println!("{}", task_id); } Commands::List { tree } => { @@ -1888,7 +1881,22 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - tj_core::db::set_task_goal(&conn, &task_id, &text)?; + + // An amend event carries the change, so a rebuild from the + // JSONL restores it; ingest applies it to the task row. + let mut event = tj_core::event::Event::new( + &task_id, + tj_core::event::EventType::Amend, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + format!("goal: {text}"), + ); + event.meta = serde_json::json!({ "goal": text }); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; println!("ok"); } Commands::External { task_id, add } => { @@ -1904,7 +1912,20 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - tj_core::db::add_task_external(&conn, &task_id, &add)?; + + let mut event = tj_core::event::Event::new( + &task_id, + tj_core::event::EventType::Amend, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + format!("external: +{add}"), + ); + event.meta = serde_json::json!({ "external_add": [add] }); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; println!("ok"); } Commands::Reclassify { task_id } => { @@ -5399,42 +5420,20 @@ fn auto_open_task_from_prompt( }; let goal: String = tj_core::title::humanize_goal(prompt, 200).unwrap_or_else(|| title.clone()); - let task_id = tj_core::new_task_id(); - let mut event = tj_core::event::Event::new( - task_id.clone(), - tj_core::event::EventType::Open, - tj_core::event::Author::User, - tj_core::event::Source::Cli, - title.clone(), - ); - event.meta = serde_json::json!({ "title": title, "auto_opened": true }); - tj_core::session_id::stamp_session_id(&mut event.meta, session_id); - - let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - - tj_core::db::ingest_new_events(conn, events_path, project_hash)?; - if !goal.is_empty() { - tj_core::db::set_task_goal(conn, &task_id, &goal)?; - } - // v0.5.0 Phase C / v0.6.0: score-based linking. Pull artifacts // from the prompt — ticket ids, commit hashes, file paths — then // ask the journal which prior tasks share enough signal to be a // probable continuation. Anything with score > 0 gets linked via // External; the strongest closed match also triggers a stderr // hint so the user can reopen instead of accumulating duplicates. + // Resolved before the open event is written so the links ride in it. + let mut linked: Vec = Vec::new(); let prompt_arts = tj_core::artifacts::extract(prompt); if !prompt_arts.is_empty() { let related = tj_core::db::find_related_tasks(conn, &prompt_arts)?; let mut warned = false; for r in related.iter().take(5) { - if r.task_id == task_id { - continue; - } - let _ = - tj_core::db::add_task_external(conn, &task_id, &format!("linked:{}", r.task_id)); + linked.push(format!("linked:{}", r.task_id)); if !warned && r.status == "closed" { eprintln!( "task-journal: this prompt looks like a continuation of closed task {} \ @@ -5446,6 +5445,30 @@ fn auto_open_task_from_prompt( } } + let task_id = tj_core::new_task_id(); + let mut event = tj_core::event::Event::new( + task_id.clone(), + tj_core::event::EventType::Open, + tj_core::event::Author::User, + tj_core::event::Source::Cli, + title.clone(), + ); + // Goal and links ride in the open event, so a rebuild restores them. + event.meta = serde_json::json!({ "title": title, "auto_opened": true }); + if !goal.is_empty() { + event.meta["goal"] = serde_json::Value::String(goal); + } + if !linked.is_empty() { + event.meta["external"] = serde_json::json!(linked); + } + tj_core::session_id::stamp_session_id(&mut event.meta, session_id); + + let mut writer = tj_core::storage::JsonlWriter::open(events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + tj_core::db::ingest_new_events(conn, events_path, project_hash)?; + Ok(Some(tj_core::classifier::TaskContext { task_id, title, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ec20e95..02670c6 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2384,6 +2384,56 @@ fn external_add_appends_references() { .stdout(contains("**External**: beads:claude-memory-rsw,github:#42")); } +#[test] +fn goal_and_external_survive_deleting_the_state_db() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let wipe_state = || { + let _ = std::fs::remove_dir_all(dir.path().join("task-journal").join("state")); + }; + let out = tj() + .args(["create", "Rebuildable", "--goal", "first goal"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + + wipe_state(); + tj().args(["pack", &task_id]) + .assert() + .success() + .stdout(contains("**Goal**: first goal")); + + tj().args(["goal", &task_id, "second goal"]) + .assert() + .success(); + tj().args(["external", &task_id, "--add", "beads:rb-1"]) + .assert() + .success(); + wipe_state(); + + tj().args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("**Goal**: second goal")) + .stdout(contains("**External**: beads:rb-1")) + .stdout(contains("amend").not()); + tj().args(["pack", "--external", "beads:rb-1"]) + .assert() + .success() + .stdout(contains("Rebuildable")); + tj().args(["events", "list"]) + .assert() + .success() + .stdout(contains("goal: second goal")); +} + #[test] fn ingest_hook_short_circuits_when_in_classifier_env_set() { // Recursion guard: classifier sets TJ_IN_CLASSIFIER=1 before @@ -3082,6 +3132,17 @@ fn auto_open_links_to_prior_task_referencing_same_issue() { // current status annotated next to the id. .stdout(contains("**Linked**:")) .stdout(contains(format!("- {} [closed]", prior))); + + // Goal and link ride in the journal: a state rebuilt from it keeps both. + std::fs::remove_dir_all(dir.path().join("task-journal").join("state")).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["pack", &new_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("**Goal**: (not set)").not()) + .stdout(contains(format!("- {} [closed]", prior))); } #[test] diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index f4b4d6d..6f4a81e 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -292,6 +292,27 @@ pub fn upsert_task_from_event( ON CONFLICT(task_id) DO UPDATE SET last_event_at = ?4", rusqlite::params![event.task_id, title, project_hash, event.timestamp, parent_id], )?; + + // Goal and external refs given at creation ride in the open + // event's meta so a rebuild from the JSONL restores them. A replay + // over an existing row keeps a goal changed since. + if let Some(goal) = event.meta.get("goal").and_then(|v| v.as_str()) { + conn.execute( + "UPDATE tasks SET goal = COALESCE(goal, ?2) WHERE task_id = ?1", + rusqlite::params![event.task_id, goal], + )?; + } + for reference in meta_strings(&event.meta, "external") { + add_task_external(conn, &event.task_id, reference)?; + } + } + EventType::Amend => { + if let Some(goal) = event.meta.get("goal").and_then(|v| v.as_str()) { + set_task_goal(conn, &event.task_id, goal)?; + } + for reference in meta_strings(&event.meta, "external_add") { + add_task_external(conn, &event.task_id, reference)?; + } } EventType::Close => { conn.execute( @@ -338,6 +359,15 @@ pub fn upsert_task_from_event( Ok(()) } +/// The string items of the JSON array at `meta[key]`; empty when absent. +fn meta_strings<'a>(meta: &'a serde_json::Value, key: &str) -> impl Iterator { + meta.get(key) + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + .filter_map(|v| v.as_str()) +} + use std::io::BufRead; pub fn list_all_projects(state_dir: impl AsRef) -> anyhow::Result> { @@ -471,17 +501,30 @@ pub fn set_task_outcome( } /// Append an external reference to `tasks.external`. The column is -/// stored as a comma-separated list — small, append-mostly, no -/// uniqueness constraint. Acceptable shapes (loose, not enforced): -/// `beads:claude-memory-rsw`, `github:#42`, `jira:PROJ-1234`. +/// stored as a comma-separated list — small, append-mostly. A reference +/// already in the list is not added again, so replaying the events that +/// carry it is idempotent; an unknown task is a no-op. Acceptable shapes +/// (loose, not enforced): `beads:claude-memory-rsw`, `github:#42`, +/// `jira:PROJ-1234`. pub fn add_task_external(conn: &Connection, task_id: &str, reference: &str) -> anyhow::Result<()> { - let current: Option = conn + let current: Option> = conn .query_row( "SELECT external FROM tasks WHERE task_id = ?1", rusqlite::params![task_id], |r| r.get::<_, Option>(0), ) + .optional() .with_context(|| format!("read external for {task_id}"))?; + let Some(current) = current else { + return Ok(()); + }; + if current + .as_deref() + .is_some_and(|s| s.split(',').any(|r| r == reference)) + { + return Ok(()); + } + let next = match current { Some(s) if !s.is_empty() => format!("{s},{reference}"), _ => reference.to_string(), @@ -920,6 +963,13 @@ pub fn ingest_new_events( } pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { + // An amend only changes task metadata, which upsert_task_from_event + // applies. It is not reasoning: keeping it out of events_index and + // search_fts keeps it out of packs, search, recall and memory sync. + if event.event_type == EventType::Amend { + return invalidate_pack_cascade(conn, &event.task_id); + } + let type_str = serde_json::to_value(event.event_type)? .as_str() .unwrap() @@ -2470,6 +2520,95 @@ mod tests { assert_eq!(meta.external.as_deref(), Some("loom:t-legacy")); } + #[test] + fn open_meta_goal_and_external_are_restored_by_a_rebuild() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let mut open_ev = make_open_event("tj-g", "Goal task"); + open_ev.meta = serde_json::json!({ + "title": "Goal task", + "goal": "Ship PKCE", + "external": ["loom:t-42", "github:#7"], + }); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + drop(f); + + let conn = open(d.path().join("fresh.sqlite")).unwrap(); + rebuild_state(&conn, &jsonl, ph).unwrap(); + + let meta = task_metadata(&conn, "tj-g").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("Ship PKCE")); + assert_eq!(meta.external.as_deref(), Some("loom:t-42,github:#7")); + assert_eq!( + task_id_by_external(&conn, "loom:t-42").unwrap().as_deref(), + Some("tj-g") + ); + } + + fn amend_event(task_id: &str, meta: serde_json::Value) -> crate::event::Event { + let mut e = crate::event::Event::new( + task_id, + crate::event::EventType::Amend, + crate::event::Author::User, + crate::event::Source::Cli, + "amend".into(), + ); + e.meta = meta; + e + } + + #[test] + fn amend_events_are_replayed_idempotently_and_stay_out_of_the_index() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let mut open_ev = make_open_event("tj-am", "Amended"); + open_ev.meta = serde_json::json!({"title": "Amended", "goal": "first goal"}); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &open_ev); + write_event_line( + &mut f, + &amend_event("tj-am", serde_json::json!({"goal": "second goal"})), + ); + for _ in 0..2 { + write_event_line( + &mut f, + &amend_event("tj-am", serde_json::json!({"external_add": ["beads:x"]})), + ); + } + // An amend for a task that does not exist must not break ingest. + write_event_line( + &mut f, + &amend_event("tj-ghost", serde_json::json!({"external_add": ["beads:y"]})), + ); + drop(f); + + let conn = open(d.path().join("fresh.sqlite")).unwrap(); + rebuild_state(&conn, &jsonl, ph).unwrap(); + // Replaying over an existing DB (e.g. after a migration) changes nothing. + rebuild_state(&conn, &jsonl, ph).unwrap(); + + let meta = task_metadata(&conn, "tj-am").unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("second goal")); + assert_eq!(meta.external.as_deref(), Some("beads:x")); + + let indexed: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events_index WHERE type = 'amend'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(indexed, 0, "amend is metadata, not a pack/search event"); + let pack = crate::pack::assemble(&conn, "tj-am", crate::pack::PackMode::Full).unwrap(); + assert!(pack.text.contains("**Goal**: second goal"), "{}", pack.text); + assert!(!pack.text.contains("[amend]"), "{}", pack.text); + } + #[test] fn open_event_meta_parent_id_is_persisted() { let d = tempfile::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/event.rs b/crates/tj-core/src/event.rs index de603a1..6dae98c 100644 --- a/crates/tj-core/src/event.rs +++ b/crates/tj-core/src/event.rs @@ -17,6 +17,11 @@ pub enum EventType { Close, Redirect, Rename, + /// Changes task metadata after creation: `meta.goal` replaces the goal, + /// `meta.external_add` (array of strings) adds external references. + /// Not reasoning — kept out of packs, search and recall. Written by the + /// `goal` / `external` commands, never accepted as a user event type. + Amend, } impl EventType { @@ -34,6 +39,7 @@ impl EventType { Self::Close, Self::Redirect, Self::Rename, + Self::Amend, ]; } diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index fbef18f..2dfcf0d 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -669,6 +669,15 @@ impl TaskJournalServer { if let Some(ref parent_id) = p.parent { event.meta["parent_id"] = serde_json::Value::String(parent_id.clone()); } + // Goal and the Loom tag ride in the open event, so ingest (and + // a rebuild from the JSONL) restores them; later resolves and + // the board → journal link find the journal by `loom:`. + if let Some(ref goal) = p.goal { + event.meta["goal"] = serde_json::Value::String(goal.clone()); + } + if let Some(ref r) = loom_ref { + event.meta["external"] = serde_json::json!([r]); + } tj_core::session_id::stamp_session_id( &mut event.meta, tj_core::session_id::session_id_from_env().as_deref(), @@ -678,30 +687,6 @@ impl TaskJournalServer { writer.append(&event)?; writer.flush_durable()?; - // v0.6.0: persist goal column when caller passed --goal / - // params.goal. We must ingest into SQLite first so the - // task row exists; without ingestion set_task_goal hits - // an empty tasks table and silently no-ops. - if let Some(goal) = p.goal.as_deref() { - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::set_task_goal(&conn, &task_id, goal)?; - } - - // Tag the new journal with its Loom task id so later resolves - // (and the board → journal link) find it. - if let Some(ref r) = loom_ref { - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::db::add_task_external(&conn, &task_id, r)?; - } - Ok(TaskCreateResult { task_id, title: p.title.clone(), @@ -1820,6 +1805,63 @@ mod tests { assert_eq!(after, before, "no orphan event may reach the journal"); } + #[tokio::test] + async fn task_create_goal_and_loom_ref_survive_a_state_rebuild() { + let _env = handler_env(); + let server = TaskJournalServer; + + let loom_id = format!("t-rebuild-{}", ulid::Ulid::new()); + std::env::set_var("LOOM_TASK_ID", &loom_id); + let created = server + .task_create(Parameters(TaskCreateParams { + title: "Loom task".into(), + initial_context: None, + goal: Some("Wire the board".into()), + parent: None, + })) + .await; + std::env::remove_var("LOOM_TASK_ID"); + let task = created.unwrap().0.task_id; + + // A fresh SQLite rebuilt from the journal alone. + let (project_hash, events_path, _) = project_paths().unwrap(); + let dir = tempfile::TempDir::new().unwrap(); + let conn = tj_core::db::open(dir.path().join("fresh.sqlite")).unwrap(); + tj_core::db::rebuild_state(&conn, &events_path, &project_hash).unwrap(); + + let meta = tj_core::db::task_metadata(&conn, &task).unwrap().unwrap(); + assert_eq!(meta.goal.as_deref(), Some("Wire the board")); + let by_loom = tj_core::db::task_id_by_external(&conn, &format!("loom:{loom_id}")).unwrap(); + assert_eq!(by_loom.as_deref(), Some(task.as_str())); + } + + #[tokio::test] + async fn event_add_rejects_the_internal_amend_type() { + let _env = handler_env(); + let server = TaskJournalServer; + + let task = create_task(&server, "No amend").await; + let res = server + .event_add(Parameters(EventAddParams { + task_id: task, + event_type: "amend".into(), + text: "goal: sneaky".into(), + corrects: None, + supersedes: None, + alternatives: None, + })) + .await; + let err = match res { + Ok(_) => panic!("amend is internal and must be rejected"), + Err(e) => e, + }; + assert!( + err.message.contains("unknown event type"), + "{}", + err.message + ); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full From 772cc6229cb00be9ec4eceb548d938b65f8dfa95 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:24:17 +0400 Subject: [PATCH 061/104] fix(cli): close records the outcome through the close event, not a direct SQLite write --- crates/tj-cli/src/main.rs | 16 +++++----- crates/tj-cli/tests/cli.rs | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 955a3bb..bde4dd0 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1699,13 +1699,6 @@ fn real_main() -> Result<()> { if !tj_core::db::task_exists(&conn, &task_id)? { anyhow::bail!("task not found: {task_id}"); } - // Persist outcome BEFORE the close event so the cache wipe - // inside set_task_outcome doesn't compete with subsequent - // assemble calls. Both columns optional — caller can pass - // neither and just get the close event. - if let Some(o) = outcome.as_deref() { - tj_core::db::set_task_outcome(&conn, &task_id, o, outcome_tag.as_deref())?; - } let open_kids = tj_core::db::count_open_children(&conn, &task_id)?; drop(conn); @@ -1720,6 +1713,15 @@ fn real_main() -> Result<()> { if let Some(r) = reason { meta.insert("reason".into(), serde_json::Value::String(r)); } + // outcome + tag ride in the close event's meta and reach the task + // row only when that event is ingested, so a failed append never + // leaves an open task with an outcome and a rebuild keeps it. + if let Some(o) = outcome { + meta.insert("outcome".into(), serde_json::Value::String(o)); + } + if let Some(t) = outcome_tag { + meta.insert("outcome_tag".into(), serde_json::Value::String(t)); + } // Layer-2 close harvest: stamp deterministic git/gh refs (commit, // branch, PR) so the closed pack reads as a clickable ledger of // what shipped. Best-effort; structured artifacts are merged in diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 02670c6..ed87672 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2283,6 +2283,70 @@ fn close_with_outcome_renders_outcome_block() { .stdout(contains("**Outcome** [done]: Shipped in v0.4.0")); } +#[test] +fn close_records_the_outcome_only_through_the_close_event() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let out = tj() + .args(["create", "Close me", "--goal", "g"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + let close = |outcome: &str| { + let mut cmd = tj(); + cmd.args([ + "close", + &task_id, + "--outcome", + outcome, + "--outcome-tag", + "done", + ]); + cmd + }; + + // The close append fails: the journal is read-only. + let journal = std::fs::read_dir(dir.path().join("task-journal").join("events")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let writable = std::fs::metadata(&journal).unwrap().permissions(); + let mut read_only = writable.clone(); + read_only.set_readonly(true); + std::fs::set_permissions(&journal, read_only).unwrap(); + close("must not stick").assert().failure(); + std::fs::set_permissions(&journal, writable).unwrap(); + + let state = std::fs::read_dir(dir.path().join("task-journal").join("state")) + .unwrap() + .map(|e| e.unwrap().path()) + .find(|p| p.extension().is_some_and(|x| x == "sqlite")) + .unwrap(); + let conn = tj_core::db::open(&state).unwrap(); + let meta = tj_core::db::task_metadata(&conn, &task_id) + .unwrap() + .unwrap(); + assert_eq!(meta.outcome, None, "failed close left an outcome behind"); + drop(conn); + + // A real close keeps its outcome through a rebuild from the journal. + close("shipped").assert().success(); + std::fs::remove_dir_all(dir.path().join("task-journal").join("state")).unwrap(); + tj().args(["pack", &task_id]) + .assert() + .success() + .stdout(contains("**Outcome** [done]: shipped")); +} + #[test] fn close_rejects_invalid_outcome_tag() { let dir = assert_fs::TempDir::new().unwrap(); From e26816569dc6df6d2aa29d9d31805e768290b303 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:25:02 +0400 Subject: [PATCH 062/104] fix(cli): search with an empty query lists tasks newest first instead of failing in FTS5 --- crates/tj-cli/src/main.rs | 19 ++++++++++++++++++- crates/tj-cli/tests/cli.rs | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index bde4dd0..3ede9d4 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -946,7 +946,7 @@ enum Commands { Reclassify { task_id: String }, /// Full-text search across events (FTS5). Search { - /// Query string. + /// Query string. Empty lists the tasks instead, newest first. query: String, #[arg(long, default_value_t = 20)] limit: usize, @@ -3704,6 +3704,23 @@ fn run_search( event_type: Option<&str>, limit: usize, ) -> Result> { + // No query: list the tasks, newest first, like MCP task_search. FTS5 + // rejects an empty MATCH, so it must not reach it. + if fts_query.is_empty() { + let mut stmt = conn.prepare( + "SELECT task_id FROM tasks \ + WHERE ?1 IS NULL OR task_id IN (SELECT task_id FROM events_index WHERE type = ?1) \ + ORDER BY last_event_at DESC LIMIT ?2", + )?; + let ids = stmt + .query_map(rusqlite::params![event_type, limit as i64], |r| { + r.get::<_, String>(0) + })? + .collect::>()?; + + return Ok(ids); + } + let (fts_sql, fts_uses_type) = match event_type { Some(_) => ( "SELECT DISTINCT task_id FROM search_fts \ diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index ed87672..51f27b0 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5018,6 +5018,44 @@ fn search_does_not_crash_on_hyphenated_identifier() { .stdout(contains("tj-")); } +#[test] +fn search_with_an_empty_query_lists_tasks_newest_first() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()).current_dir(&workdir); + cmd + }; + let create = |title: &str| { + let out = tj().args(["create", title]).assert().success(); + String::from_utf8(out.get_output().stdout.clone()) + .unwrap() + .trim() + .to_string() + }; + let search = |args: &[&str]| { + let out = tj().arg("search").args(args).assert().success(); + String::from_utf8(out.get_output().stdout.clone()) + .unwrap() + .lines() + .map(str::to_string) + .collect::>() + }; + + let older = create("Older task"); + let newer = create("Newer task"); + assert_eq!(search(&[""]), [newer.clone(), older.clone()]); + + tj().args(["event", &older, "--type", "decision", "--text", "Pick X"]) + .assert() + .success(); + assert_eq!(search(&[" "]), [older.clone(), newer.clone()]); + assert_eq!(search(&["", "--type", "decision"]), [older.as_str()]); + assert_eq!(search(&["", "--limit", "1"]), [older]); +} + #[test] fn search_does_not_crash_on_slash_or_colon() { // Same B1 family — paths and `ttl:30s`-style tokens used to crash. From 7259aa5c42fe959dfda577c30defcf98f8b50463 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:26:23 +0400 Subject: [PATCH 063/104] fix(search): searching a project without a journal returns nothing and creates no state DB --- crates/tj-cli/src/main.rs | 9 ++++++--- crates/tj-cli/tests/cli.rs | 24 ++++++++++++++++++++++++ crates/tj-mcp/src/main.rs | 28 +++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 3ede9d4..ee4741f 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3334,11 +3334,14 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + // No journal, no tasks — and no empty state DB left behind to + // show up in `--all-projects` and project lists later. + if !events_path.exists() { + return Ok(()); + } let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; let ids = run_search(&conn, &fts_query, &like_query, event_type.as_deref(), limit)?; for id in ids { println!("{id}"); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 51f27b0..74a17ee 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5056,6 +5056,30 @@ fn search_with_an_empty_query_lists_tasks_newest_first() { assert_eq!(search(&["", "--limit", "1"]), [older]); } +#[test] +fn search_in_a_project_without_a_journal_creates_no_state_db() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + + for query in ["", "anything"] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["search", query]) + .assert() + .success() + .stdout(""); + } + + let state = dir.path().join("task-journal").join("state"); + let created: Vec<_> = std::fs::read_dir(&state) + .map(|rd| rd.map(|e| e.unwrap().path()).collect()) + .unwrap_or_default(); + assert!(created.is_empty(), "{created:?}"); +} + #[test] fn search_does_not_crash_on_slash_or_colon() { // Same B1 family — paths and `ttl:30s`-style tokens used to crash. diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 2dfcf0d..27206ba 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -524,14 +524,17 @@ impl TaskJournalServer { Some(dir) => resolve_project_paths(Path::new(dir))?, None => project_paths()?, }; + // No journal, no tasks — and no empty state DB left behind to + // show up in project lists later. + if !events_path.exists() { + return Ok(Vec::new()); + } let conn_arc = cached_open(&state_path)?; let conn = conn_arc .lock() .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; // No query: list the project's tasks, newest first. FTS5 // rejects an empty MATCH, so it must not reach it. @@ -1805,6 +1808,25 @@ mod tests { assert_eq!(after, before, "no orphan event may reach the journal"); } + #[tokio::test] + async fn task_search_of_a_project_without_a_journal_creates_no_state_db() { + let _env = handler_env(); + let server = TaskJournalServer; + + let other = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(other.path().join(".git")).unwrap(); + let (_, _, other_state) = resolve_project_paths(other.path()).unwrap(); + let other_dir = other.path().to_str().unwrap(); + + for query in ["", "anything"] { + let res = search(&server, query, None, Some(other_dir), None) + .await + .unwrap(); + assert!(res.results.is_empty(), "{:?}", res.results); + } + assert!(!other_state.exists(), "created {other_state:?}"); + } + #[tokio::test] async fn task_create_goal_and_loom_ref_survive_a_state_rebuild() { let _env = handler_env(); From 6100afc743227b50895bfe8a44f77c4b0b1bb5b1 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:27:57 +0400 Subject: [PATCH 064/104] feat(mcp): optional session_id param on task_create, event_add, artifact_add and task_close --- crates/tj-mcp/src/main.rs | 133 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 27206ba..db1ca7f 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -235,6 +235,8 @@ pub struct TaskCreateParams { /// Parent task id — makes this a subtask of the given id. Validated: the /// parent must exist and the link must not introduce a cycle. pub parent: Option, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskCreateResult { @@ -254,6 +256,8 @@ pub struct EventAddParams { /// options and the final choice explicit. Stamped onto /// `meta.alternatives`. Rejected with an error on any non-decision type. pub alternatives: Option, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct EventAddResult { @@ -271,6 +275,8 @@ pub struct ArtifactAddParams { pub url: String, /// Human label shown on the card. pub label: String, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct ArtifactAddResult { @@ -286,6 +292,8 @@ pub struct TaskCloseParams { /// v0.4.0+: structured outcome tag — `done`, `abandoned`, or /// `superseded`. Filterable; the free-form text lives in `outcome`. pub outcome_tag: Option, + /// Filled in by the client or the Task Journal Claude Code mod; leave it out. + pub session_id: Option, } #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskCloseResult { @@ -357,6 +365,15 @@ fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::Pat resolve_project_paths(&project_dir()?) } +/// The session id to stamp on an event: the caller's `session_id` param when +/// given (empty counts as absent), else the live one from the environment. +fn session_id_or_env(explicit: Option<&str>) -> Option { + explicit + .filter(|s| !s.is_empty()) + .map(str::to_string) + .or_else(tj_core::session_id::session_id_from_env) +} + /// Ingest the journal tail, then fail on an unknown `task_id` so a typo never /// writes an orphan event. fn require_task( @@ -683,7 +700,7 @@ impl TaskJournalServer { } tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -742,7 +759,7 @@ impl TaskJournalServer { } tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -785,7 +802,7 @@ impl TaskJournalServer { event.meta = tj_core::artifacts::link_event_meta(&p.kind, &p.url, &p.label); tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -874,7 +891,7 @@ impl TaskJournalServer { event.meta = serde_json::Value::Object(meta); tj_core::session_id::stamp_session_id( &mut event.meta, - tj_core::session_id::session_id_from_env().as_deref(), + session_id_or_env(p.session_id.as_deref()).as_deref(), ); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; @@ -1261,6 +1278,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1279,6 +1297,7 @@ mod tests { corrects: None, supersedes: None, alternatives: Some(alts.clone()), + session_id: None, })) .await .unwrap() @@ -1305,6 +1324,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1319,6 +1339,7 @@ mod tests { corrects: None, supersedes: None, alternatives: Some(serde_json::json!([{"option": "x", "chosen": true}])), + session_id: None, })) .await; let err = match res { @@ -1347,6 +1368,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1359,6 +1381,7 @@ mod tests { initial_context: None, goal: None, parent: Some(parent.clone()), + session_id: None, })) .await .unwrap() @@ -1389,6 +1412,7 @@ mod tests { initial_context: None, goal: None, parent: None, + session_id: None, })) .await .unwrap() @@ -1402,6 +1426,7 @@ mod tests { initial_context: None, goal: None, parent: Some(parent.clone()), + session_id: None, })) .await .unwrap(); @@ -1412,6 +1437,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap() @@ -1431,6 +1457,7 @@ mod tests { initial_context: None, goal: Some("ship it".into()), parent: None, + session_id: None, })) .await .unwrap() @@ -1444,6 +1471,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap() @@ -1470,6 +1498,7 @@ mod tests { initial_context: None, goal: Some("ship it".into()), parent: None, + session_id: None, })) .await .unwrap() @@ -1485,6 +1514,7 @@ mod tests { corrects: None, supersedes: None, alternatives: None, + session_id: None, })) .await .unwrap(); @@ -1514,6 +1544,7 @@ mod tests { initial_context: None, goal: Some(format!("goal of {title}")), parent: None, + session_id: None, })) .await .unwrap() @@ -1559,6 +1590,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap(); @@ -1608,6 +1640,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap(); @@ -1697,6 +1730,7 @@ mod tests { reason: "done".into(), outcome: None, outcome_tag: None, + session_id: None, })) .await .unwrap(); @@ -1726,6 +1760,7 @@ mod tests { reason: "done".into(), outcome: Some(outcome.into()), outcome_tag: Some("done".into()), + session_id: None, }; // Make the close append fail: the journal is read-only. @@ -1774,6 +1809,7 @@ mod tests { corrects: None, supersedes: None, alternatives: None, + session_id: None, })) .await; let err = match res { @@ -1792,6 +1828,7 @@ mod tests { kind: "doc".into(), url: "https://example.com/spec".into(), label: "Spec".into(), + session_id: None, })) .await; let err = match res { @@ -1808,6 +1845,92 @@ mod tests { assert_eq!(after, before, "no orphan event may reach the journal"); } + #[tokio::test] + async fn an_explicit_session_id_is_stamped_instead_of_the_env_one() { + let _env = handler_env(); + let server = TaskJournalServer; + let prev = std::env::var("CLAUDE_CODE_SESSION_ID").ok(); + std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); + + let task = server + .task_create(Parameters(TaskCreateParams { + title: "Session task".into(), + initial_context: None, + goal: None, + parent: None, + session_id: Some("s-create".into()), + })) + .await + .unwrap() + .0 + .task_id; + let event = |session_id: Option<&str>| EventAddParams { + task_id: task.clone(), + event_type: "finding".into(), + text: "seen".into(), + corrects: None, + supersedes: None, + alternatives: None, + session_id: session_id.map(Into::into), + }; + let explicit = server.event_add(Parameters(event(Some("s-event")))).await; + let from_env = server.event_add(Parameters(event(None))).await; + let artifact = server + .artifact_add(Parameters(ArtifactAddParams { + task_id: task.clone(), + kind: "doc".into(), + url: "https://example.com/doc".into(), + label: "Doc".into(), + session_id: Some("s-artifact".into()), + })) + .await; + let closed = server + .task_close(Parameters(TaskCloseParams { + task_id: task.clone(), + reason: "done".into(), + outcome: None, + outcome_tag: None, + session_id: Some("s-close".into()), + })) + .await; + match prev { + Some(v) => std::env::set_var("CLAUDE_CODE_SESSION_ID", v), + None => std::env::remove_var("CLAUDE_CODE_SESSION_ID"), + } + let (explicit, from_env, artifact) = ( + explicit.unwrap().0.event_id, + from_env.unwrap().0.event_id, + artifact.unwrap().0.event_id, + ); + closed.unwrap(); + + let (_, events_path, _) = project_paths().unwrap(); + let session_of = |pick: &dyn Fn(&serde_json::Value) -> bool| { + std::fs::read_to_string(&events_path) + .unwrap() + .lines() + .map(|l| serde_json::from_str::(l).unwrap()) + .find(|v| pick(v)) + .and_then(|v| v["meta"]["session_id"].as_str().map(str::to_string)) + }; + let by_id = |id: &str| session_of(&|v| v["event_id"] == id); + let by_type = |ty: &str| session_of(&|v| v["task_id"] == task.as_str() && v["type"] == ty); + assert_eq!(by_type("open").as_deref(), Some("s-create")); + assert_eq!(by_id(&explicit).as_deref(), Some("s-event")); + assert_eq!(by_id(&from_env).as_deref(), Some("from-env")); + assert_eq!(by_id(&artifact).as_deref(), Some("s-artifact")); + assert_eq!(by_type("close").as_deref(), Some("s-close")); + } + + #[test] + fn session_id_params_are_optional() { + let p: EventAddParams = serde_json::from_value(serde_json::json!({ + "task_id": "tj-x", "event_type": "finding", "text": "t" + })) + .unwrap(); + assert_eq!(p.session_id, None); + } + #[tokio::test] async fn task_search_of_a_project_without_a_journal_creates_no_state_db() { let _env = handler_env(); @@ -1840,6 +1963,7 @@ mod tests { initial_context: None, goal: Some("Wire the board".into()), parent: None, + session_id: None, })) .await; std::env::remove_var("LOOM_TASK_ID"); @@ -1871,6 +1995,7 @@ mod tests { corrects: None, supersedes: None, alternatives: None, + session_id: None, })) .await; let err = match res { From c4ec608b108ce6026274e6e0712002b3917b6d75 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:38:11 +0400 Subject: [PATCH 065/104] test(core): serialize every env-touching tj-core test on one shared lock --- crates/tj-core/src/classifier/hybrid.rs | 3 ++- crates/tj-core/src/classifier/mod.rs | 3 ++- crates/tj-core/src/consolidate.rs | 1 + crates/tj-core/src/lib.rs | 10 ++++++++++ crates/tj-core/src/llm.rs | 17 +++++++---------- crates/tj-core/src/pack.rs | 1 + crates/tj-core/src/paths.rs | 6 ++++-- crates/tj-core/src/session/discovery.rs | 22 ++++++---------------- crates/tj-core/src/session_id.rs | 13 +++++-------- 9 files changed, 38 insertions(+), 38 deletions(-) diff --git a/crates/tj-core/src/classifier/hybrid.rs b/crates/tj-core/src/classifier/hybrid.rs index f2375c2..5a07de3 100644 --- a/crates/tj-core/src/classifier/hybrid.rs +++ b/crates/tj-core/src/classifier/hybrid.rs @@ -174,7 +174,8 @@ mod tests { #[test] fn from_env_constructs_without_key() { - // SAFETY: tests in this crate do not concurrently read these env vars. + // SAFETY: the shared env lock serializes the crate's env tests. + let _env = crate::test_env_lock(); let prev_key = std::env::var("ANTHROPIC_API_KEY").ok(); // Force heuristic-only by disabling both LLM backends via an order that // names no real one, so this stays deterministic regardless of whether diff --git a/crates/tj-core/src/classifier/mod.rs b/crates/tj-core/src/classifier/mod.rs index b682465..bd22b1e 100644 --- a/crates/tj-core/src/classifier/mod.rs +++ b/crates/tj-core/src/classifier/mod.rs @@ -91,10 +91,11 @@ mod tests { /// other tests in this crate. #[test] fn tj_classifier_model_env_var_overrides_http_default() { + let _env = crate::test_env_lock(); let prev_model = std::env::var("TJ_CLASSIFIER_MODEL").ok(); let prev_key = std::env::var("ANTHROPIC_API_KEY").ok(); - // SAFETY: tests in this crate do not concurrently read these env vars. + // SAFETY: the shared env lock serializes the crate's env tests. unsafe { std::env::remove_var("TJ_CLASSIFIER_MODEL"); std::env::set_var("ANTHROPIC_API_KEY", "test-key-do-not-use"); diff --git a/crates/tj-core/src/consolidate.rs b/crates/tj-core/src/consolidate.rs index ffbe61c..646302d 100644 --- a/crates/tj-core/src/consolidate.rs +++ b/crates/tj-core/src/consolidate.rs @@ -202,6 +202,7 @@ mod tests { #[test] fn summarize_skips_when_backend_forced_none() { + let _env = crate::test_env_lock(); std::env::set_var("TJ_CONSOLIDATE_BACKEND", "none"); let r = summarize(&["chose ledger".into()], 5, None).unwrap(); std::env::remove_var("TJ_CONSOLIDATE_BACKEND"); diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index df98c4a..8b12e6b 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -19,6 +19,16 @@ pub fn new_task_id() -> String { ) } +/// One lock for every tj-core test that sets env vars or asserts on +/// something read from them: std env is process-global, so tests running in +/// parallel see each other's values. Poisoning is ignored so one failed test +/// does not fail every other env test after it. +#[cfg(test)] +pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + #[cfg(test)] mod task_id_tests { use super::new_task_id; diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs index f96498b..465bc88 100644 --- a/crates/tj-core/src/llm.rs +++ b/crates/tj-core/src/llm.rs @@ -499,15 +499,12 @@ mod tests { } } - // Serialise env-touching tests (process-global env). - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - /// `codex` used to resolve to the OpenAI API backend, so an OPENAI_API_KEY /// in the environment must no longer pull the wrong backend in. Whether the /// Codex CLI is installed decides between `Some(codex-exec)` and `None`. #[test] fn codex_backend_is_the_codex_cli_not_the_openai_api() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _key = EnvGuard::set("OPENAI_API_KEY", "sk-should-not-be-used"); match backend_from_env(Some("codex")).unwrap() { Some(b) => assert_eq!(b.name(), "codex-exec"), @@ -546,7 +543,7 @@ mod tests { #[cfg(unix)] #[test] fn codex_child_runs_with_the_recursion_marker() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let dir = fake_codex_dir(); let _path = path_with(dir.path()); let _marker = EnvGuard::unset("TJ_IN_CLASSIFIER"); @@ -562,7 +559,7 @@ mod tests { #[cfg(unix)] #[test] fn codex_output_file_is_removed_when_the_call_times_out() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let dir = fake_codex_dir(); let _path = path_with(dir.path()); let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); @@ -582,20 +579,20 @@ mod tests { #[test] fn unknown_backend_errors() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); assert!(backend_from_env(Some("nonsense")).is_err()); } #[test] fn anthropic_unavailable_without_key_is_none() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _g = EnvGuard::unset("ANTHROPIC_API_KEY"); assert!(backend_from_env(Some("anthropic")).unwrap().is_none()); } #[test] fn anthropic_with_key_resolves() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let _g = EnvGuard::set("ANTHROPIC_API_KEY", "k"); let b = backend_from_env(Some("anthropic")).unwrap().unwrap(); assert_eq!(b.name(), "anthropic"); @@ -603,7 +600,7 @@ mod tests { #[test] fn ollama_always_resolves_no_key() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let b = backend_from_env(Some("ollama")).unwrap().unwrap(); assert_eq!(b.name(), "ollama"); } diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index b4dab00..3d076cb 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -592,6 +592,7 @@ mod tests { // pending_count() resolves `/pending`. Point the data dir at // the isolated tempdir (no pending/ child) so the PendingLeak rule // stays silent regardless of the real dev environment. + let _env = crate::test_env_lock(); std::env::set_var("TASK_JOURNAL_DATA_DIR", d.path()); let pack = assemble(&conn, "g2", PackMode::Compact).unwrap(); diff --git a/crates/tj-core/src/paths.rs b/crates/tj-core/src/paths.rs index d854209..cda0a12 100644 --- a/crates/tj-core/src/paths.rs +++ b/crates/tj-core/src/paths.rs @@ -54,6 +54,7 @@ mod tests { #[test] fn data_dir_returns_a_path_containing_task_journal() { + let _env = crate::test_env_lock(); let p = data_dir().expect("data_dir"); let s = p.to_string_lossy(); assert!(s.contains("task-journal"), "got: {s}"); @@ -67,14 +68,15 @@ mod tests { /// Regression: on macOS/Windows the `directories` crate ignores XDG_DATA_HOME, but our /// tests (and power users) need a portable override. data_dir() must respect XDG_DATA_HOME - /// and TASK_JOURNAL_DATA_DIR on every OS. Using a thread-isolated env block since std env - /// is process-global; one test exercises both vars by serially restoring state. + /// and TASK_JOURNAL_DATA_DIR on every OS. std env is process-global, so the test holds + /// the shared env lock; one test exercises both vars by serially restoring state. #[test] #[cfg_attr( not(unix), ignore = "env semantics differ on Windows test runners; covered by integration tests" )] fn env_overrides_take_precedence() { + let _env = crate::test_env_lock(); // Snapshot existing values (best-effort cleanup). let prev_tjdd = std::env::var("TASK_JOURNAL_DATA_DIR").ok(); let prev_xdg = std::env::var("XDG_DATA_HOME").ok(); diff --git a/crates/tj-core/src/session/discovery.rs b/crates/tj-core/src/session/discovery.rs index e9a26a0..89bd975 100644 --- a/crates/tj-core/src/session/discovery.rs +++ b/crates/tj-core/src/session/discovery.rs @@ -108,16 +108,6 @@ fn dirs_home() -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - use std::sync::Mutex; - - /// Serialize every test that touches `CLAUDE_CONFIG_DIR`. Cargo runs - /// unit tests in parallel by default; two tests mutating the same - /// process env race (set in A, observed in B) and flaked Windows CI - /// (saw "C:\Users\runneradmin\.claude" when expecting the override). - /// Tests that touch the env take this lock before the first set_var. - /// `lock().unwrap_or_else(|p| p.into_inner())` swallows poisoning - /// from a panicking sibling test — env is restored regardless. - static ENV_LOCK: Mutex<()> = Mutex::new(()); #[test] fn encode_path_replaces_separators() { @@ -205,7 +195,7 @@ mod tests { #[test] fn find_project_dir_with_env_override() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); let projects = dir.path().join("projects"); std::fs::create_dir_all(&projects).unwrap(); @@ -230,7 +220,7 @@ mod tests { #[test] fn find_project_dir_returns_none_when_no_match() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); let projects = dir.path().join("projects"); std::fs::create_dir_all(&projects).unwrap(); @@ -246,7 +236,7 @@ mod tests { #[test] fn find_project_dir_returns_none_when_projects_dir_missing() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let _g = crate::test_env_lock(); let dir = tempfile::tempdir().unwrap(); // Don't create a "projects" subdir — it doesn't exist. @@ -270,9 +260,9 @@ mod tests { /// than the hardcoded "/tmp/..." that doesn't exist on Windows. #[test] fn claude_config_dir_handles_env_var() { - let _g = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner()); - // SAFETY: the ENV_LOCK above serializes us against the other - // CLAUDE_CONFIG_DIR tests in this module; prev → restore at + let _g = crate::test_env_lock(); + // SAFETY: the shared env lock serializes us against the other + // env-touching tests; prev → restore at // the end gives a clean exit regardless of panic. let prev = std::env::var_os("CLAUDE_CONFIG_DIR"); diff --git a/crates/tj-core/src/session_id.rs b/crates/tj-core/src/session_id.rs index 37e6703..ee3710d 100644 --- a/crates/tj-core/src/session_id.rs +++ b/crates/tj-core/src/session_id.rs @@ -59,11 +59,8 @@ pub fn stamp_session_id(meta: &mut Value, sid: Option<&str>) { #[cfg(test)] mod tests { use super::*; + use crate::test_env_lock; use serde_json::json; - use std::sync::Mutex; - - // Serialises the env-touching tests — std env is process-global. - static ENV_LOCK: Mutex<()> = Mutex::new(()); #[test] fn payload_session_id_extracted() { @@ -102,7 +99,7 @@ mod tests { #[test] fn live_payload_wins_over_env() { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); let p = json!({"session_id": "from-payload"}); assert_eq!(live_session_id(Some(&p)).as_deref(), Some("from-payload")); @@ -111,7 +108,7 @@ mod tests { #[test] fn live_falls_back_to_env() { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); std::env::set_var("CLAUDE_CODE_SESSION_ID", "from-env"); let p = json!({"hook_event_name": "Stop"}); assert_eq!(live_session_id(Some(&p)).as_deref(), Some("from-env")); @@ -121,7 +118,7 @@ mod tests { #[test] fn live_none_when_no_source() { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); for var in SESSION_ENV_VARS { std::env::remove_var(var); } @@ -137,7 +134,7 @@ mod tests { /// Run `f` with exactly the given session env vars set (the rest unset). fn with_session_env(set: &[(&str, &str)], f: impl FnOnce()) { - let _g = ENV_LOCK.lock().unwrap(); + let _g = test_env_lock(); for var in SESSION_ENV_VARS { std::env::remove_var(var); } From 0ddb4ff776b962b3494c0c01e7dff5f033a73d73 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:40:14 +0400 Subject: [PATCH 066/104] test(core): use the shared env lock in the codex timeout test after merge --- crates/tj-core/src/llm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs index 4691732..498bef9 100644 --- a/crates/tj-core/src/llm.rs +++ b/crates/tj-core/src/llm.rs @@ -581,7 +581,7 @@ mod tests { #[cfg(unix)] #[test] fn codex_timeout_error_names_codex() { - let _l = ENV_LOCK.lock().unwrap(); + let _l = crate::test_env_lock(); let dir = fake_codex_dir(); let _path = path_with(dir.path()); let _timeout = EnvGuard::set("TJ_CODEX_TIMEOUT_SECS", "1"); From 5eb4fca97944964816a278c9910e9932579111f4 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:41:50 +0400 Subject: [PATCH 067/104] feat(cli): state command and suggested/session/origin flags on event for integrations --- crates/tj-cli/src/main.rs | 83 ++++++++++-- crates/tj-cli/tests/cli.rs | 144 +++++++++++++++++++++ crates/tj-core/src/lib.rs | 1 + crates/tj-core/src/session_state.rs | 188 ++++++++++++++++++++++++++++ plugin/hooks/register.ts | 2 +- 5 files changed, 409 insertions(+), 9 deletions(-) create mode 100644 crates/tj-core/src/session_state.rs diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 550e359..9644563 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -870,6 +870,26 @@ enum Commands { /// Optional event id this supersedes (for type=supersede). #[arg(long)] supersedes: Option, + /// Record it as `suggested` by a classifier rather than confirmed + /// by a person: what a model inferred after the fact. + #[arg(long)] + suggested: bool, + /// Session the event belongs to; defaults to the live session id + /// (CLAUDE_CODE_SESSION_ID, then Codex's CODEX_THREAD_ID). + #[arg(long)] + session: Option, + /// Who wrote it, kept as `meta.origin` (e.g. `mod-distill`). + #[arg(long)] + origin: Option, + }, + /// Print a session's journal state as JSON: its active task (the open + /// task it last wrote to), that task's counts and latest entries, and + /// how many tasks are open. Read by the Claude Code mod; schema + /// `tj-state/1`. + State { + /// Session id; defaults to the live session id. + #[arg(long)] + session: Option, }, /// Close a task (writes a `close` event). Close { @@ -1627,28 +1647,75 @@ fn real_main() -> Result<()> { text, corrects, supersedes, + suggested, + session, + origin, } => { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); - std::fs::create_dir_all(events_path.parent().unwrap())?; + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + // Same guard as the MCP tool: a mistyped id must not leave an + // orphan event behind. let event_type = parse_event_type(&r#type)?; - let mut event = tj_core::event::Event::new( - &task_id, - event_type, - tj_core::event::Author::User, - tj_core::event::Source::Cli, - text, - ); + if !events_path.exists() { + anyhow::bail!("task not found: {task_id}"); + } + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + if !tj_core::db::task_exists(&conn, &task_id)? { + anyhow::bail!("task not found: {task_id}"); + } + + let (author, source) = if suggested { + ( + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + ) + } else { + (tj_core::event::Author::User, tj_core::event::Source::Cli) + }; + let mut event = tj_core::event::Event::new(&task_id, event_type, author, source, text); event.corrects = corrects; event.supersedes = supersedes; + if suggested { + event.status = tj_core::event::EventStatus::Suggested; + } + if let Some(origin) = origin { + event.meta["origin"] = serde_json::Value::String(origin); + } + let session = session.or_else(tj_core::session_id::session_id_from_env); + tj_core::session_id::stamp_session_id(&mut event.meta, session.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; writer.flush_durable()?; println!("{}", event.event_id); } + Commands::State { session } => { + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let session = session.or_else(tj_core::session_id::session_id_from_env); + + // No journal yet: an empty state, and no state DB created for it. + let state = if events_path.exists() { + let conn = tj_core::db::open(&state_path)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + tj_core::session_state::session_state(&conn, &project_hash, session.as_deref())? + } else { + tj_core::session_state::SessionState { + schema: tj_core::session_state::SCHEMA, + session_id: session, + open_tasks: 0, + active: None, + } + }; + + println!("{}", serde_json::to_string(&state)?); + } Commands::ArtifactAdd { task_id, kind, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 609e4bb..10f51ae 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -7810,3 +7810,147 @@ fn artifact_add_renders_clickable_link_in_pack() { .success() .stdout(contains("[Design spec](https://example.com/spec.md) (doc)")); } + +#[test] +fn state_reports_the_sessions_own_task_as_json() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let state = |session: &str| -> serde_json::Value { + let out = tj() + .args(["state", "--session", session]) + .assert() + .success() + .get_output() + .stdout + .clone(); + serde_json::from_slice(&out).unwrap() + }; + + let empty = state("s1"); + assert_eq!(empty["schema"], "tj-state/1"); + assert!(empty["active"].is_null()); + assert!( + !dir.path().join("task-journal").join("state").exists(), + "no journal yet, so no state DB either" + ); + + let out = tj() + .args(["create", "Fix refresh", "--goal", "Stop dropping the token"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + tj().args([ + "event", + &task_id, + "--type", + "decision", + "--text", + "Use <= on expiry", + "--session", + "s1", + ]) + .assert() + .success(); + + let mine = state("s1"); + assert_eq!(mine["active"]["task_id"], task_id.as_str()); + assert_eq!(mine["active"]["goal"], "Stop dropping the token"); + assert_eq!(mine["active"]["counts"]["decision"], 1); + assert_eq!(mine["active"]["recent"][0]["text"], "Use <= on expiry"); + assert_eq!(mine["open_tasks"], 1); + + let other = state("s2"); + assert!( + other["active"].is_null(), + "another session must not inherit the task" + ); + assert_eq!(other["open_tasks"], 1); +} + +#[test] +fn event_can_record_a_suggested_entry_for_a_session() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + let out = tj() + .args(["create", "Distill target"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let task_id = String::from_utf8(out).unwrap().trim().to_string(); + + tj().args([ + "event", + &task_id, + "--type", + "rejection", + "--text", + "Ruled out a cache", + "--suggested", + "--session", + "s9", + "--origin", + "mod-distill", + ]) + .assert() + .success(); + + let events_dir = dir.path().join("task-journal").join("events"); + let journal = std::fs::read_dir(&events_dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let last: serde_json::Value = serde_json::from_str( + std::fs::read_to_string(journal) + .unwrap() + .lines() + .last() + .unwrap(), + ) + .unwrap(); + assert_eq!(last["type"], "rejection"); + assert_eq!(last["status"], "suggested"); + assert_eq!(last["author"], "classifier"); + assert_eq!(last["meta"]["session_id"], "s9"); + assert_eq!(last["meta"]["origin"], "mod-distill"); +} + +#[test] +fn event_on_an_unknown_task_fails_and_writes_nothing() { + let dir = assert_fs::TempDir::new().unwrap(); + let tj = || { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()); + cmd + }; + tj().args(["create", "Real task"]).assert().success(); + let events_dir = dir.path().join("task-journal").join("events"); + let journal = std::fs::read_dir(&events_dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let before = std::fs::read_to_string(&journal).unwrap(); + + tj().args(["event", "tj-typo", "--type", "decision", "--text", "x"]) + .assert() + .failure() + .stderr(contains("task not found: tj-typo")); + + assert_eq!(std::fs::read_to_string(&journal).unwrap(), before); +} diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index 8b12e6b..e66599a 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -78,6 +78,7 @@ pub mod recall; pub mod reminder; pub mod session; pub mod session_id; +pub mod session_state; pub mod storage; pub mod title; diff --git a/crates/tj-core/src/session_state.rs b/crates/tj-core/src/session_state.rs new file mode 100644 index 0000000..e464005 --- /dev/null +++ b/crates/tj-core/src/session_state.rs @@ -0,0 +1,188 @@ +//! What an integration shows for one session: the session's active task, +//! what that task holds so far, and how many tasks are open. Read by the +//! Claude Code mod through `task-journal state`; any client can call it. +//! +//! The active task is the open task this session last wrote to (by +//! `meta.session_id`), never "the newest open task", so two sessions in one +//! project don't share a task. + +use std::collections::BTreeMap; + +use rusqlite::Connection; +use serde::Serialize; + +/// Bumped when a field changes meaning; readers refuse other schemas. +pub const SCHEMA: &str = "tj-state/1"; + +/// How many of the task's latest entries ride along: enough for a model to +/// tell what is already recorded, small enough to stay cheap. +const RECENT: i64 = 20; + +#[derive(Debug, Serialize)] +pub struct SessionState { + pub schema: &'static str, + pub session_id: Option, + pub open_tasks: i64, + pub active: Option, +} + +#[derive(Debug, Serialize)] +pub struct ActiveTask { + pub task_id: String, + pub title: String, + pub goal: Option, + /// Entries per event type, e.g. `{"decision": 4, "evidence": 1}`. + pub counts: BTreeMap, + /// The latest entries, oldest first. No `open` / `amend` bookkeeping. + pub recent: Vec, +} + +#[derive(Debug, Serialize)] +pub struct RecentEntry { + #[serde(rename = "type")] + pub kind: String, + pub text: String, +} + +/// Assemble the state of `session_id` in one project. The caller has +/// ingested the journal into `conn` already. +pub fn session_state( + conn: &Connection, + project_hash: &str, + session_id: Option<&str>, +) -> anyhow::Result { + let open_tasks = conn.query_row( + "SELECT COUNT(*) FROM tasks WHERE project_hash = ?1 AND status = 'open'", + [project_hash], + |r| r.get::<_, i64>(0), + )?; + + let active_id = match session_id { + Some(sid) => crate::db::active_task_for_session(conn, project_hash, sid)?, + None => None, + }; + let active = active_id.map(|id| active_task(conn, &id)).transpose()?; + + Ok(SessionState { + schema: SCHEMA, + session_id: session_id.map(str::to_string), + open_tasks, + active, + }) +} + +fn active_task(conn: &Connection, task_id: &str) -> anyhow::Result { + let (title, goal) = conn.query_row( + "SELECT title, goal FROM tasks WHERE task_id = ?1", + [task_id], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)), + )?; + + let mut counts = BTreeMap::new(); + let mut stmt = + conn.prepare("SELECT type, COUNT(*) FROM events_index WHERE task_id = ?1 GROUP BY type")?; + for row in stmt.query_map([task_id], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)) + })? { + let (kind, n) = row?; + counts.insert(kind, n); + } + + let mut stmt = conn.prepare( + "SELECT e.type, f.text FROM events_index e + JOIN search_fts f ON f.event_id = e.event_id + WHERE e.task_id = ?1 AND e.type NOT IN ('open', 'amend') + ORDER BY e.timestamp DESC, e.event_id DESC LIMIT ?2", + )?; + let mut recent = stmt + .query_map(rusqlite::params![task_id, RECENT], |r| { + Ok(RecentEntry { + kind: r.get(0)?, + text: r.get(1)?, + }) + })? + .collect::, _>>()?; + recent.reverse(); + + Ok(ActiveTask { + task_id: task_id.to_string(), + title, + goal: goal.filter(|g| !g.trim().is_empty()), + counts, + recent, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::{Author, Event, EventType, Source}; + + fn write(conn: &Connection, task: &str, kind: EventType, text: &str, session: Option<&str>) { + let mut event = Event::new(task, kind, Author::Agent, Source::Chat, text.into()); + if kind == EventType::Open { + event.meta = + serde_json::json!({ "title": format!("Title of {task}"), "goal": "Ship it" }); + } + crate::session_id::stamp_session_id(&mut event.meta, session); + + crate::db::upsert_task_from_event(conn, &event, "p").unwrap(); + crate::db::index_event(conn, &event).unwrap(); + } + + // The TempDir rides along so the database outlives the test body. + fn conn() -> (tempfile::TempDir, Connection) { + let dir = tempfile::tempdir().unwrap(); + let conn = crate::db::open(dir.path().join("s.sqlite")).unwrap(); + (dir, conn) + } + + #[test] + fn active_task_is_the_sessions_own_not_the_newest() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + write(&conn, "tj-a", EventType::Decision, "Use <=", Some("s1")); + write(&conn, "tj-b", EventType::Open, "b", Some("s2")); + + let s1 = session_state(&conn, "p", Some("s1")).unwrap(); + let active = s1.active.expect("s1 has a task"); + assert_eq!(active.task_id, "tj-a"); + assert_eq!(active.title, "Title of tj-a"); + assert_eq!(active.goal.as_deref(), Some("Ship it")); + assert_eq!(active.counts.get("decision"), Some(&1)); + assert_eq!(active.recent.len(), 1); + assert_eq!(active.recent[0].kind, "decision"); + assert_eq!(s1.open_tasks, 2); + + let other = session_state(&conn, "p", Some("s3")).unwrap(); + assert!(other.active.is_none()); + assert_eq!(other.open_tasks, 2); + } + + #[test] + fn a_closed_task_is_no_longer_active() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + write(&conn, "tj-a", EventType::Close, "done", Some("s1")); + + let s = session_state(&conn, "p", Some("s1")).unwrap(); + assert!(s.active.is_none()); + assert_eq!(s.open_tasks, 0); + } + + #[test] + fn json_shape_is_the_contract() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + + let v = serde_json::to_value(session_state(&conn, "p", Some("s1")).unwrap()).unwrap(); + assert_eq!(v["schema"], "tj-state/1"); + assert_eq!(v["session_id"], "s1"); + assert_eq!(v["active"]["task_id"], "tj-a"); + assert!(v["active"]["recent"].as_array().unwrap().is_empty()); + + let none = serde_json::to_value(session_state(&conn, "p", None).unwrap()).unwrap(); + assert!(none["active"].is_null()); + assert!(none["session_id"].is_null()); + } +} diff --git a/plugin/hooks/register.ts b/plugin/hooks/register.ts index 6f7053c..f35c2ca 100644 --- a/plugin/hooks/register.ts +++ b/plugin/hooks/register.ts @@ -57,7 +57,7 @@ async function cli($: EngineInterface, argv: string[]): Promise { async function refresh($: EngineInterface): Promise { const session = await $.session.id() - const out = await cli($, ['state', '--session', session, '--json']) + const out = await cli($, ['state', '--session', session]) const state = out === null ? null : parseState(out) if (state === null) { From 28eebc5849ab3cacc8c0d8674d550e753c78f10c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:46:49 +0400 Subject: [PATCH 068/104] fix(pack): drop corrected events from every pack section and export-pr --- crates/tj-cli/src/main.rs | 3 +- crates/tj-cli/tests/cli.rs | 58 +++++++++++++++++++++++ crates/tj-core/src/db.rs | 75 +++++++++++++++++++++++++++++ crates/tj-core/src/pack.rs | 97 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 228 insertions(+), 5 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 550e359..ce9eca2 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3959,7 +3959,8 @@ fn run_export_pr(task_id: &str) -> Result<()> { let mut stmt = conn.prepare( "SELECT ei.type, sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 ORDER BY ei.timestamp ASC", + WHERE ei.task_id = ?1 AND ei.corrected_by IS NULL + ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { let ty: String = r.get(0)?; diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 609e4bb..bda921b 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5007,6 +5007,64 @@ fn export_pr_omits_optional_sections_when_no_data() { ); } +/// Run `task-journal ` in `workdir` under `xdg` and return trimmed stdout. +fn tj_stdout(xdg: &std::path::Path, workdir: &std::path::Path, args: &[&str]) -> String { + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg) + .current_dir(workdir) + .args(args) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(out).unwrap().trim().to_string() +} + +#[test] +fn export_pr_leaves_out_corrected_events() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let xdg = dir.path(); + + let task = tj_stdout(xdg, &workdir, &["create", "Corrected PR", "--goal", "g"]); + for (ty, text) in [ + ("decision", "Wrong decision: use Mongo"), + ("rejection", "Wrong rejection: SQLite too slow"), + ("evidence", "Wrong evidence: bench 5ms"), + ] { + let id = tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", ty, "--text", text], + ); + tj_stdout( + xdg, + &workdir, + &[ + "event-correct", + "--corrects", + &id, + "--task", + &task, + "--text", + "That was a mistake", + ], + ); + } + tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", "decision", "--text", "Use SQLite"], + ); + + let pr = tj_stdout(xdg, &workdir, &["export-pr", &task]); + assert!(!pr.contains("Wrong "), "{pr}"); + assert!(pr.contains("- Use SQLite"), "{pr}"); +} + #[test] fn export_pr_unknown_task_id_exits_one_with_stderr_message() { let dir = assert_fs::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 6f4a81e..521c098 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -160,6 +160,15 @@ CREATE INDEX IF NOT EXISTS idx_events_session_time ON events_index(session_id, t DELETE FROM index_state; "#; +/// v0.30.0 corrections — `corrected_by` holds the `event_id` of the +/// `correction` event whose `corrects` points at this event, so packs leave +/// the corrected event out without re-reading the log. Clearing +/// `index_state` replays the log once to fill it for existing events. +const MIGRATION_010: &str = r#" +ALTER TABLE events_index ADD COLUMN corrected_by TEXT; +DELETE FROM index_state; +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -199,6 +208,10 @@ const MIGRATIONS: &[Migration] = &[ version: 9, sql: MIGRATION_009, }, + Migration { + version: 10, + sql: MIGRATION_010, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -1043,6 +1056,27 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { } } + // A correction retires the event it corrects from packs and export-pr. + // The target may sit in another task, whose cached pack is now stale too. + if event.event_type == EventType::Correction { + if let Some(target) = &event.corrects { + conn.execute( + "UPDATE events_index SET corrected_by=?1 WHERE event_id=?2", + rusqlite::params![event.event_id, target], + )?; + let target_task: Option = conn + .query_row( + "SELECT task_id FROM events_index WHERE event_id=?1", + rusqlite::params![target], + |r| r.get(0), + ) + .optional()?; + if let Some(t) = target_task.filter(|t| *t != event.task_id) { + invalidate_pack_cascade(conn, &t)?; + } + } + } + if event.event_type == EventType::Evidence { let strength_str = event .evidence_strength @@ -2520,6 +2554,47 @@ mod tests { assert_eq!(meta.external.as_deref(), Some("loom:t-legacy")); } + #[test] + fn upgrading_an_existing_db_links_corrections_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-x", "Upgrade"); + let wrong = make_text_event("Migration done (wrong)"); + let mut corr = make_text_event("Migration NOT done"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &wrong, &corr] { + write_event_line(&mut f, e); + } + drop(f); + + // A pre-corrected_by database that has already indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "ALTER TABLE events_index DROP COLUMN corrected_by; + DELETE FROM schema_migrations WHERE version = 10;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + let corrected_by: Option = conn + .query_row( + "SELECT corrected_by FROM events_index WHERE event_id = ?1", + rusqlite::params![wrong.event_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); + } + #[test] fn open_meta_goal_and_external_are_restored_by_a_rebuild() { let d = TempDir::new().unwrap(); diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index 3d076cb..d3f98f4 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -47,7 +47,8 @@ fn render_recent_events(conn: &Connection, task_id: &str, limit: usize) -> anyho let mut stmt = conn.prepare( "SELECT ei.timestamp, ei.type, ei.status, sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id=?1 ORDER BY ei.timestamp DESC LIMIT ?2", + WHERE ei.task_id=?1 AND ei.corrected_by IS NULL + ORDER BY ei.timestamp DESC LIMIT ?2", )?; let rows = stmt.query_map(rusqlite::params![task_id, limit as i64], |r| { let ts: String = r.get(0)?; @@ -80,7 +81,10 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result { // v0.10.3: newest evidence first (ULID DESC). Matches the // decision-ordering fix so truncation prefers older rows. let mut stmt = conn.prepare( - "SELECT text, strength FROM evidence WHERE task_id=?1 ORDER BY evidence_id DESC", + "SELECT e.text, e.strength FROM evidence e + JOIN events_index ei ON ei.event_id = e.evidence_id + WHERE e.task_id=?1 AND ei.corrected_by IS NULL + ORDER BY e.evidence_id DESC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { let t: String = r.get(0)?; @@ -106,7 +110,7 @@ fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result { // OLDEST rejections, not the latest decision the agent recorded. let mut id_stmt = conn.prepare( "SELECT event_id FROM events_index - WHERE task_id=?1 AND type='rejection' + WHERE task_id=?1 AND type='rejection' AND corrected_by IS NULL ORDER BY timestamp DESC", )?; let mut text_stmt = conn.prepare("SELECT text FROM search_fts WHERE event_id=?1 LIMIT 1")?; @@ -137,7 +141,10 @@ fn render_active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result(0)?, r.get::<_, Option>(1)?)) @@ -1208,4 +1215,86 @@ mod tests { assert_eq!(pack.metadata.source_event_count, 1); assert!(!pack.metadata.cache_hit); } + + /// Upsert + index one event, the way the tests above do inline. + fn put(conn: &Connection, e: &crate::event::Event) { + crate::db::upsert_task_from_event(conn, e, "feedface").unwrap(); + crate::db::index_event(conn, e).unwrap(); + } + + fn ev(task: &str, ty: crate::event::EventType, text: &str) -> crate::event::Event { + crate::event::Event::new( + task, + ty, + crate::event::Author::Agent, + crate::event::Source::Chat, + text.into(), + ) + } + + fn open_task(conn: &Connection, task: &str, title: &str) { + let mut e = ev(task, crate::event::EventType::Open, "x"); + e.meta = serde_json::json!({ "title": title }); + put(conn, &e); + } + + #[test] + fn corrected_events_drop_out_of_every_pack_section() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-cx", "Corrected"); + + let wrong = [ + ev("tj-cx", EventType::Decision, "Wrong decision: use Mongo"), + ev( + "tj-cx", + EventType::Rejection, + "Wrong rejection: SQLite too slow", + ), + ev("tj-cx", EventType::Evidence, "Wrong evidence: bench 5ms"), + ev("tj-cx", EventType::Finding, "Wrong finding: migration done"), + ]; + for w in &wrong { + put(&conn, w); + let mut c = ev("tj-cx", EventType::Correction, "That was a mistake"); + c.corrects = Some(w.event_id.clone()); + put(&conn, &c); + } + put(&conn, &ev("tj-cx", EventType::Decision, "Use SQLite")); + + let pack = assemble(&conn, "tj-cx", PackMode::Full).unwrap(); + assert!(!pack.text.contains("Wrong "), "{}", pack.text); + assert!(pack.text.contains("Use SQLite"), "{}", pack.text); + assert_eq!( + pack.text.matches("[correction] That was a mistake").count(), + 4, + "{}", + pack.text + ); + } + + #[test] + fn correcting_another_tasks_event_refreshes_that_tasks_cached_pack() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-a", "A"); + open_task(&conn, "tj-b", "B"); + let wrong = ev("tj-a", EventType::Decision, "Wrong decision in A"); + put(&conn, &wrong); + assert!(assemble(&conn, "tj-a", PackMode::Full) + .unwrap() + .text + .contains("Wrong decision in A")); + + let mut c = ev("tj-b", EventType::Correction, "A's decision was wrong"); + c.corrects = Some(wrong.event_id.clone()); + put(&conn, &c); + + let pack = assemble(&conn, "tj-a", PackMode::Full).unwrap(); + assert!(!pack.text.contains("Wrong decision in A"), "{}", pack.text); + } } From d25552d402c754d99709d7003f483e6d7aa5df88 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:47:50 +0400 Subject: [PATCH 069/104] fix(pack): mark suggested decisions and rejections as unconfirmed --- crates/tj-core/src/pack.rs | 77 +++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index d3f98f4..29128dd 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -104,27 +104,37 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result { Ok(out) } +/// A classifier's guess (`status = suggested`) must not read like a choice +/// the agent made: such decisions and rejections stay listed but marked. +fn unconfirmed_marker(status: &str) -> &'static str { + if status == "suggested" { + " _(unconfirmed)_" + } else { + "" + } +} + fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result { let mut out = String::from("## Rejected\n"); // v0.10.3: newest-first so end-of-pack truncation drops the // OLDEST rejections, not the latest decision the agent recorded. let mut id_stmt = conn.prepare( - "SELECT event_id FROM events_index + "SELECT event_id, status FROM events_index WHERE task_id=?1 AND type='rejection' AND corrected_by IS NULL ORDER BY timestamp DESC", )?; let mut text_stmt = conn.prepare("SELECT text FROM search_fts WHERE event_id=?1 LIMIT 1")?; - let event_ids: Vec = id_stmt - .query_map(rusqlite::params![task_id], |r| r.get::<_, String>(0))? + let event_ids: Vec<(String, String)> = id_stmt + .query_map(rusqlite::params![task_id], |r| Ok((r.get(0)?, r.get(1)?)))? .collect::>()?; let mut count = 0; let mut seen: HashSet = HashSet::new(); - for eid in event_ids { + for (eid, status) in event_ids { let text: String = text_stmt.query_row(rusqlite::params![eid], |r| r.get(0))?; if is_noise(&text) || !seen.insert(text.trim().to_string()) { continue; } - out.push_str(&format!("- {text}\n")); + out.push_str(&format!("- {text}{}\n", unconfirmed_marker(&status))); count += 1; } if count == 0 { @@ -141,24 +151,28 @@ fn render_active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result(0)?, r.get::<_, Option>(1)?)) + Ok(( + r.get::<_, String>(0)?, + r.get::<_, Option>(1)?, + r.get::<_, String>(2)?, + )) })?; let mut count = 0; let mut seen: HashSet = HashSet::new(); for row in rows { - let (text, alternatives) = row?; + let (text, alternatives, status) = row?; // Skip machine noise (compaction markers) and exact duplicates so the // section reads as crisp decisions, not repeated essays. if is_noise(&text) || !seen.insert(text.trim().to_string()) { continue; } - out.push_str(&format!("- {text}\n")); + out.push_str(&format!("- {text}{}\n", unconfirmed_marker(&status))); // v0.12.0: structured alternatives render under the decision so the // pack shows "considered A/B/C, chose X" without reconstructing it // from the hypothesis+rejection chain. @@ -1297,4 +1311,49 @@ mod tests { let pack = assemble(&conn, "tj-a", PackMode::Full).unwrap(); assert!(!pack.text.contains("Wrong decision in A"), "{}", pack.text); } + + /// The body of the `## {heading}` section, up to the next `## ` heading. + fn section<'a>(text: &'a str, heading: &str) -> &'a str { + let start = text + .find(&format!("## {heading}")) + .unwrap_or_else(|| panic!("no {heading} section in:\n{text}")); + let end = text[start + 3..] + .find("\n## ") + .map_or(text.len(), |i| start + 3 + i); + &text[start..end] + } + + #[test] + fn suggested_decisions_and_rejections_are_marked_unconfirmed() { + use crate::event::{EventStatus, EventType}; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-un", "Unconfirmed"); + let mut guess = ev("tj-un", EventType::Decision, "Maybe adopt Rust"); + guess.status = EventStatus::Suggested; + put(&conn, &guess); + let mut guess_rej = ev("tj-un", EventType::Rejection, "Maybe drop TypeScript"); + guess_rej.status = EventStatus::Suggested; + put(&conn, &guess_rej); + put(&conn, &ev("tj-un", EventType::Decision, "Use SQLite")); + put( + &conn, + &ev("tj-un", EventType::Rejection, "Postgres: too heavy"), + ); + + let pack = assemble(&conn, "tj-un", PackMode::Full).unwrap(); + let active = section(&pack.text, "Active decisions"); + assert!( + active.contains("- Maybe adopt Rust _(unconfirmed)_\n"), + "{active}" + ); + assert!(active.contains("- Use SQLite\n"), "{active}"); + let rejected = section(&pack.text, "Rejected"); + assert!( + rejected.contains("- Maybe drop TypeScript _(unconfirmed)_\n"), + "{rejected}" + ); + assert!(rejected.contains("- Postgres: too heavy\n"), "{rejected}"); + } } From 773c8e8ef5fcbdb81429d932be3031dd68bc323a Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:48:38 +0400 Subject: [PATCH 070/104] fix(plugin): mod re-reads state when the session id moves and distills without a fork --- plugin/hooks/journal.test.ts | 24 +++++++++++++++++ plugin/hooks/journal.ts | 30 ++++++++++++++++++++++ plugin/hooks/register.ts | 50 ++++++++++++++++++++++++++++-------- 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/plugin/hooks/journal.test.ts b/plugin/hooks/journal.test.ts index 28c9e66..1c269b6 100644 --- a/plugin/hooks/journal.test.ts +++ b/plugin/hooks/journal.test.ts @@ -7,6 +7,7 @@ import { parseState, sectionText, statusText, + transcriptExcerpt, type JournalState, } from './journal' @@ -115,3 +116,26 @@ describe('parseDistill', () => { expect(parseDistill('[not json', known)).toEqual([]) }) }) + +describe('transcriptExcerpt', () => { + const messages = [ + { role: 'user', text: 'first question' }, + { role: 'assistant', text: '' }, + { role: 'assistant', text: 'first answer' }, + { role: 'user', text: 'second question' }, + ] + + test('keeps the turns in order and skips empty ones', () => { + expect(transcriptExcerpt(messages, 1000)).toBe('user: first question\nassistant: first answer\nuser: second question') + }) + + test('keeps the newest turns when it has to cut', () => { + expect(transcriptExcerpt(messages, 50)).toBe('assistant: first answer\nuser: second question') + }) + + test('cuts a single huge turn from its start, keeping its end', () => { + const out = transcriptExcerpt([{ role: 'user', text: 'x'.repeat(100) + 'END' }], 20) + expect(out.length).toBe(20) + expect(out.endsWith('END')).toBe(true) + }) +}) diff --git a/plugin/hooks/journal.ts b/plugin/hooks/journal.ts index 7e4073d..64f7b9f 100644 --- a/plugin/hooks/journal.ts +++ b/plugin/hooks/journal.ts @@ -127,11 +127,41 @@ export function distillPrompt(task: ActiveTask): string { known, '', 'From the conversation above, list the decisions, rejections, findings, evidence, constraints and hypotheses that matter for this task and are NOT already recorded.', + 'Do not restate an entry already recorded in other words, and do not turn a recorded decision into a hypothesis or finding.', `Each one sentence, specific (names, files, numbers). At most ${MAX_DISTILLED}. Nothing worth adding → [].`, 'Reply with a JSON array only: [{"type": "decision", "text": "..."}]. Allowed types: decision, rejection, finding, evidence, constraint, hypothesis.', ].join('\n') } +export type TranscriptLine = { role: string; text: string } + +/** + * The newest turns of a conversation as `role: text` lines, cut to + * `maxChars` from the end: what a model reads when it cannot fork the + * session's own cached transcript. + */ +export function transcriptExcerpt(messages: readonly TranscriptLine[], maxChars: number): string { + const lines: string[] = [] + let size = 0 + + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + const text = m?.text.trim() ?? '' + if (m === undefined || text === '') continue + + const line = `${m.role}: ${text}` + if (size + line.length > maxChars) { + if (lines.length === 0) lines.push(line.slice(line.length - maxChars)) + break + } + + lines.push(line) + size += line.length + 1 + } + + return lines.reverse().join('\n') +} + function normalized(text: string): string { return text.toLowerCase().replace(/\s+/g, ' ').trim() } diff --git a/plugin/hooks/register.ts b/plugin/hooks/register.ts index f35c2ca..2ef8c03 100644 --- a/plugin/hooks/register.ts +++ b/plugin/hooks/register.ts @@ -25,12 +25,18 @@ import { parseState, sectionText, statusText, + transcriptExcerpt, WRITE_TOOLS, type JournalState, + type TranscriptLine, } from './journal' const WORK_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'Bash']) +// How much of the conversation the no-fork distill reads: the tail that +// fits a small model's prompt comfortably. +const EXCERPT_CHARS = 60_000 + // What the mod knows about the session it runs in. A module variable: it // lives as long as the process, which is as long as the session. const mod = { @@ -55,13 +61,19 @@ async function cli($: EngineInterface, argv: string[]): Promise { } } +// Reads the state of the session the engine names now. The id it was read +// for is kept even when the read fails, so a missing CLI costs one spawn +// per session, not one per request. async function refresh($: EngineInterface): Promise { const session = await $.session.id() const out = await cli($, ['state', '--session', session]) const state = out === null ? null : parseState(out) + mod.session = session + mod.state = state + if (state === null) { - if (mod.state === null && !mod.hasWarned) { + if (!mod.hasWarned) { mod.hasWarned = true $.ui.log( 'task-journal: the task-journal CLI 0.30+ was not found on PATH, so the journal mod is off. Install it with `cargo install task-journal-cli task-journal-mcp --force`.', @@ -71,24 +83,37 @@ async function refresh($: EngineInterface): Promise { return } - mod.state = state - mod.session = session $.ui.status(statusText(state)) } -// A /clear ends the session without a new session.start: catch up on the -// first prompt under the new id. +// The session id can change under the mod: a /clear starts a new one with +// no session.start, and a resumed session may only take its id after +// session.start ran. Re-read whenever the id moved. async function current($: EngineInterface): Promise { - if (mod.state === null || mod.session !== (await $.session.id())) await refresh($) + if (mod.session !== (await $.session.id())) await refresh($) return mod.state } -async function distill($: EngineInterface, state: JournalState): Promise { +// Asks what the conversation decided that the journal lacks. A fork reuses +// the session's own cached transcript; a process that has sent nothing yet +// (a session resumed straight into /compact) has nothing to fork, and then +// the messages being compacted go to a small model instead. +async function distill($: EngineInterface, state: JournalState, messages: readonly TranscriptLine[]): Promise { const task = state.active if (task === null) return - const reply = await $.model.fork({ prompt: distillPrompt(task) }) + const ask = distillPrompt(task) + let reply = await $.model.fork({ prompt: ask }) + if (!reply.isAnswered && reply.reason === 'nothing-to-fork') { + const conversation = transcriptExcerpt(messages, EXCERPT_CHARS) + reply = await $.model.complete({ + model: 'haiku', + prompt: `\n${conversation}\n\n\n${ask}`, + maxTokens: 1024, + timeoutMs: 60_000, + }) + } if (!reply.isAnswered) return const session = await $.session.id() @@ -134,10 +159,13 @@ export const register: Register = (on, options) => { on('prompt.compose', async ($, e, next) => { const composed = await next(e) - if (mod.isOff || mod.state === null) return composed + if (mod.isOff) return composed + + const state = await current($) + if (state === null) return composed return { - sections: [...composed.sections, { id: 'task-journal:active', text: sectionText(mod.state), scope: 'session' }], + sections: [...composed.sections, { id: 'task-journal:active', text: sectionText(state), scope: 'session' }], } }) @@ -191,7 +219,7 @@ export const register: Register = (on, options) => { if (distillOnCompact) { try { - await distill($, state) + await distill($, state, e.messages) } catch { // The compaction matters more than the catch-up: go on without it. } From 2478e397836886fbc0953551aa35204cfa7faf92 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:51:18 +0400 Subject: [PATCH 071/104] fix(core): keep compaction markers and model switches out of decisions, export-pr, recall and global memory --- crates/tj-cli/src/main.rs | 5 +- crates/tj-cli/tests/cli.rs | 5 ++ crates/tj-core/src/completeness.rs | 24 ++++++++- crates/tj-core/src/db.rs | 80 ++++++++++++++++++++++++++++-- crates/tj-core/src/memory.rs | 78 ++++++++++++++++++++++++++++- crates/tj-core/src/pack.rs | 20 ++++++++ crates/tj-core/src/recall.rs | 25 ++++++++++ 7 files changed, 230 insertions(+), 7 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index ce9eca2..b8add89 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2774,6 +2774,9 @@ runs in the background and won't block you; it only fills gaps and never closes ); event.confidence = Some(1.0); event.status = tj_core::event::EventStatus::Confirmed; + // Bookkeeping, not a decision: kept out of active decisions, + // export-pr, recall and global memory — see db::is_bookkeeping. + event.meta = serde_json::json!({ "kind": "compaction_marker" }); tj_core::session_id::stamp_session_id(&mut event.meta, live_session_id.as_deref()); let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; writer.append(&event)?; @@ -3959,7 +3962,7 @@ fn run_export_pr(task_id: &str) -> Result<()> { let mut stmt = conn.prepare( "SELECT ei.type, sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 AND ei.corrected_by IS NULL + WHERE ei.task_id = ?1 AND ei.corrected_by IS NULL AND ei.bookkeeping = 0 ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index bda921b..6713724 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -4368,6 +4368,7 @@ fn precompact_hook_appends_marker_decision_to_open_task() { .unwrap_or("") .contains("Conversation compacted at") { + assert_eq!(v["meta"]["kind"], "compaction_marker", "{v}"); marker_lines += 1; } } @@ -4391,6 +4392,10 @@ fn precompact_hook_appends_marker_decision_to_open_task() { .not() .and(contains("single reasoning unit").not()), ); + + // It is bookkeeping, not a change the PR made. + let pr = tj_stdout(dir.path(), &workdir, &["export-pr", &task_id]); + assert!(!pr.contains("Conversation compacted at"), "{pr}"); } #[test] diff --git a/crates/tj-core/src/completeness.rs b/crates/tj-core/src/completeness.rs index dc327d8..fdcebc8 100644 --- a/crates/tj-core/src/completeness.rs +++ b/crates/tj-core/src/completeness.rs @@ -109,7 +109,10 @@ pub fn assess( let mut evidence = 0usize; let mut suggested = 0usize; { - let mut stmt = conn.prepare("SELECT type, status FROM events_index WHERE task_id = ?1")?; + // Bookkeeping (compaction markers) is not a decision to verify. + let mut stmt = conn.prepare( + "SELECT type, status FROM events_index WHERE task_id = ?1 AND bookkeeping = 0", + )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) })?; @@ -439,6 +442,25 @@ mod tests { .any(|g| g.kind == GapKind::DecisionNoEvidence)); } + #[test] + fn compaction_marker_alone_is_not_an_unverified_decision() { + let (_d, c) = conn(); + open_task(&c, "t3m"); + c.execute("UPDATE tasks SET goal='g' WHERE task_id='t3m'", []) + .unwrap(); + let marker = Event::new( + "t3m", + EventType::Decision, + Author::Classifier, + Source::Hook, + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events…".into(), + ); + crate::db::index_event(&c, &marker).unwrap(); + + let r = assess(&c, "t3m", 0).unwrap(); + assert!(r.is_complete(), "{:?}", r.gaps); + } + #[test] fn suggested_unconfirmed_counts() { use crate::event::EventStatus; diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 521c098..23396fb 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -169,6 +169,15 @@ ALTER TABLE events_index ADD COLUMN corrected_by TEXT; DELETE FROM index_state; "#; +/// v0.30.0 bookkeeping — `bookkeeping` flags machine-written events that are +/// not reasoning (see [`is_bookkeeping`]) so active decisions, export-pr, +/// recall and the global memory can skip them. Clearing `index_state` +/// replays the log once to flag existing events. +const MIGRATION_011: &str = r#" +ALTER TABLE events_index ADD COLUMN bookkeeping INTEGER NOT NULL DEFAULT 0; +DELETE FROM index_state; +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -212,6 +221,10 @@ const MIGRATIONS: &[Migration] = &[ version: 10, sql: MIGRATION_010, }, + Migration { + version: 11, + sql: MIGRATION_011, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -975,6 +988,22 @@ pub fn ingest_new_events( Ok(count) } +/// Machine-written bookkeeping, not reasoning: the PreCompact boundary marker +/// (a `decision`) and the model-switch note (a `constraint`). Known by +/// `meta.kind`; events written before the writers set it, by text prefix. +pub fn is_bookkeeping(event: &Event) -> bool { + match event.meta.get("kind").and_then(|v| v.as_str()) { + Some(kind) => matches!(kind, "compaction_marker" | "model_switch"), + None => match event.event_type { + EventType::Decision => event.text.starts_with("Conversation compacted at"), + EventType::Constraint => event + .text + .starts_with(crate::reminder::MODEL_SWITCH_TEXT_PREFIX), + _ => false, + }, + } +} + pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { // An amend only changes task metadata, which upsert_task_from_event // applies. It is not reasoning: keeping it out of events_index and @@ -1015,11 +1044,12 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { }; let session_id = event.meta.get("session_id").and_then(|v| v.as_str()); conn.execute( - "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id, bookkeeping) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", rusqlite::params![ event.event_id, event.task_id, type_str, - event.timestamp, event.confidence, status_str, artifacts_json, session_id + event.timestamp, event.confidence, status_str, artifacts_json, session_id, + is_bookkeeping(event) ], )?; // search_fts has no PK; clear then insert to keep idempotent across rebuild_state replays. @@ -2595,6 +2625,50 @@ mod tests { assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); } + #[test] + fn upgrading_an_existing_db_flags_old_bookkeeping_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + + // Written before the writers set meta.kind: known by text alone. + let open_ev = make_open_event("tj-x", "Upgrade"); + let mut marker = make_text_event("Conversation compacted at 2026-01-01T00:00:00Z; …"); + marker.event_type = crate::event::EventType::Decision; + let mut switch = make_text_event("Model switched (user): opus → sonnet"); + switch.event_type = crate::event::EventType::Constraint; + let real = make_text_event("Use SQLite"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &marker, &switch, &real] { + write_event_line(&mut f, e); + } + drop(f); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "ALTER TABLE events_index DROP COLUMN bookkeeping; + DELETE FROM schema_migrations WHERE version = 11;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + let flagged: Vec = conn + .prepare("SELECT event_id FROM events_index WHERE bookkeeping = 1 ORDER BY event_id") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .collect::>() + .unwrap(); + let mut expected = vec![marker.event_id.clone(), switch.event_id.clone()]; + expected.sort(); + assert_eq!(flagged, expected); + } + #[test] fn open_meta_goal_and_external_are_restored_by_a_rebuild() { let d = TempDir::new().unwrap(); diff --git a/crates/tj-core/src/memory.rs b/crates/tj-core/src/memory.rs index 395831f..0aa2229 100644 --- a/crates/tj-core/src/memory.rs +++ b/crates/tj-core/src/memory.rs @@ -85,10 +85,12 @@ pub fn sync_from_project( .join(","); let sql = format!( "SELECT e.event_id, e.task_id, f.type, e.tier, f.text, e.model, e.dim, e.vec, e.created_at, - CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END, + COALESCE(ei.bookkeeping, 0) FROM embeddings e JOIN search_fts f ON f.event_id = e.event_id LEFT JOIN decisions d ON d.decision_id = e.event_id + LEFT JOIN events_index ei ON ei.event_id = e.event_id WHERE f.type IN ({placeholders})" ); let mut stmt = project.prepare(&sql)?; @@ -104,12 +106,38 @@ pub fn sync_from_project( r.get::<_, Vec>(7)?, // vec r.get::<_, String>(8)?, // created_at r.get::<_, i64>(9)?, // superseded + r.get::<_, bool>(10)?, // bookkeeping )) })?; let mut n = 0usize; for row in rows { - let (event_id, task_id, ty, tier, text, model, dim, vec, created_at, superseded) = row?; + let ( + event_id, + task_id, + ty, + tier, + text, + model, + dim, + vec, + created_at, + superseded, + bookkeeping, + ) = row?; + // Compaction markers / model switches are not reasoning to recall; + // also drop any copy an older version already synced. + if bookkeeping { + global.execute( + "DELETE FROM global_memory WHERE event_id = ?1", + rusqlite::params![event_id], + )?; + global.execute( + "DELETE FROM global_fts WHERE event_id = ?1", + rusqlite::params![event_id], + )?; + continue; + } global.execute( "INSERT OR REPLACE INTO global_memory (event_id, project_hash, task_id, type, tier, text, model, dim, vec, created_at, superseded) @@ -346,6 +374,52 @@ mod tests { .is_empty()); } + #[test] + fn sync_leaves_out_and_drops_bookkeeping_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + // An old marker, recognised by its text alone. + let marker = finding( + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events should be treated as a single reasoning unit.", + ); + let mut switch = finding("Model switched (user): opus → sonnet"); + switch.event_type = crate::event::EventType::Constraint; + switch.meta = serde_json::json!({ "kind": "model_switch" }); + for e in [ + &marker, + &switch, + &finding("chose the idempotent payment ledger"), + ] { + crate::db::index_event(&proj, e).unwrap(); + } + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + // A marker an older version already copied into the global index. + global + .execute( + "INSERT INTO global_memory + (event_id, project_hash, task_id, type, text, model, dim, vec, created_at) + VALUES (?1, 'ph', 'tj-x', 'decision', ?2, 't', 0, x'', '')", + rusqlite::params![marker.event_id, marker.text], + ) + .unwrap(); + global + .execute( + "INSERT INTO global_fts(event_id, text) VALUES (?1, ?2)", + rusqlite::params![marker.event_id, marker.text], + ) + .unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 1); + assert_eq!(count(&global).unwrap(), 1); + assert!(keyword_search(&global, "compacted reasoning switched", 5) + .unwrap() + .is_empty()); + } + #[test] fn preferences_store_dedup_and_list_in_order() { let d = tempfile::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index 29128dd..cab1bce 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -154,6 +154,7 @@ fn render_active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result Date: Sun, 4 Oct 2026 13:53:32 +0400 Subject: [PATCH 072/104] feat(mcp): take a Codex session id from the request _meta --- crates/tj-core/src/session_id.rs | 39 ++++++++++++++ crates/tj-mcp/src/main.rs | 88 ++++++++++++++++++++++++++++++-- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/crates/tj-core/src/session_id.rs b/crates/tj-core/src/session_id.rs index ee3710d..d8942e6 100644 --- a/crates/tj-core/src/session_id.rs +++ b/crates/tj-core/src/session_id.rs @@ -25,6 +25,20 @@ pub fn session_id_from_payload(payload: &Value) -> Option { .map(str::to_string) } +/// Pull the session id out of an MCP request's `_meta`. Codex sends it on +/// every `tools/call` as `x-codex-turn-metadata.session_id` (and the same +/// value as `thread_id`); its MCP servers get no session env var, so this is +/// the only way a Codex event learns its session. Empty counts as absent. +pub fn session_id_from_mcp_meta(meta: &serde_json::Map) -> Option { + let turn = meta.get("x-codex-turn-metadata")?; + + ["session_id", "thread_id"] + .iter() + .filter_map(|key| turn.get(*key).and_then(|v| v.as_str())) + .find(|s| !s.is_empty()) + .map(str::to_string) +} + /// Read the live session id from the environment: `CLAUDE_CODE_SESSION_ID`, /// then Codex's `CODEX_THREAD_ID`, then `CODEX_SESSION_ID`. Empty counts as /// absent. @@ -68,6 +82,31 @@ mod tests { assert_eq!(session_id_from_payload(&p).as_deref(), Some("abc-123")); } + #[test] + fn mcp_meta_reads_codex_turn_metadata() { + let meta = json!({ + "callId": "exec-1", + "x-codex-turn-metadata": { "session_id": "c-1", "thread_id": "c-1", "model": "x" } + }); + let meta = meta.as_object().unwrap(); + assert_eq!(session_id_from_mcp_meta(meta).as_deref(), Some("c-1")); + + let thread_only = + json!({ "x-codex-turn-metadata": { "session_id": "", "thread_id": "t-2" } }); + assert_eq!( + session_id_from_mcp_meta(thread_only.as_object().unwrap()).as_deref(), + Some("t-2") + ); + } + + #[test] + fn mcp_meta_without_codex_metadata_is_none() { + // Claude Code sends only its tool-use id. + let claude = json!({ "claudecode/toolUseId": "toolu_1", "progressToken": 2 }); + assert_eq!(session_id_from_mcp_meta(claude.as_object().unwrap()), None); + assert_eq!(session_id_from_mcp_meta(&serde_json::Map::new()), None); + } + #[test] fn payload_empty_or_missing_is_none() { assert_eq!(session_id_from_payload(&json!({"session_id": ""})), None); diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index db1ca7f..331c224 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -5,8 +5,8 @@ use anyhow::{Context, Result}; use clap::Parser; use rmcp::{ - handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_handler, - tool_router, transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, + handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_router, + transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, }; use rusqlite::Connection; use serde::{Deserialize, Serialize}; @@ -955,7 +955,8 @@ impl TaskJournalServer { } } -#[tool_handler(router = Self::tool_router())] +// Written out instead of `#[tool_handler]`, which generates the same two +// methods, so `call_tool` can read the request's `_meta` first. impl ServerHandler for TaskJournalServer { fn get_info(&self) -> rmcp::model::ServerInfo { rmcp::model::ServerInfo { @@ -970,6 +971,49 @@ impl ServerHandler for TaskJournalServer { ..Default::default() } } + + async fn call_tool( + &self, + mut request: rmcp::model::CallToolRequestParam, + context: rmcp::service::RequestContext, + ) -> Result { + stamp_client_session(&mut request, &context.meta); + + let tcc = rmcp::handler::server::tool::ToolCallContext::new(self, request, context); + Self::tool_router().call(tcc).await + } + + async fn list_tools( + &self, + _request: Option, + _context: rmcp::service::RequestContext, + ) -> Result { + Ok(rmcp::model::ListToolsResult::with_all_items( + Self::tool_router().list_all(), + )) + } +} + +/// Codex names the session in each call's `_meta`, never in the MCP +/// server's environment. Carry it into the `session_id` argument of the +/// tools that stamp one, unless the caller passed its own. +fn stamp_client_session(request: &mut rmcp::model::CallToolRequestParam, meta: &rmcp::model::Meta) { + const STAMPING: [&str; 4] = ["task_create", "event_add", "artifact_add", "task_close"]; + if !STAMPING.contains(&request.name.as_ref()) { + return; + } + let Some(session) = tj_core::session_id::session_id_from_mcp_meta(&meta.0) else { + return; + }; + + let args = request.arguments.get_or_insert_with(Default::default); + let has_own = args + .get("session_id") + .and_then(|v| v.as_str()) + .is_some_and(|s| !s.is_empty()); + if !has_own { + args.insert("session_id".into(), serde_json::Value::String(session)); + } } /// Resolve when the process should shut down: Ctrl-C on every platform, @@ -1042,6 +1086,44 @@ mod tests { use super::*; + fn call(name: &'static str, args: serde_json::Value) -> rmcp::model::CallToolRequestParam { + rmcp::model::CallToolRequestParam { + name: name.into(), + arguments: args.as_object().cloned(), + } + } + + fn codex_meta(session: &str) -> rmcp::model::Meta { + let meta = serde_json::json!({ "x-codex-turn-metadata": { "session_id": session } }); + rmcp::model::Meta(meta.as_object().unwrap().clone()) + } + + #[test] + fn codex_session_from_meta_reaches_the_stamping_tools() { + let mut add = call("event_add", serde_json::json!({ "task_id": "tj-1" })); + stamp_client_session(&mut add, &codex_meta("c-1")); + assert_eq!(add.arguments.unwrap()["session_id"], "c-1"); + + let mut create = call("task_create", serde_json::json!(null)); + stamp_client_session(&mut create, &codex_meta("c-1")); + assert_eq!(create.arguments.unwrap()["session_id"], "c-1"); + } + + #[test] + fn an_explicit_session_id_and_other_tools_are_left_alone() { + let mut own = call("event_add", serde_json::json!({ "session_id": "mine" })); + stamp_client_session(&mut own, &codex_meta("c-1")); + assert_eq!(own.arguments.unwrap()["session_id"], "mine"); + + let mut search = call("task_search", serde_json::json!({ "query": "x" })); + stamp_client_session(&mut search, &codex_meta("c-1")); + assert!(search.arguments.unwrap().get("session_id").is_none()); + + let mut claude = call("event_add", serde_json::json!({ "task_id": "tj-1" })); + stamp_client_session(&mut claude, &rmcp::model::Meta::default()); + assert!(claude.arguments.unwrap().get("session_id").is_none()); + } + /// Handler tests touch process-global state (PROJECT_DIR_OVERRIDE OnceLock /// and the XDG_DATA_HOME env var), so they must run one at a time and share /// a single project dir. This guard serializes them and lazily pins the From ebebb366da3f2a054c3bbed1b1af0bc15a7e038d Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:54:22 +0400 Subject: [PATCH 073/104] fix(pack): reserve room for the gap summary so truncation never cuts it --- crates/tj-core/src/pack.rs | 101 +++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index cab1bce..3d07e25 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -269,6 +269,29 @@ fn render_subtasks(conn: &Connection, task_id: &str) -> anyhow::Result Option { + const GAP_LINE_MAX: usize = 400; + + if report.gaps.is_empty() { + return None; + } + let details = report + .gaps + .iter() + .map(|g| g.detail.as_str()) + .collect::>() + .join("; "); + let mut line = format!("- ⚠ honesty score: {}/100 — {details}", report.score()); + truncate_to_budget(&mut line, GAP_LINE_MAX, "…"); + + Some(format!( + "\n## Completeness ({})\n{line}\n", + report.gaps.len() + )) +} + fn truncate_to_budget(text: &mut String, budget: usize, marker: &str) { if text.len() <= budget { return; @@ -448,6 +471,13 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res }; text.push_str(&render_recent_events(conn, task_id, recent_limit)?); + // One-level roll-up of direct children (parents only). Appended before + // truncation so it shares the pack budget. Task 5 busts the parent cache + // when a child changes, so the next assemble regenerates fresh. + if let Some(subtasks) = render_subtasks(conn, task_id)? { + text.push_str(&subtasks); + } + let mut report = crate::completeness::assess(conn, task_id, crate::completeness::pending_count())?; // Honesty drift: flag artifacts (files/commits/local links) that no longer @@ -455,16 +485,13 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res report .gaps .extend(crate::completeness::artifact_gaps_for_cwd(&arts)); - if let Some(section) = crate::completeness::render_section(&report) { - text.push_str(§ion); - } - - // One-level roll-up of direct children (parents only). Appended before - // truncation so it shares the pack budget. Task 5 busts the parent cache - // when a child changes, so the next assemble regenerates fresh. - if let Some(subtasks) = render_subtasks(conn, task_id)? { - text.push_str(&subtasks); + // The gaps tell the next agent what is missing, so they go last with + // their room reserved: truncation cuts the reasoning above, never them. + let gaps = match mode { + PackMode::Full => crate::completeness::render_section(&report), + PackMode::Compact => render_gap_line(&report), } + .unwrap_or_default(); // Token-budget truncation: cap pack size so it always fits an LLM context window. // v0.10.3: full bumped 10K → 24K → 32K. Real tasks accumulate 50-100 events @@ -478,10 +505,11 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res PackMode::Full => FULL_BUDGET, PackMode::Compact => COMPACT_BUDGET, }; - let truncated = text.len() > budget; + let truncated = text.len() + gaps.len() > budget; if truncated { - truncate_to_budget(&mut text, budget, TRUNC_MARKER); + truncate_to_budget(&mut text, budget.saturating_sub(gaps.len()), TRUNC_MARKER); } + text.push_str(&gaps); let generated_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); @@ -1376,4 +1404,55 @@ mod tests { assert!(!active.contains("Context boundary"), "{active}"); assert!(active.contains("Use SQLite"), "{active}"); } + + #[test] + fn truncated_packs_keep_the_gap_summary() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-gap", "Gaps"); // no goal → a gap + for i in 0..60 { + let long = format!("Decision #{i}: {}", "lorem ipsum ".repeat(60)); + put(&conn, &ev("tj-gap", EventType::Decision, &long)); + } + + for mode in [PackMode::Compact, PackMode::Full] { + let pack = assemble(&conn, "tj-gap", mode).unwrap(); + assert!(pack.metadata.truncated, "{mode:?}"); + let gaps = section(&pack.text, "Completeness"); + assert!(gaps.contains("no goal recorded"), "{mode:?}: {gaps}"); + } + + let compact = assemble(&conn, "tj-gap", PackMode::Compact).unwrap(); + assert!( + compact.text.len() <= 2 * 1024 + 64, + "{}", + compact.text.len() + ); + let gaps = section(&compact.text, "Completeness"); + assert_eq!( + gaps.trim_end().lines().count(), + 2, + "one summary line: {gaps}" + ); + } + + #[test] + fn compact_gap_line_stays_short_with_many_gaps() { + use crate::completeness::{CompletenessReport, Gap, GapKind}; + + let report = CompletenessReport { + gaps: (0..50) + .map(|i| Gap { + kind: GapKind::MissingFile, + detail: format!("referenced file no longer exists: src/deep/path/file_{i}.rs"), + }) + .collect(), + }; + let line = render_gap_line(&report).unwrap(); + assert!(line.len() <= 512, "{} bytes: {line}", line.len()); + assert!(line.contains("## Completeness (50)"), "{line}"); + assert!(line.contains("honesty score: 0/100"), "{line}"); + } } From 8db72e3ed2759861263405244a0f83e5ceb6764b Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:55:17 +0400 Subject: [PATCH 074/104] fix(hooks): put the session's own task first for reminders and classification --- crates/tj-cli/src/main.rs | 91 +++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 9644563..7531940 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2576,7 +2576,8 @@ fn real_main() -> Result<()> { tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 3)?; + let recent = + recent_task_contexts(&conn, &project_hash, 3, live_session_id.as_deref())?; if recent.is_empty() { if !prefs_block.is_empty() { emit_session_context(&prefs_block); @@ -2690,7 +2691,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2751,7 +2753,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2888,7 +2891,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -2975,7 +2979,8 @@ runs in the background and won't block you; it only fills gaps and never closes tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - let recent = recent_task_contexts(&conn, 1)?; + let recent = + recent_task_contexts(&conn, &project_hash, 1, live_session_id.as_deref())?; let Some(tc) = recent.into_iter().next() else { return Ok(()); }; @@ -3063,7 +3068,8 @@ runs in the background and won't block you; it only fills gaps and never closes if events_path.exists() { tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } - let mut recent = recent_task_contexts(&conn, 5)?; + let mut recent = + recent_task_contexts(&conn, &project_hash, 5, live_session_id.as_deref())?; if recent.is_empty() { // No open tasks. v0.5.0 Phase A: auto-open a new // task from the user's prompt so subsequent @@ -4326,15 +4332,26 @@ fn run_export_memory(task: Option<&str>, _all_closed: bool, dry_run: bool) -> Re /// the classifier prompt. Kept small so the prompt stays bounded. const CONSTRAINT_CONTEXT_LIMIT: i64 = 5; +/// The open tasks a hook works with, newest first — except that the task +/// this session last wrote to comes first, so two sessions in one project +/// each see (and classify into) their own task, not just the newest one. fn recent_task_contexts( conn: &rusqlite::Connection, + project_hash: &str, limit: usize, + session: Option<&str>, ) -> anyhow::Result> { + let own = match session { + Some(sid) => tj_core::db::active_task_for_session(conn, project_hash, sid)?, + None => None, + }; + let mut stmt = conn.prepare( - "SELECT task_id, title FROM tasks WHERE status='open' ORDER BY last_event_at DESC LIMIT ?1", + "SELECT task_id, title FROM tasks WHERE status='open' + ORDER BY (task_id = ?2) DESC, last_event_at DESC LIMIT ?1", )?; let task_rows: Vec<(String, String)> = stmt - .query_map(rusqlite::params![limit as i64], |r| { + .query_map(rusqlite::params![limit as i64, own], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) })? .collect::>()?; @@ -4536,7 +4553,10 @@ fn run_session_end_catchup( let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; - let Some(tc) = recent_task_contexts(&conn, 1)?.into_iter().next() else { + let Some(tc) = recent_task_contexts(&conn, project_hash, 1, live_session_id)? + .into_iter() + .next() + else { return Ok(()); }; let last_event_ts: Option = conn @@ -6001,7 +6021,7 @@ fn classify_chunk( tj_core::db::ingest_new_events(&conn, events_path, project_hash)?; } - let mut recent = recent_task_contexts(&conn, 5)?; + let mut recent = recent_task_contexts(&conn, project_hash, 5, session_id)?; if recent.is_empty() { let auto_open_disabled = std::env::var("TJ_AUTO_OPEN_TASKS") .ok() @@ -6895,6 +6915,51 @@ mod inline_tests { assert!(!is_rewind_prompt("/rewinder")); } + #[test] + fn recent_task_contexts_puts_the_sessions_own_task_first() { + use tj_core::event::{Author, Event, EventType, Source}; + let dir = tempfile::tempdir().unwrap(); + let events_path = dir.path().join("events").join("h.jsonl"); + std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); + let state_path = dir.path().join("h.sqlite"); + + // s1 works on the older task; another session touched tj-2 later. + let mut writer = tj_core::storage::JsonlWriter::open(&events_path).unwrap(); + for (task, at, session) in [ + ("tj-1", "2026-01-01T00:00:00Z", Some("s1")), + ("tj-2", "2026-01-01T00:00:05Z", Some("s2")), + ] { + let mut open = Event::new( + task, + EventType::Open, + Author::Agent, + Source::Chat, + task.into(), + ); + open.timestamp = at.into(); + tj_core::session_id::stamp_session_id(&mut open.meta, session); + writer.append(&open).unwrap(); + } + writer.flush_durable().unwrap(); + + let conn = tj_core::db::open(&state_path).unwrap(); + tj_core::db::ingest_new_events(&conn, &events_path, "h").unwrap(); + + let first = |session| { + recent_task_contexts(&conn, "h", 2, session).unwrap()[0] + .task_id + .clone() + }; + assert_eq!(first(Some("s1")), "tj-1", "s1 sees its own task first"); + assert_eq!(first(Some("s2")), "tj-2"); + assert_eq!( + first(Some("s9")), + "tj-2", + "an unknown session falls back to the newest" + ); + assert_eq!(first(None), "tj-2"); + } + #[test] fn recent_task_contexts_gathers_constraints() { use tj_core::event::{Author, Event, EventType, Source}; @@ -6938,7 +7003,7 @@ mod inline_tests { let conn = tj_core::db::open(&state_path).unwrap(); tj_core::db::ingest_new_events(&conn, &events_path, project_hash).unwrap(); - let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); assert!( ctx.constraints @@ -6991,7 +7056,7 @@ mod inline_tests { let conn = tj_core::db::open(&state_path).unwrap(); tj_core::db::ingest_new_events(&conn, &events_path, project_hash).unwrap(); - let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); assert_eq!( ctx.constraints.len(), @@ -7054,7 +7119,7 @@ mod inline_tests { let conn = tj_core::db::open(&state_path).unwrap(); tj_core::db::ingest_new_events(&conn, &events_path, "h").unwrap(); - let ctxs = recent_task_contexts(&conn, 5).unwrap(); + let ctxs = recent_task_contexts(&conn, "h", 5, None).unwrap(); let ctx = ctxs.iter().find(|c| c.task_id == "tj-1").unwrap(); assert_eq!(ctx.constraints.len(), 5, "{:?}", ctx.constraints); assert!( From 0832b8e1a64fe83e2acf44b6b7cf5fbe40890b33 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sun, 4 Oct 2026 13:56:43 +0400 Subject: [PATCH 075/104] docs: install guide per client, the Claude Code mod, Codex tool approval and session binding --- INSTALL.md | 202 ++++++++++++++++++++++++++++++++++------------------- README.md | 35 +++++++--- 2 files changed, 155 insertions(+), 82 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 4f6293f..39b4235 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,35 +1,81 @@ # Installation +Task Journal is one Rust core (the `task-journal` CLI and the `task-journal-mcp` +server) plus thin integrations for each agent: + +| Client | What you install | What it gets | +|--------|------------------|--------------| +| Claude Code 2.1.287+ | the plugin (MCP server + skill + mod) | journal tools, the session's task kept in the system prompt, catch-up before compaction, status line | +| Claude Code, older | the plugin, optionally `install-hooks` | journal tools, resume packs on session start | +| Codex | `codex mcp add` + `install-hooks --client codex` | journal tools, resume packs, optional auto-capture | +| Any other MCP client | the MCP server | the seven journal tools | + ## Prerequisites -- **Rust toolchain 1.83+** — `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- (Optional, for auto-capture) **`ANTHROPIC_API_KEY`** exported in your shell +- **Rust 1.88+** — `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` +- No API key is needed. Self-tagging through the MCP tools is free; the optional + LLM-backed features reuse your logged-in `claude` or `codex` CLI, or + `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` if you prefer. + +## Install the binaries + +```bash +cargo install task-journal-cli task-journal-mcp +``` -## Build & install +or from a clone: ```bash -git clone -cd claude-memory -cargo build --release --workspace +git clone https://github.com/Digital-Threads/Task-Journal +cd Task-Journal cargo install --path crates/tj-cli cargo install --path crates/tj-mcp ``` -This installs two binaries to `~/.cargo/bin/`: +This installs `task-journal` (CLI) and `task-journal-mcp` (MCP server) into +`~/.cargo/bin/`. Check with `task-journal --version`. -- `task-journal` — CLI -- `task-journal-mcp` — MCP server - -Verify: +## Claude Code ```bash -task-journal --version -task-journal-mcp --help 2>/dev/null || true # MCP server speaks JSON-RPC over stdin +claude plugin install github:Digital-Threads/Task-Journal ``` -## MCP server (Claude Code) +The plugin brings the MCP server, the `task-journal` skill and, on Claude Code +2.1.287 or newer, the **Task Journal mod**. Restart Claude Code after installing. + +### The mod (Claude Code 2.1.287+) -Add to `~/.claude/settings.json`: +The mod runs inside Claude Code and needs no setup. It: + +- keeps the session's active task (id, title, goal) in the system prompt, so a + compaction can't lose it, and asks the compaction summary to keep its id; +- gives every session its own active task, even with several sessions open in + one project; +- shows the task in the status line and a short toast when an entry is recorded; +- reminds the agent to log only after several prompts without an entry, instead + of on every prompt; +- right before a compaction, asks the model which decisions, rejections and + findings were never logged and records them as `suggested` events (one extra + model call per compaction, billed to your plan, mostly served from the prompt + cache). + +Two options live in `/config` (or `claude plugin configure task-journal`): +`nudge_after_turns` (default 6, 0 turns the reminder off) and +`distill_on_compact` (default on). + +The mod calls the `task-journal` CLI, so the binaries must be installed and +current (0.30+). If they aren't, it says so once and stays off. + +**Classic hooks next to the mod.** If you ran `task-journal install-hooks` with +`--auto-capture` before, you can keep it: the mod tells those hooks it is active +(`TJ_MOD_ACTIVE=1`) and they stand down, except the session-start resume pack and +the model-switch record. To remove them anyway: +`task-journal install-hooks --scope user --uninstall`. + +### Older Claude Code, or without the plugin + +Wire the MCP server yourself in `~/.claude/settings.json`: ```json { @@ -39,59 +85,72 @@ Add to `~/.claude/settings.json`: } ``` -Restart Claude Code. The 5 tools (`task_pack`, `task_search`, `task_create`, `event_add`, `task_close`) become available to the agent. - -## Auto-capture hooks - -Install once for your user: - -```bash -export ANTHROPIC_API_KEY=sk-ant-... -task-journal install-hooks --scope user -# → /home//.claude/settings.json -``` - -Or per-project: +and, for resume packs at session start, install the hooks: ```bash -cd /path/to/project -task-journal install-hooks --scope project -# → .claude/settings.json +task-journal install-hooks --scope user # → ~/.claude/settings.json +task-journal install-hooks --scope project # → .claude/settings.json +task-journal install-hooks --scope user --auto-capture # also classify prompts and tool calls in the background ``` -The installer: +The default install adds only a read-only `SessionStart` hook (resume packs) and +a `UserPromptSubmit` reminder. `--auto-capture` adds background classification +of prompts, tool calls and session ends; it is a safety net under self-tagging, +not a replacement, and with `--backend hybrid` (the default) it may spawn +`claude -p` for chunks the local heuristic can't classify. -- Preserves all unrelated keys in `settings.json` (e.g. `theme`, other MCP servers). -- Wraps the hook command with `|| true` so a classifier failure (network down, rate limit, missing key) **never** breaks Claude Code. -- Failures land in `/pending/.json` and are replayed on next successful ingest. +The installer keeps every unrelated key in the settings file, wraps each hook in +`|| true` so a failure never breaks the session, runs the capture hooks `async` +so the chat never waits, and gives `SessionEnd` an explicit timeout. -- Gives each hook the timing it needs: the capture hooks run `async` so the chat never waits on them, and `SessionEnd` carries an explicit `timeout` — without one Claude Code cancels it after 1.5 seconds and the tail of the session is lost. - -> **Note**: the hook payload arrives as JSON on stdin (`hook_event_name`, `source`, `cwd`, `transcript_path`, …). There are no `$CLAUDE_HOOK_*` environment variables — an older template used them and always fed the classifier empty text. +> The hook payload arrives as JSON on stdin. There are no `$CLAUDE_HOOK_*` +> environment variables. ## Codex -Codex speaks the same hook protocol, so the same journal works there: - ```bash codex mcp add task-journal -- task-journal-mcp -task-journal install-hooks --scope user --client codex -# → /home//.codex/hooks.json +task-journal install-hooks --scope user --client codex # → ~/.codex/hooks.json +``` + +Codex asks before each MCP tool call. To let the agent journal without a prompt +every time, approve the server's tools in `~/.codex/config.toml`: + +```toml +[mcp_servers.task-journal] +command = "task-journal-mcp" +default_tools_approval_mode = "approve" ``` -`--client codex` writes to Codex's own hooks file and trims the wiring to what Codex supports: its `SessionEnd` budget caps at 3 seconds, and it has no `PostModelSwitch` event. Everything else — the resume pack on `SessionStart`, the nudge, the background capture — behaves the same. +In `codex exec` (non-interactive) an unapproved MCP call is refused outright, so +this setting is required there. + +Codex sends its session id with every tool call, so events recorded from Codex +are bound to their session like Claude Code's: each Codex session resumes its own +task. `--client codex` trims the hook wiring to what Codex supports (its +`SessionEnd` budget is 3 seconds and it has no `PostModelSwitch` event). `dream`, +`backfill` and `complete --enrich` read Claude Code transcripts only. + +## Other MCP clients + +Point the client at the `task-journal-mcp` command (stdio). The server's +built-in instructions describe the workflow, and the seven tools work the same: +`task_create`, `event_add`, `artifact_add`, `task_close`, `task_pack`, +`task_search`, `task_check`. The project is the server's working directory; start +it with `--project-dir ` if the client launches it elsewhere. A client that +knows its session id can pass `session_id` to the write tools so each session +keeps its own active task. -## Verify install +## Verify ```bash -task-journal create "Test task" -# → tj-xxxxxx -task-journal event tj-xxxxxx --type decision --text "Adopt my plan" -task-journal pack tj-xxxxxx --mode full +task-journal create "Test task" --goal "Check the install" +# → tj-xxxxxxxxxx +task-journal event tj-xxxxxxxxxx --type decision --text "Adopt my plan" +task-journal pack tj-xxxxxxxxxx --mode full +task-journal doctor ``` -You should see Markdown output with the title and the `[decision]` event. - ## Uninstall hooks ```bash @@ -99,7 +158,11 @@ task-journal install-hooks --scope user --uninstall # add --client codex to remove them from ~/.codex/hooks.json ``` -This removes only task-journal's own hook entries (and its `TJ_CLASSIFIER_CLI` env key) from `~/.claude/settings.json`, or from `~/.codex/hooks.json` with `--client codex`. Other plugins' hooks, theme, servers and everything else stay as they are. With `--scope project` the file is `.claude/settings.json` or `.codex/hooks.json` in the current directory. +This removes only task-journal's own hook entries (and its old +`TJ_CLASSIFIER_CLI` env key) from `~/.claude/settings.json`, or from +`~/.codex/hooks.json` with `--client codex`. Everything else stays. With +`--scope project` the file is `.claude/settings.json` or `.codex/hooks.json` in +the current directory. ## Where data lives @@ -109,35 +172,28 @@ This removes only task-journal's own hook entries (and its `TJ_CLASSIFIER_CLI` e | macOS | `~/Library/Application Support/task-journal` | | Windows | `%LOCALAPPDATA%\task-journal` | -Inside: +`TASK_JOURNAL_DATA_DIR` overrides it. Inside: ``` task-journal/ -├── events/.jsonl # append-only event log (source of truth) -├── state/.sqlite # derived state (rebuildable from JSONL) -├── metrics/.jsonl # classifier telemetry -└── pending/.json # failed classifications awaiting retry +├── events/.jsonl # append-only event log (source of truth) +├── state/.sqlite # derived state (rebuildable from the log) +├── metrics/.jsonl # classifier telemetry +└── pending/..json # auto-capture chunks waiting for the classifier ``` -To reset one project's state: `rm /state/.sqlite` (regenerated on next read). - -To wipe entire journal: `rm -rf `. - -## Diagnostics - -| Issue | Check | -|-------|-------| -| Hook didn't write anything | `ls /events/` — JSONL exists? | -| `[?]` markers everywhere | Classifier confidence consistently below 0.85; consider `task-journal stats` for accuracy ratio | -| Pack is huge | Check `metadata.truncated` flag; pack is capped at 10 KB (full) / 2 KB (compact) | -| `ANTHROPIC_API_KEY env var not set` | Hook runs with same env as Claude Code; export the key in your shell rc file | -| `pending/` growing | Real classifier is failing; inspect `/pending/*.json` for last error | +To reset one project's state, delete `state/.sqlite`; it is rebuilt from +the log on the next read, goals and external links included. To wipe the whole +journal: `rm -rf `. ## Troubleshooting | Symptom | Fix | |---------|-----| -| `task-journal-mcp: command not found` | Не сделал `cargo install`, либо `~/.cargo/bin` не в PATH. Проверь `which task-journal-mcp`. | -| Plugin виден, но MCP tools не подцепились | Перезапусти Claude Code полностью (закрыть, открыть). | -| Hook ничего не пишет в `events/` | `ANTHROPIC_API_KEY` не выставлен — экспортируй в `~/.bashrc`. | -| Pack возвращает "task not found" | Сначала `task-journal create`, потом `pack` — порядок важен. Или после нескольких ингестов запусти `task-journal rebuild-state`. | +| `task-journal-mcp: command not found` | `cargo install` didn't run, or `~/.cargo/bin` isn't on PATH (`which task-journal-mcp`). | +| The plugin is listed but the tools are missing | Restart Claude Code completely. | +| "the task-journal CLI 0.30+ was not found" in the transcript | The mod needs the binaries: `cargo install task-journal-cli task-journal-mcp --force`. | +| Codex says an MCP call "requires approval" | Add `default_tools_approval_mode = "approve"` to the server's block in `~/.codex/config.toml`. | +| `pending/` keeps growing | Auto-capture's classifier is failing; `task-journal pending list` shows the last error, `task-journal pending retry` retries. | +| A pack says "task not found" | The id is wrong or from another project; `task-journal search ""` lists this project's tasks. | +| A pack looks cut | Packs are capped at 2 KB (compact) and 32 KB (full); `task_pack` reports `truncated`. | diff --git a/README.md b/README.md index e565096..08c2dc8 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ Then install the Rust binaries that the plugin calls into: cargo install task-journal-cli task-journal-mcp ``` -That's it. Restart Claude Code, start working, and the journal fills itself. +That's it. Restart Claude Code, start working, and the journal fills itself. On +Claude Code 2.1.287+ the plugin also brings the **Task Journal mod** (see +[How it works](#how-it-works)). **Alternative installs:** [pre-built binaries](https://github.com/Digital-Threads/Task-Journal/releases), `cargo install` only (manual MCP wiring), or build from source — see [Manual Setup](#manual-setup). @@ -80,6 +82,11 @@ codex mcp add task-journal -- task-journal-mcp task-journal install-hooks --scope user --client codex ``` +Codex asks before every MCP tool call; add `default_tools_approval_mode = "approve"` +under `[mcp_servers.task-journal]` in `~/.codex/config.toml` so the agent can journal +without a prompt each time (in `codex exec` it is required). Any other MCP client +works with the server alone — see [INSTALL.md](INSTALL.md) for every client. + ## Why not just use the agent's own memory? Claude Code has auto-memory (`MEMORY.md`, `/memory`) and Codex has its own memory @@ -94,6 +101,7 @@ that tie it back to the code. ## How it works +- **The mod (Claude Code 2.1.287+).** A plugin of function hooks that runs inside Claude Code and calls the `task-journal` CLI. It keeps the session's active task in the system prompt (a compaction can't drop it), gives each session its own active task, shows the task in the status line, reminds the agent to log only after several prompts without an entry, and — right before a compaction — asks the model what was never logged and records it as `suggested` events. It needs no setup; when it runs, the classic capture hooks stand down so nothing is recorded twice. Codex and older Claude Code keep using the hooks below. - **Self-tagging is the primary path (recommended).** You — the agent in the live session — record reasoning directly via the five MCP tools: open a task with a `goal`, append a typed `decision` / `finding` / `rejection` / `evidence` event at the moment of commitment, and `task_close` with a written `outcome`. This is free (it rides the interactive session), language-agnostic, and higher-fidelity than any after-the-fact classifier. The bundled `task-journal` skill drives this automatically. See [MCP tools](#mcp-tools). - **Auto-capture is an opt-in backstop — OFF by default (v0.14.0).** A fresh `install-hooks` wires only a cheap, read-only SessionStart resume hook: no per-message classifier runs, no `claude -p` is ever spawned, nothing is charged. Self-tagging is the capture mechanism. Opt in with `install-hooks --auto-capture` and Claude Code hooks also run every prompt, tool call, and reply through a two-stage classifier that lands typed events on its own: Stage 1 is a fast in-process heuristic (obvious EN+RU phrasing, zero cost); Stage 2 falls back to an LLM only when the heuristic is uncertain (and only if you pick `--backend agent-sdk` / `api`). Even opted in it is a safety net under your explicit self-tagging, not the main mechanism, and it misses real reasoning — especially non-English prose. - **Artifact extraction.** Each event scans its text for commit hashes, PR URLs, file paths, issue IDs, and branch names. Aggregated artifacts are how Task Journal links related tasks: when you start a new task touching the same issue or file, the prior task is surfaced automatically. @@ -163,14 +171,15 @@ task-journal pack tj-x9rz1f --mode full |---------|--------------| | `create [--goal "..."]` | Open a task with optional goal | | `goal <id> "..."` | Set or replace a task's goal | -| `event <id> --type X --text Y` | Append a typed event | +| `event <id> --type X --text Y [--suggested] [--session S] [--origin O]` | Append a typed event | +| `state [--session S]` | The session's active task, counts and latest entries as JSON (what the mod reads) | | `event-correct --corrects <eid> --task <id> --text "..."` | Correct an earlier event | | `external <id> "..."` | Append an external reference (URL, ticket, linked task) | | `close <id> --outcome "..." --outcome-tag done\|abandoned\|superseded` | Close with outcome | | `reopen <id> --reason "..."` | Reopen a closed task | | `pack <id> --mode compact\|full` | Render a resume pack | | `events list [--limit N]` | List recent events | -| `search <query> [--all-projects]` | Full-text search (FTS5) | +| `search <query> [--all-projects]` | Full-text search (FTS5); an empty query lists tasks | | `rejected <topic> [--all-projects] [--limit N] [--since DAYS]` | Cross-task rejection lookup — surfaces approaches already turned down | | `export-pr <id>` | Render a task as PR-description Markdown | | `statusline` | One-liner for `~/.claude/settings.json` `statusLine` (sub-100ms) | @@ -189,15 +198,21 @@ task-journal pack tj-x9rz1f --mode full ## MCP tools -The MCP server exposes five tools to Claude Code (and any MCP client): +The MCP server exposes seven tools to Claude Code, Codex and any MCP client: | Tool | Purpose | |------|---------| -| `task_create` | Open a task with optional goal | +| `task_create` | Open a task with a goal (and optional parent) | | `event_add` | Append a typed reasoning event | -| `task_pack` | Render a resume pack | -| `task_search` | Full-text search across events | +| `artifact_add` | Attach a typed link (doc, deploy, dashboard, …) | | `task_close` | Close with outcome and outcome tag | +| `task_pack` | Render a resume pack | +| `task_search` | Search tasks: full text, or `status="open"` with no query to list them | +| `task_check` | Score a task's completeness and list its gaps | + +The write tools take an optional `session_id`; Claude Code (through the mod) and +Codex (through each call's metadata) fill it in, so every session keeps its own +active task. ## Configuration @@ -261,6 +276,8 @@ PATH), `dream` says so and stops. Model overrides per backend are in the table b | `supersede` | Replaces an earlier event | | `close` | Task completed with outcome and outcome tag | | `redirect` | Task re-routed to a different task | +| `rename` | Task title changed | +| `amend` | Goal or external references changed (bookkeeping, kept out of packs and search) | ## Manual Setup @@ -311,8 +328,8 @@ cargo install task-journal-cli task-journal-mcp --force Restart Claude Code and verify: ```bash -task-journal --version # 0.6.3 -task-journal-mcp --version # 0.6.3 +task-journal --version # both print the same version as the plugin +task-journal-mcp --version ``` If you installed from source: From 2012dec5eeed8c2e3cf6790b2aeda2d33a3768ad Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 13:56:52 +0400 Subject: [PATCH 076/104] fix(pack): cache only the stable body and render linked status and gaps on every call --- crates/tj-core/src/db.rs | 30 +++++++ crates/tj-core/src/pack.rs | 163 +++++++++++++++++++++++-------------- 2 files changed, 132 insertions(+), 61 deletions(-) diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 23396fb..2e898b6 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -178,6 +178,12 @@ ALTER TABLE events_index ADD COLUMN bookkeeping INTEGER NOT NULL DEFAULT 0; DELETE FROM index_state; "#; +/// v0.30.0 pack cache holds only the stable body; the header and the gaps +/// are rendered on every call. Rows cached as the whole pack text go. +const MIGRATION_012: &str = r#" +DELETE FROM task_pack_cache; +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -225,6 +231,10 @@ const MIGRATIONS: &[Migration] = &[ version: 11, sql: MIGRATION_011, }, + Migration { + version: 12, + sql: MIGRATION_012, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -2625,6 +2635,26 @@ mod tests { assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); } + #[test] + fn upgrading_wipes_packs_cached_as_whole_text() { + let d = TempDir::new().unwrap(); + let db = d.path().join("s.sqlite"); + let conn = open(&db).unwrap(); + conn.execute_batch( + "INSERT INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) + VALUES ('tj-x', 'compact', '# Old whole pack', '', 1); + DELETE FROM schema_migrations WHERE version = 12;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + let cached: i64 = conn + .query_row("SELECT COUNT(*) FROM task_pack_cache", [], |r| r.get(0)) + .unwrap(); + assert_eq!(cached, 0); + } + #[test] fn upgrading_an_existing_db_flags_old_bookkeeping_on_the_next_ingest() { let d = TempDir::new().unwrap(); diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index 3d07e25..ad5417f 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -310,32 +310,7 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res PackMode::Compact => "compact", PackMode::Full => "full", }; - - // Read-through cache: if we have a stored pack with the same mode, return it. - let cached: Option<(String, String, i64)> = conn - .query_row( - "SELECT text, generated_at, source_event_count FROM task_pack_cache - WHERE task_id=?1 AND mode=?2", - rusqlite::params![task_id, mode_str], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - ) - .ok(); - if let Some((cached_text, cached_at, cached_count)) = cached { - // Detect truncation by re-checking for the marker. - let was_truncated = cached_text.contains("_(truncated to fit pack budget)_"); - return Ok(TaskPack { - task_id: task_id.to_string(), - mode, - schema_version: crate::SCHEMA_VERSION.into(), - text: cached_text, - metadata: PackMetadata { - generated_at: cached_at, - source_event_count: cached_count as usize, - cache_hit: true, - truncated: was_truncated, - }, - }); - } + let generated_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); let (title, status, goal, outcome, outcome_tag, external): ( String, @@ -361,12 +336,6 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res ) .with_context(|| format!("task not found: {task_id}"))?; - let event_count: usize = conn.query_row( - "SELECT COUNT(*) FROM events_index WHERE task_id=?1", - rusqlite::params![task_id], - |r| r.get::<_, i64>(0).map(|n| n as usize), - )?; - let mut text = format!("# {title} [status: {status}]\n\n"); // v0.4.0 task-as-goal block. Goal renders even when empty so the @@ -457,26 +426,36 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res } text.push('\n'); - if matches!(mode, PackMode::Full) { - text.push_str(&render_lifecycle(conn, task_id)?); - } - text.push_str(&render_active_decisions(conn, task_id)?); - if matches!(mode, PackMode::Full) { - text.push_str(&render_rejected(conn, task_id)?); - text.push_str(&render_evidence(conn, task_id)?); - } - let recent_limit = match mode { - PackMode::Compact => 3, - PackMode::Full => 10, + // Read-through cache of the body. It changes only with this task's own + // events (or a child's), and index_event clears it then. The header above + // (live status of linked tasks) and the gaps below (pending queue, cwd git + // state) change without such an event, so they are rendered every call. + let cached: Option<(String, i64)> = conn + .query_row( + "SELECT text, source_event_count FROM task_pack_cache WHERE task_id=?1 AND mode=?2", + rusqlite::params![task_id, mode_str], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .ok(); + let cache_hit = cached.is_some(); + let (body, event_count) = match cached { + Some((body, count)) => (body, count as usize), + None => { + let body = render_body(conn, task_id, mode)?; + let event_count: usize = conn.query_row( + "SELECT COUNT(*) FROM events_index WHERE task_id=?1", + rusqlite::params![task_id], + |r| r.get::<_, i64>(0).map(|n| n as usize), + )?; + conn.execute( + "INSERT OR REPLACE INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![task_id, mode_str, body, generated_at, event_count as i64], + )?; + (body, event_count) + } }; - text.push_str(&render_recent_events(conn, task_id, recent_limit)?); - - // One-level roll-up of direct children (parents only). Appended before - // truncation so it shares the pack budget. Task 5 busts the parent cache - // when a child changes, so the next assemble regenerates fresh. - if let Some(subtasks) = render_subtasks(conn, task_id)? { - text.push_str(&subtasks); - } + text.push_str(&body); let mut report = crate::completeness::assess(conn, task_id, crate::completeness::pending_count())?; @@ -511,15 +490,6 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res } text.push_str(&gaps); - let generated_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); - - // Write-through cache. - conn.execute( - "INSERT OR REPLACE INTO task_pack_cache(task_id, mode, text, generated_at, source_event_count) - VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![task_id, mode_str, text, generated_at, event_count as i64], - )?; - Ok(TaskPack { task_id: task_id.to_string(), mode, @@ -528,12 +498,40 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res metadata: PackMetadata { generated_at, source_event_count: event_count, - cache_hit: false, + cache_hit, truncated, }, }) } +/// The cacheable part of a pack: every section built from this task's own +/// events and its children's, in pack order. +fn render_body(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Result<String> { + let mut text = String::new(); + if matches!(mode, PackMode::Full) { + text.push_str(&render_lifecycle(conn, task_id)?); + } + text.push_str(&render_active_decisions(conn, task_id)?); + if matches!(mode, PackMode::Full) { + text.push_str(&render_rejected(conn, task_id)?); + text.push_str(&render_evidence(conn, task_id)?); + } + let recent_limit = match mode { + PackMode::Compact => 3, + PackMode::Full => 10, + }; + text.push_str(&render_recent_events(conn, task_id, recent_limit)?); + + // One-level roll-up of direct children (parents only). Appended before + // truncation so it shares the pack budget. Task 5 busts the parent cache + // when a child changes, so the next assemble regenerates fresh. + if let Some(subtasks) = render_subtasks(conn, task_id)? { + text.push_str(&subtasks); + } + + Ok(text) +} + #[cfg(test)] mod tests { use super::*; @@ -1455,4 +1453,47 @@ mod tests { assert!(line.contains("## Completeness (50)"), "{line}"); assert!(line.contains("honesty score: 0/100"), "{line}"); } + + #[test] + fn cached_pack_shows_live_status_of_linked_tasks() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-a", "A"); + open_task(&conn, "tj-b", "B"); + crate::db::add_task_external(&conn, "tj-a", "linked:tj-b").unwrap(); + assert!(assemble(&conn, "tj-a", PackMode::Compact) + .unwrap() + .text + .contains("- tj-b [open]")); + + // Closing B touches only B's events, so A's cached body stays valid. + put(&conn, &ev("tj-b", EventType::Close, "done")); + + let pack = assemble(&conn, "tj-a", PackMode::Compact).unwrap(); + assert!(pack.metadata.cache_hit); + assert!(pack.text.contains("- tj-b [closed]"), "{}", pack.text); + } + + #[test] + fn cached_pack_shows_live_gaps() { + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-g", "G"); + crate::db::set_task_goal(&conn, "tj-g", "g").unwrap(); + let _env = crate::test_env_lock(); + std::env::set_var("TASK_JOURNAL_DATA_DIR", d.path()); + let first = assemble(&conn, "tj-g", PackMode::Compact).unwrap(); + + // A pending entry appears without any new event on the task. + std::fs::create_dir_all(d.path().join("pending")).unwrap(); + std::fs::write(d.path().join("pending/x.json"), "{}").unwrap(); + let second = assemble(&conn, "tj-g", PackMode::Compact).unwrap(); + std::env::remove_var("TASK_JOURNAL_DATA_DIR"); + + assert!(!first.text.contains("## Completeness"), "{}", first.text); + assert!(second.metadata.cache_hit); + assert!(second.text.contains("1 pending entry"), "{}", second.text); + } } From 52b6842762c36391fac6b71f091b134f62c0e6d6 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 13:59:20 +0400 Subject: [PATCH 077/104] fix(export-pr): list the pack's active decisions and rejections, marking unconfirmed ones --- crates/tj-cli/src/main.rs | 40 ++++++------ crates/tj-cli/tests/cli.rs | 59 +++++++++++++++++ crates/tj-core/src/pack.rs | 127 +++++++++++++++++++++++-------------- 3 files changed, 159 insertions(+), 67 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index b8add89..66dbb28 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3957,33 +3957,37 @@ fn run_export_pr(task_id: &str) -> Result<()> { let meta = tj_core::db::task_metadata(&conn, task_id)?.unwrap_or_default(); let summary = meta.goal.unwrap_or_else(|| title.clone()); - // Pull all events ordered ASC so the PR description reads like a - // narrative (oldest decision first → newest). + // Changes / Why list exactly the pack's Active decisions / Rejected, + // reversed so the PR description reads like a narrative (oldest first). + let one_lines = |choices: Vec<tj_core::pack::Choice>| -> Vec<String> { + choices + .iter() + .rev() + .filter_map(|c| { + let one_line = c.text.lines().next().unwrap_or("").trim(); + (!one_line.is_empty()).then(|| format!("{one_line}{}", c.marker())) + }) + .collect() + }; + let decisions = one_lines(tj_core::pack::active_decisions(&conn, task_id)?); + let rejections = one_lines(tj_core::pack::rejections(&conn, task_id)?); + let mut stmt = conn.prepare( - "SELECT ei.type, sf.text FROM events_index ei + "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 AND ei.corrected_by IS NULL AND ei.bookkeeping = 0 + WHERE ei.task_id = ?1 AND ei.type = 'evidence' AND ei.corrected_by IS NULL ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { - let ty: String = r.get(0)?; - let txt: Option<String> = r.get(1)?; - Ok((ty, txt.unwrap_or_default())) + let txt: Option<String> = r.get(0)?; + Ok(txt.unwrap_or_default()) })?; - let mut decisions: Vec<String> = Vec::new(); - let mut rejections: Vec<String> = Vec::new(); let mut evidence: Vec<String> = Vec::new(); for row in rows { - let (ty, text) = row?; + let text = row?; let one_line: String = text.lines().next().unwrap_or("").trim().to_string(); - if one_line.is_empty() { - continue; - } - match ty.as_str() { - "decision" => decisions.push(one_line), - "rejection" => rejections.push(one_line), - "evidence" => evidence.push(one_line), - _ => {} + if !one_line.is_empty() { + evidence.push(one_line); } } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 6713724..3235713 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5070,6 +5070,65 @@ fn export_pr_leaves_out_corrected_events() { assert!(pr.contains("- Use SQLite"), "{pr}"); } +#[test] +fn export_pr_lists_the_same_decisions_and_rejections_as_the_pack() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let xdg = dir.path(); + + let task = tj_stdout(xdg, &workdir, &["create", "Same selection", "--goal", "g"]); + let event = |ty: &str, text: &str| { + tj_stdout( + xdg, + &workdir, + &["event", &task, "--type", ty, "--text", text], + ) + }; + let ts = event("decision", "Adopt TypeScript"); + tj_stdout( + xdg, + &workdir, + &[ + "event", + &task, + "--type", + "supersede", + "--text", + "TS replaced", + "--supersedes", + &ts, + ], + ); + event("decision", "Adopt Rust"); + event("decision", "Adopt Rust"); + // Classifier guesses, not yet confirmed. + for (ty, text) in [ + (tj_core::event::EventType::Decision, "Maybe cache packs"), + (tj_core::event::EventType::Rejection, "Maybe drop Postgres"), + ] { + let mut e = tj_core::event::Event::new( + task.clone(), + ty, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text.to_string(), + ); + e.status = tj_core::event::EventStatus::Suggested; + append_jsonl_line(xdg, &serde_json::to_string(&e).unwrap()); + } + + let pr = tj_stdout(xdg, &workdir, &["export-pr", &task]); + let changes = pr.split("## ").find(|s| s.starts_with("Changes")).unwrap(); + assert!(!changes.contains("Adopt TypeScript"), "{pr}"); + assert_eq!(changes.matches("Adopt Rust").count(), 1, "{pr}"); + assert!( + changes.contains("- Maybe cache packs _(unconfirmed)_\n"), + "{pr}" + ); + assert!(pr.contains("- Maybe drop Postgres _(unconfirmed)_"), "{pr}"); +} + #[test] fn export_pr_unknown_task_id_exits_one_with_stderr_message() { let dir = assert_fs::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index ad5417f..cf662b6 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -104,18 +104,66 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result<String> { Ok(out) } -/// A classifier's guess (`status = suggested`) must not read like a choice -/// the agent made: such decisions and rejections stay listed but marked. -fn unconfirmed_marker(status: &str) -> &'static str { - if status == "suggested" { - " _(unconfirmed)_" - } else { - "" +/// A decision or rejection as the pack lists it. +pub struct Choice { + pub text: String, + /// A decision's `meta.alternatives` JSON; always `None` for a rejection. + pub alternatives: Option<String>, + /// A classifier's guess (`status = suggested`). + pub unconfirmed: bool, +} + +impl Choice { + /// A guess must not read like a choice the agent made: it stays listed + /// but carries this trailing marker. + pub fn marker(&self) -> &'static str { + if self.unconfirmed { + " _(unconfirmed)_" + } else { + "" + } } } -fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { - let mut out = String::from("## Rejected\n"); +/// Keep the first of each text, minus machine noise (compaction markers), so +/// a section reads as crisp choices, not repeated essays. +fn dedupe(choices: impl IntoIterator<Item = Choice>) -> Vec<Choice> { + let mut seen: HashSet<String> = HashSet::new(); + choices + .into_iter() + .filter(|c| !is_noise(&c.text) && seen.insert(c.text.trim().to_string())) + .collect() +} + +/// The pack's Active decisions, newest first: neither superseded, corrected +/// nor bookkeeping. export-pr lists the same ones. +pub fn active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result<Vec<Choice>> { + // v0.10.3: newest decision first. `decision_id` is a ULID so DESC + // gives reverse-chronological order. The summary/final-decision + // event the agent records just before close is now the FIRST line + // of this section, surviving end-of-pack truncation. + let mut stmt = conn.prepare( + "SELECT d.text, d.alternatives, ei.status FROM decisions d + JOIN events_index ei ON ei.event_id = d.decision_id + WHERE d.task_id=?1 AND d.status='active' AND ei.corrected_by IS NULL + AND ei.bookkeeping = 0 + ORDER BY d.decision_id DESC", + )?; + let rows = stmt + .query_map(rusqlite::params![task_id], |r| { + Ok(Choice { + text: r.get(0)?, + alternatives: r.get(1)?, + unconfirmed: r.get::<_, String>(2)? == "suggested", + }) + })? + .collect::<Result<Vec<_>, _>>()?; + Ok(dedupe(rows)) +} + +/// The pack's Rejected list, newest first, without corrected rejections. +/// export-pr lists the same ones. +pub fn rejections(conn: &Connection, task_id: &str) -> anyhow::Result<Vec<Choice>> { // v0.10.3: newest-first so end-of-pack truncation drops the // OLDEST rejections, not the latest decision the agent recorded. let mut id_stmt = conn.prepare( @@ -127,17 +175,24 @@ fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { let event_ids: Vec<(String, String)> = id_stmt .query_map(rusqlite::params![task_id], |r| Ok((r.get(0)?, r.get(1)?)))? .collect::<Result<_, _>>()?; - let mut count = 0; - let mut seen: HashSet<String> = HashSet::new(); + let mut rows = Vec::new(); for (eid, status) in event_ids { - let text: String = text_stmt.query_row(rusqlite::params![eid], |r| r.get(0))?; - if is_noise(&text) || !seen.insert(text.trim().to_string()) { - continue; - } - out.push_str(&format!("- {text}{}\n", unconfirmed_marker(&status))); - count += 1; + rows.push(Choice { + text: text_stmt.query_row(rusqlite::params![eid], |r| r.get(0))?, + alternatives: None, + unconfirmed: status == "suggested", + }); } - if count == 0 { + Ok(dedupe(rows)) +} + +fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { + let mut out = String::from("## Rejected\n"); + let rejected = rejections(conn, task_id)?; + for r in &rejected { + out.push_str(&format!("- {}{}\n", r.text, r.marker())); + } + if rejected.is_empty() { out.push_str("- (none)\n"); } out.push('\n'); @@ -146,43 +201,17 @@ fn render_rejected(conn: &Connection, task_id: &str) -> anyhow::Result<String> { fn render_active_decisions(conn: &Connection, task_id: &str) -> anyhow::Result<String> { let mut out = String::from("## Active decisions\n"); - // v0.10.3: newest decision first. `decision_id` is a ULID so DESC - // gives reverse-chronological order. The summary/final-decision - // event the agent records just before close is now the FIRST line - // of this section, surviving end-of-pack truncation. - let mut stmt = conn.prepare( - "SELECT d.text, d.alternatives, ei.status FROM decisions d - JOIN events_index ei ON ei.event_id = d.decision_id - WHERE d.task_id=?1 AND d.status='active' AND ei.corrected_by IS NULL - AND ei.bookkeeping = 0 - ORDER BY d.decision_id DESC", - )?; - let rows = stmt.query_map(rusqlite::params![task_id], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option<String>>(1)?, - r.get::<_, String>(2)?, - )) - })?; - let mut count = 0; - let mut seen: HashSet<String> = HashSet::new(); - for row in rows { - let (text, alternatives, status) = row?; - // Skip machine noise (compaction markers) and exact duplicates so the - // section reads as crisp decisions, not repeated essays. - if is_noise(&text) || !seen.insert(text.trim().to_string()) { - continue; - } - out.push_str(&format!("- {text}{}\n", unconfirmed_marker(&status))); + let decisions = active_decisions(conn, task_id)?; + for d in &decisions { + out.push_str(&format!("- {}{}\n", d.text, d.marker())); // v0.12.0: structured alternatives render under the decision so the // pack shows "considered A/B/C, chose X" without reconstructing it // from the hypothesis+rejection chain. - if let Some(block) = render_alternatives(alternatives.as_deref()) { + if let Some(block) = render_alternatives(d.alternatives.as_deref()) { out.push_str(&block); } - count += 1; } - if count == 0 { + if decisions.is_empty() { out.push_str("- (none)\n"); } out.push('\n'); From 92739f2460857555983fc2c9f9e28d65f8cb1ceb Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:00:19 +0400 Subject: [PATCH 078/104] fix(pack): skip only malformed alternatives and dedupe evidence like the other sections --- crates/tj-core/src/pack.rs | 51 ++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index cf662b6..4759c76 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -92,8 +92,13 @@ fn render_evidence(conn: &Connection, task_id: &str) -> anyhow::Result<String> { Ok((t, s)) })?; let mut count = 0; + let mut seen: HashSet<String> = HashSet::new(); for row in rows { let (t, s) = row?; + // Same noise / repeat filter as Active decisions and Rejected. + if is_noise(&t) || !seen.insert(t.trim().to_string()) { + continue; + } out.push_str(&format!("- {t} ({s})\n")); count += 1; } @@ -227,12 +232,13 @@ fn render_alternatives(raw: Option<&str>) -> Option<String> { let raw = raw?; let parsed: serde_json::Value = serde_json::from_str(raw).ok()?; let arr = parsed.as_array()?; - if arr.is_empty() { - return None; - } let mut block = String::from(" - considered:\n"); + let header_len = block.len(); for entry in arr { - let option = entry.get("option").and_then(|v| v.as_str())?; + // Skip only the entry without an option, not the whole block. + let Some(option) = entry.get("option").and_then(|v| v.as_str()) else { + continue; + }; let chosen = entry .get("chosen") .and_then(|v| v.as_bool()) @@ -244,7 +250,8 @@ fn render_alternatives(raw: Option<&str>) -> Option<String> { None => block.push_str(&format!(" - {marker} {option}\n")), } } - Some(block) + + (block.len() > header_len).then_some(block) } fn render_lifecycle(conn: &Connection, task_id: &str) -> anyhow::Result<String> { @@ -1525,4 +1532,38 @@ mod tests { assert!(second.metadata.cache_hit); assert!(second.text.contains("1 pending entry"), "{}", second.text); } + + #[test] + fn one_bad_alternative_does_not_hide_the_others() { + let raw = r#"[ + {"chosen": true, "rationale": "no option field"}, + {"option": "SQLite", "chosen": true}, + {"option": "Postgres", "rationale": "too heavy"} + ]"#; + let block = render_alternatives(Some(raw)).expect("good entries still render"); + assert!(block.contains("✓ chose SQLite"), "{block}"); + assert!(block.contains("✗ Postgres — too heavy"), "{block}"); + assert!(!block.contains("no option field"), "{block}"); + } + + #[test] + fn evidence_drops_noise_and_repeats() { + use crate::event::EventType; + + let d = tempfile::TempDir::new().unwrap(); + let conn = crate::db::open(d.path().join("s.sqlite")).unwrap(); + open_task(&conn, "tj-ev2", "Evidence"); + for text in [ + "Tests green: 142 passed", + "Tests green: 142 passed", + "Conversation compacted at 2026-01-01T00:00:00Z; preceding events…", + ] { + put(&conn, &ev("tj-ev2", EventType::Evidence, text)); + } + + let pack = assemble(&conn, "tj-ev2", PackMode::Full).unwrap(); + let evidence = section(&pack.text, "Evidence"); + assert_eq!(evidence.matches("Tests green").count(), 1, "{evidence}"); + assert!(!evidence.contains("Conversation compacted"), "{evidence}"); + } } From 311508cd84a00416ed650ede4231841a6f84adf0 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:01:06 +0400 Subject: [PATCH 079/104] fix(mcp): return generated_at and truncated in task_pack metadata --- crates/tj-mcp/src/main.rs | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index db1ca7f..cad3ba5 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -179,6 +179,10 @@ pub struct TaskPackResult { pub struct TaskPackMetadata { pub source_event_count: Option<usize>, pub cache_hit: Option<bool>, + /// RFC 3339 time the pack text was assembled. + pub generated_at: Option<String>, + /// True when the pack was cut to fit its size budget. + pub truncated: Option<bool>, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -451,6 +455,8 @@ impl TaskJournalServer { metadata: TaskPackMetadata { source_event_count: Some(pack.metadata.source_event_count), cache_hit: Some(pack.metadata.cache_hit), + generated_at: Some(pack.metadata.generated_at), + truncated: Some(pack.metadata.truncated), }, }) }) @@ -1094,6 +1100,8 @@ mod tests { metadata: TaskPackMetadata { source_event_count: None, cache_hit: None, + generated_at: None, + truncated: None, }, }; let pack_v = serde_json::to_value(&pack).unwrap(); @@ -1445,6 +1453,42 @@ mod tests { assert_eq!(res.note.as_deref(), Some("note: 1 open subtask(s)")); } + #[tokio::test] + async fn task_pack_reports_generated_at_and_truncated() { + let _env = handler_env(); + let server = TaskJournalServer; + let task = server + .task_create(Parameters(TaskCreateParams { + title: "Pack metadata".into(), + initial_context: None, + goal: Some("g".into()), + parent: None, + session_id: None, + })) + .await + .unwrap() + .0 + .task_id; + + let pack = server + .task_pack(Parameters(TaskPackParams { + task_id: task, + mode: None, + })) + .await + .unwrap() + .0; + + let meta = serde_json::to_value(&pack.metadata).unwrap(); + assert!( + meta["generated_at"].as_str().is_some_and(|s| !s.is_empty()), + "{meta}" + ); + assert_eq!(meta["truncated"], false, "{meta}"); + assert!(meta["cache_hit"].is_boolean(), "{meta}"); + assert!(meta["source_event_count"].is_number(), "{meta}"); + } + #[tokio::test] async fn task_close_reports_completeness_gaps() { let _env = handler_env(); From 924471f9ba36a576867d9d26435fdb964d757481 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:01:39 +0400 Subject: [PATCH 080/104] test(pack): lock the header and section order of compact and full golden packs --- crates/tj-core/tests/golden_pack.rs | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/tj-core/tests/golden_pack.rs b/crates/tj-core/tests/golden_pack.rs index da80f13..fa01f36 100644 --- a/crates/tj-core/tests/golden_pack.rs +++ b/crates/tj-core/tests/golden_pack.rs @@ -25,6 +25,19 @@ fn fixture_a_compact_pack_for_simple_task() { insta_assert_contains(&pack.text, "Adopt PKCE flow"); insta_assert_contains(&pack.text, "Recent events"); assert_eq!(pack.metadata.source_event_count, 5); + + assert!( + pack.text + .starts_with("# Add OAuth login [status: open]\n\n**Goal**: (not set)\n"), + "header:\n{}", + pack.text + ); + assert_eq!( + section_order(&pack.text), + ["Active decisions", "Recent events", "Completeness"], + "{}", + pack.text + ); } fn build_fixture_a() -> Vec<Event> { @@ -107,6 +120,35 @@ fn fixture_b_full_pack_with_supersede_and_correction() { ); assert_eq!(pack.metadata.source_event_count, 12); + + assert!( + pack.text.starts_with( + "# Stack choice for journal [status: closed]\n\n**Goal**: (not set)\n**Outcome**: (not recorded)\n" + ), + "header:\n{}", + pack.text + ); + assert_eq!( + section_order(&pack.text), + [ + "Lifecycle", + "Active decisions", + "Rejected", + "Evidence", + "Recent events", + "Completeness" + ], + "{}", + pack.text + ); +} + +/// The pack's `## ` section headings in order, without a `(count)` suffix. +fn section_order(text: &str) -> Vec<&str> { + text.lines() + .filter_map(|l| l.strip_prefix("## ")) + .map(|h| h.split(" (").next().unwrap_or(h)) + .collect() } fn build_fixture_b() -> Vec<Event> { From a816a39573b4f111c4fba8b678fba1815e6886fb Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:03:46 +0400 Subject: [PATCH 081/104] chore: bump to 0.30.0 --- .claude-plugin/marketplace.json | 4 ++-- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- plugin/.claude-plugin/plugin.json | 2 +- plugin/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8548135..da354d1 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,14 +6,14 @@ }, "metadata": { "description": "Task Journal — append-only reasoning chain memory for AI-coding tasks", - "version": "0.29.0" + "version": "0.30.0" }, "plugins": [ { "name": "task-journal", "source": "./plugin", "description": "Append-only journal of AI-coding task reasoning chains. Captures hypotheses, decisions, rejections, evidence — renders compact resume packs so an agent can pick up a 2-week-old task with full context.", - "version": "0.29.0", + "version": "0.30.0", "author": { "name": "Digital-Threads" }, diff --git a/Cargo.lock b/Cargo.lock index 529d838..4a60fdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2516,7 +2516,7 @@ dependencies = [ [[package]] name = "task-journal-cli" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "assert_cmd", @@ -2540,7 +2540,7 @@ dependencies = [ [[package]] name = "task-journal-core" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "chrono", @@ -2565,7 +2565,7 @@ dependencies = [ [[package]] name = "task-journal-mcp" -version = "0.29.0" +version = "0.30.0" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index e626480..42b3c2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.29.0" +version = "0.30.0" edition = "2021" rust-version = "1.88" license = "MIT" @@ -20,7 +20,7 @@ categories = ["command-line-utilities", "development-tools"] [workspace.dependencies] # `version` must equal workspace.package.version: `cargo publish` uploads it as # the requirement on core. Cargo can't inherit it; plugin_metadata.rs checks it. -tj-core = { package = "task-journal-core", version = "0.29.0", path = "crates/tj-core", default-features = false } +tj-core = { package = "task-journal-core", version = "0.30.0", path = "crates/tj-core", default-features = false } anyhow = "1" thiserror = "1" serde = { version = "1", features = ["derive"] } diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 7ba1073..a2b653e 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "task-journal", - "version": "0.29.0", + "version": "0.30.0", "description": "Append-only journal of AI-coding task reasoning chains: hypotheses, decisions, rejections, evidence. Renders compact resume packs so an agent can pick up a 2-week-old task with full context.", "author": { "name": "Mher Shahinyan" diff --git a/plugin/package.json b/plugin/package.json index c29a72c..6aba016 100644 --- a/plugin/package.json +++ b/plugin/package.json @@ -1,6 +1,6 @@ { "name": "task-journal", - "version": "0.29.0", + "version": "0.30.0", "description": "Append-only journal of AI-coding task reasoning chains. Captures hypotheses, decisions, rejections, evidence — renders compact resume packs so an agent can pick up a 2-week-old task with full context.", "author": { "name": "Mher Shahinyan", From 49883f46247f9bf21ab48ce8df31f9d28884c723 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:05:47 +0400 Subject: [PATCH 082/104] fix(recall): skip corrected events in recall and global memory sync --- crates/tj-core/src/memory.rs | 51 +++++++++++++++++++++++------------- crates/tj-core/src/recall.rs | 25 ++++++++++++++++++ 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/crates/tj-core/src/memory.rs b/crates/tj-core/src/memory.rs index 0aa2229..340c898 100644 --- a/crates/tj-core/src/memory.rs +++ b/crates/tj-core/src/memory.rs @@ -86,7 +86,7 @@ pub fn sync_from_project( let sql = format!( "SELECT e.event_id, e.task_id, f.type, e.tier, f.text, e.model, e.dim, e.vec, e.created_at, CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END, - COALESCE(ei.bookkeeping, 0) + COALESCE(ei.bookkeeping, 0) OR ei.corrected_by IS NOT NULL FROM embeddings e JOIN search_fts f ON f.event_id = e.event_id LEFT JOIN decisions d ON d.decision_id = e.event_id @@ -106,28 +106,17 @@ pub fn sync_from_project( r.get::<_, Vec<u8>>(7)?, // vec r.get::<_, String>(8)?, // created_at r.get::<_, i64>(9)?, // superseded - r.get::<_, bool>(10)?, // bookkeeping + r.get::<_, bool>(10)?, // bookkeeping or corrected )) })?; let mut n = 0usize; for row in rows { - let ( - event_id, - task_id, - ty, - tier, - text, - model, - dim, - vec, - created_at, - superseded, - bookkeeping, - ) = row?; - // Compaction markers / model switches are not reasoning to recall; - // also drop any copy an older version already synced. - if bookkeeping { + let (event_id, task_id, ty, tier, text, model, dim, vec, created_at, superseded, retired) = + row?; + // Compaction markers / model switches are not reasoning to recall, and a + // corrected event is no longer true; also drop any copy synced earlier. + if retired { global.execute( "DELETE FROM global_memory WHERE event_id = ?1", rusqlite::params![event_id], @@ -420,6 +409,32 @@ mod tests { .is_empty()); } + #[test] + fn sync_leaves_out_and_drops_corrected_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + let wrong = finding("chose the kafka audit pipeline"); + crate::db::index_event(&proj, &wrong).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 1); + + // Corrected after the first sync: the next sync must drop the copy. + let mut corr = finding("that kafka choice was a misread of the brief"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + crate::db::index_event(&proj, &corr).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 0); + assert_eq!(count(&global).unwrap(), 0); + assert!(keyword_search(&global, "kafka audit pipeline", 5) + .unwrap() + .is_empty()); + } + #[test] fn preferences_store_dedup_and_list_in_order() { let d = tempfile::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/recall.rs b/crates/tj-core/src/recall.rs index 5913ffd..c8b96ea 100644 --- a/crates/tj-core/src/recall.rs +++ b/crates/tj-core/src/recall.rs @@ -88,6 +88,7 @@ pub fn relevant_recall( WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND search_fts MATCH ?1" } else { "SELECT ei.event_id, ei.task_id, ei.type, sf.text @@ -96,6 +97,7 @@ pub fn relevant_recall( WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND sf.text LIKE ?1" }; let bind = if let Some(or_query) = fts_or { @@ -140,6 +142,7 @@ pub fn relevant_recall( WHERE ei.status = 'confirmed' AND ei.type IN ('rejection','decision') AND ei.bookkeeping = 0 + AND ei.corrected_by IS NULL AND ei.artifacts LIKE ?1", ) { let rows = stmt.query_map(rusqlite::params![pattern], |r| { @@ -355,4 +358,26 @@ mod tests { let hits = relevant_recall(&conn, "reasoning unit", DEFAULT_MAX_HITS).unwrap(); assert!(hits.is_empty(), "got: {hits:?}"); } + + #[test] + fn corrected_events_are_never_recalled() { + let rej = ev( + "tj-1", + EventType::Rejection, + "Rejected axum for the server in src/server.rs.", + EventStatus::Confirmed, + ); + let mut corr = ev( + "tj-1", + EventType::Correction, + "That was wrong: the stdio issue was unrelated.", + EventStatus::Confirmed, + ); + corr.corrects = Some(rej.event_id.clone()); + let (_d, conn) = seeded(&[rej, corr]); + + // Both the text and the artifact path must skip the corrected event. + let hits = relevant_recall(&conn, "axum src/server.rs", DEFAULT_MAX_HITS).unwrap(); + assert!(hits.is_empty(), "got: {hits:?}"); + } } From ff817753c792a84bc7e80e8a6a9f0f2a07421b24 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:06:33 +0400 Subject: [PATCH 083/104] fix(db): leave artifacts of corrected events out of task_artifacts --- crates/tj-core/src/db.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 2e898b6..106a8cd 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -859,14 +859,15 @@ pub fn reclassify_task_artifacts(conn: &Connection, task_id: &str) -> anyhow::Re /// branches) across every event of a task, deduplicated. Reads the /// per-event JSON payload that `ingest_new_events` populated. Skips /// events whose `artifacts` column is NULL or unparseable rather than -/// failing the pack render. +/// failing the pack render, and corrected events, so a wrong hash or path +/// stops raising a gap once a correction retires it. pub fn task_artifacts( conn: &Connection, task_id: &str, ) -> anyhow::Result<crate::artifacts::Artifacts> { let mut stmt = conn.prepare( "SELECT artifacts FROM events_index - WHERE task_id = ?1 AND artifacts IS NOT NULL + WHERE task_id = ?1 AND artifacts IS NOT NULL AND corrected_by IS NULL ORDER BY timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| r.get::<_, String>(0))?; @@ -1789,6 +1790,24 @@ mod tests { ); } + #[test] + fn task_artifacts_leave_out_corrected_events() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let wrong = make_text_event("fixed in commit dead00beef, see src/gone.rs"); + let kept = make_text_event("also touched src/kept.rs"); + let mut corr = make_text_event("the hash and path above were wrong"); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + for e in [&wrong, &kept, &corr] { + index_event(&conn, e).unwrap(); + } + + let arts = task_artifacts(&conn, "tj-x").unwrap(); + assert!(arts.commit_hashes.is_empty(), "{:?}", arts.commit_hashes); + assert_eq!(arts.files, vec!["src/kept.rs".to_string()]); + } + #[test] fn embed_pending_embeds_all_then_is_idempotent() { let d = TempDir::new().unwrap(); From fcdf90dfbea95473dcd385ef6efd64a0b13de7e8 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:07:07 +0400 Subject: [PATCH 084/104] fix(completeness): do not count corrected events as unconfirmed --- crates/tj-core/src/completeness.rs | 37 ++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/crates/tj-core/src/completeness.rs b/crates/tj-core/src/completeness.rs index fdcebc8..ce1d0a6 100644 --- a/crates/tj-core/src/completeness.rs +++ b/crates/tj-core/src/completeness.rs @@ -109,9 +109,11 @@ pub fn assess( let mut evidence = 0usize; let mut suggested = 0usize; { - // Bookkeeping (compaction markers) is not a decision to verify. + // Bookkeeping (compaction markers) is not a decision to verify, and a + // corrected event no longer needs confirming. let mut stmt = conn.prepare( - "SELECT type, status FROM events_index WHERE task_id = ?1 AND bookkeeping = 0", + "SELECT type, status FROM events_index + WHERE task_id = ?1 AND bookkeeping = 0 AND corrected_by IS NULL", )?; let rows = stmt.query_map(rusqlite::params![task_id], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) @@ -479,6 +481,37 @@ mod tests { assert!(g.detail.contains('2')); } + #[test] + fn corrected_suggested_event_is_not_unconfirmed() { + use crate::event::EventStatus; + let (_d, c) = conn(); + open_task(&c, "t4c"); + c.execute("UPDATE tasks SET goal='g' WHERE task_id='t4c'", []) + .unwrap(); + let mut wrong = Event::new( + "t4c", + EventType::Finding, + Author::Agent, + Source::Hook, + "x".into(), + ); + wrong.status = EventStatus::Suggested; + let mut corr = Event::new( + "t4c", + EventType::Correction, + Author::User, + Source::Cli, + "y".into(), + ); + corr.status = EventStatus::Confirmed; + corr.corrects = Some(wrong.event_id.clone()); + crate::db::index_event(&c, &wrong).unwrap(); + crate::db::index_event(&c, &corr).unwrap(); + + let r = assess(&c, "t4c", 0).unwrap(); + assert!(r.is_complete(), "{:?}", r.gaps); + } + #[test] fn pending_leak_fires_when_count_positive() { let (_d, c) = conn(); From 96dbea18271f7694eab3716c47cee08901e7c062 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:10:05 +0400 Subject: [PATCH 085/104] fix(cli): export-memory selects decisions and constraints like the pack --- crates/tj-cli/src/main.rs | 36 +++++++++------- crates/tj-cli/tests/cli.rs | 88 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 16 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d9b01d8..3132dc5 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -4287,31 +4287,35 @@ fn run_export_memory(task: Option<&str>, _all_closed: bool, dry_run: bool) -> Re )?; let meta = tj_core::db::task_metadata(&conn, id)?.unwrap_or_default(); - // decision + constraint one-liners, oldest-first (== run_export_pr style). + // One-liners, oldest-first (== run_export_pr style). Decisions are the + // pack's Active decisions; constraints skip bookkeeping (model + // switches) and corrected ones by the same rules. + let decisions: Vec<String> = tj_core::pack::active_decisions(&conn, id)? + .iter() + .rev() + .filter_map(|c| { + let line = c.text.lines().next().unwrap_or("").trim(); + (!line.is_empty()).then(|| format!("{line}{}", c.marker())) + }) + .take(MAX_ITEMS) + .collect(); + let mut stmt = conn.prepare( - "SELECT ei.type, sf.text FROM events_index ei + "SELECT sf.text FROM events_index ei LEFT JOIN search_fts sf ON sf.event_id = ei.event_id - WHERE ei.task_id = ?1 AND ei.type IN ('decision','constraint') + WHERE ei.task_id = ?1 AND ei.type = 'constraint' + AND ei.bookkeeping = 0 AND ei.corrected_by IS NULL ORDER BY ei.timestamp ASC", )?; let rows = stmt.query_map(rusqlite::params![id], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option<String>>(1)?.unwrap_or_default(), - )) + Ok(r.get::<_, Option<String>>(0)?.unwrap_or_default()) })?; - let mut decisions = Vec::new(); let mut constraints = Vec::new(); for row in rows { - let (ty, text) = row?; + let text = row?; let line = text.lines().next().unwrap_or("").trim().to_string(); - if line.is_empty() { - continue; - } - match ty.as_str() { - "decision" if decisions.len() < MAX_ITEMS => decisions.push(line), - "constraint" if constraints.len() < MAX_ITEMS => constraints.push(line), - _ => {} + if !line.is_empty() && constraints.len() < MAX_ITEMS { + constraints.push(line); } } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a68d49f..4a3ac4b 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -6872,6 +6872,94 @@ fn export_memory_missing_task_exits_1() { .stderr(contains("task not found")); } +#[test] +fn export_memory_lists_the_same_decisions_as_the_pack_and_real_constraints() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let claude = assert_fs::TempDir::new().unwrap(); + let (x, p) = (xdg.path(), proj.path()); + + let task = tj_stdout(x, p, &["create", "Memory selection", "--goal", "g"]); + let event = + |ty: &str, text: &str| tj_stdout(x, p, &["event", &task, "--type", ty, "--text", text]); + let ts = event("decision", "Adopt TypeScript"); + tj_stdout( + x, + p, + &[ + "event", + &task, + "--type", + "supersede", + "--text", + "TS replaced", + "--supersedes", + &ts, + ], + ); + for (ty, text) in [ + ("decision", "Wrong decision: use Mongo"), + ("constraint", "Wrong constraint: no disk"), + ] { + let id = event(ty, text); + tj_stdout( + x, + p, + &[ + "event-correct", + "--corrects", + &id, + "--task", + &task, + "--text", + "That was a mistake", + ], + ); + } + for (ty, text, kind) in [ + ( + tj_core::event::EventType::Decision, + "Conversation compacted at 2026-01-01T00:00:00Z", + "compaction_marker", + ), + ( + tj_core::event::EventType::Constraint, + "Model switched (auto): opus → haiku", + "model_switch", + ), + ] { + let mut e = tj_core::event::Event::new( + task.clone(), + ty, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text.to_string(), + ); + e.meta = serde_json::json!({ "kind": kind }); + append_jsonl_line(x, &serde_json::to_string(&e).unwrap()); + } + event("decision", "Adopt Rust"); + event("constraint", "Must run offline"); + + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", x) + .env("CLAUDE_CONFIG_DIR", claude.path()) + .current_dir(p) + .args(["export-memory", "--task", &task, "--dry-run"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); + for gone in ["TypeScript", "Wrong ", "compacted", "Model switched"] { + assert!(!out.contains(gone), "{gone} must be left out: {out}"); + } + assert!(out.contains("- Adopt Rust\n"), "{out}"); + assert!(out.contains("- Must run offline\n"), "{out}"); +} + #[test] fn embed_backfill_vectorises_events_then_idempotent() { // Pillar A / Phase 0: `embed --backfill` computes a vector per event using From 5ec6e659a7a9716acf250a8896a830bca01a9dce Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:21:16 +0400 Subject: [PATCH 086/104] fix(plugin): pin the mod's task, read the start dir's journal, honest CLI errors, no distill on precompute --- crates/tj-cli/src/main.rs | 13 ++- crates/tj-core/src/session_state.rs | 53 ++++++++++-- plugin/hooks/register.ts | 130 +++++++++++++++++++++------- 3 files changed, 156 insertions(+), 40 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d9b01d8..835a25a 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -890,6 +890,10 @@ enum Commands { /// Session id; defaults to the live session id. #[arg(long)] session: Option<String>, + /// Keep this task as the active one while it is open (the task the + /// caller already shows), instead of the session's latest write. + #[arg(long)] + prefer: Option<String>, }, /// Close a task (writes a `close` event). Close { @@ -1693,7 +1697,7 @@ fn real_main() -> Result<()> { writer.flush_durable()?; println!("{}", event.event_id); } - Commands::State { session } => { + Commands::State { session, prefer } => { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); @@ -1704,7 +1708,12 @@ fn real_main() -> Result<()> { let state = if events_path.exists() { let conn = tj_core::db::open(&state_path)?; tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - tj_core::session_state::session_state(&conn, &project_hash, session.as_deref())? + tj_core::session_state::session_state( + &conn, + &project_hash, + session.as_deref(), + prefer.as_deref(), + )? } else { tj_core::session_state::SessionState { schema: tj_core::session_state::SCHEMA, diff --git a/crates/tj-core/src/session_state.rs b/crates/tj-core/src/session_state.rs index e464005..a59f803 100644 --- a/crates/tj-core/src/session_state.rs +++ b/crates/tj-core/src/session_state.rs @@ -8,7 +8,7 @@ use std::collections::BTreeMap; -use rusqlite::Connection; +use rusqlite::{Connection, OptionalExtension}; use serde::Serialize; /// Bumped when a field changes meaning; readers refuse other schemas. @@ -45,11 +45,15 @@ pub struct RecentEntry { } /// Assemble the state of `session_id` in one project. The caller has -/// ingested the journal into `conn` already. +/// ingested the journal into `conn` already. `prefer` names the task the +/// caller already shows: it stays the active one while it is open, so a +/// session logging to a parent and a subtask in turn doesn't flip between +/// them. pub fn session_state( conn: &Connection, project_hash: &str, session_id: Option<&str>, + prefer: Option<&str>, ) -> anyhow::Result<SessionState> { let open_tasks = conn.query_row( "SELECT COUNT(*) FROM tasks WHERE project_hash = ?1 AND status = 'open'", @@ -57,10 +61,21 @@ pub fn session_state( |r| r.get::<_, i64>(0), )?; - let active_id = match session_id { - Some(sid) => crate::db::active_task_for_session(conn, project_hash, sid)?, + let preferred = match prefer { + Some(id) => conn + .query_row( + "SELECT task_id FROM tasks WHERE task_id = ?1 AND project_hash = ?2 AND status = 'open'", + [id, project_hash], + |r| r.get::<_, String>(0), + ) + .optional()?, None => None, }; + let active_id = match (preferred, session_id) { + (Some(id), _) => Some(id), + (None, Some(sid)) => crate::db::active_task_for_session(conn, project_hash, sid)?, + (None, None) => None, + }; let active = active_id.map(|id| active_task(conn, &id)).transpose()?; Ok(SessionState { @@ -144,7 +159,7 @@ mod tests { write(&conn, "tj-a", EventType::Decision, "Use <=", Some("s1")); write(&conn, "tj-b", EventType::Open, "b", Some("s2")); - let s1 = session_state(&conn, "p", Some("s1")).unwrap(); + let s1 = session_state(&conn, "p", Some("s1"), None).unwrap(); let active = s1.active.expect("s1 has a task"); assert_eq!(active.task_id, "tj-a"); assert_eq!(active.title, "Title of tj-a"); @@ -154,18 +169,38 @@ mod tests { assert_eq!(active.recent[0].kind, "decision"); assert_eq!(s1.open_tasks, 2); - let other = session_state(&conn, "p", Some("s3")).unwrap(); + let other = session_state(&conn, "p", Some("s3"), None).unwrap(); assert!(other.active.is_none()); assert_eq!(other.open_tasks, 2); } + #[test] + fn a_preferred_open_task_stays_active() { + let (_dir, conn) = conn(); + write(&conn, "tj-parent", EventType::Open, "p", Some("s1")); + write(&conn, "tj-sub", EventType::Open, "s", Some("s1")); + + // Without a preference the newest write wins; with one, the pin holds. + let plain = session_state(&conn, "p", Some("s1"), None).unwrap(); + assert_eq!(plain.active.unwrap().task_id, "tj-sub"); + let pinned = session_state(&conn, "p", Some("s1"), Some("tj-parent")).unwrap(); + assert_eq!(pinned.active.unwrap().task_id, "tj-parent"); + + // A closed or unknown preference falls back to the session's own task. + write(&conn, "tj-parent", EventType::Close, "done", Some("s1")); + let closed = session_state(&conn, "p", Some("s1"), Some("tj-parent")).unwrap(); + assert_eq!(closed.active.unwrap().task_id, "tj-sub"); + let unknown = session_state(&conn, "p", Some("s1"), Some("tj-nope")).unwrap(); + assert_eq!(unknown.active.unwrap().task_id, "tj-sub"); + } + #[test] fn a_closed_task_is_no_longer_active() { let (_dir, conn) = conn(); write(&conn, "tj-a", EventType::Open, "a", Some("s1")); write(&conn, "tj-a", EventType::Close, "done", Some("s1")); - let s = session_state(&conn, "p", Some("s1")).unwrap(); + let s = session_state(&conn, "p", Some("s1"), None).unwrap(); assert!(s.active.is_none()); assert_eq!(s.open_tasks, 0); } @@ -175,13 +210,13 @@ mod tests { let (_dir, conn) = conn(); write(&conn, "tj-a", EventType::Open, "a", Some("s1")); - let v = serde_json::to_value(session_state(&conn, "p", Some("s1")).unwrap()).unwrap(); + let v = serde_json::to_value(session_state(&conn, "p", Some("s1"), None).unwrap()).unwrap(); assert_eq!(v["schema"], "tj-state/1"); assert_eq!(v["session_id"], "s1"); assert_eq!(v["active"]["task_id"], "tj-a"); assert!(v["active"]["recent"].as_array().unwrap().is_empty()); - let none = serde_json::to_value(session_state(&conn, "p", None).unwrap()).unwrap(); + let none = serde_json::to_value(session_state(&conn, "p", None, None).unwrap()).unwrap(); assert!(none["active"].is_null()); assert!(none["session_id"].is_null()); } diff --git a/plugin/hooks/register.ts b/plugin/hooks/register.ts index 2ef8c03..3d8e700 100644 --- a/plugin/hooks/register.ts +++ b/plugin/hooks/register.ts @@ -11,8 +11,8 @@ // was not logged and records it as suggested events. // // It sets TJ_MOD_ACTIVE=1 for the hooks Claude Code starts, so the classic -// `task-journal ingest-hook` capture and `nudge` stand down instead of -// doing the same work twice. +// hooks skip what the mod does here (the reminder, per-message +// classification, the transcript catch-ups) and keep the rest. import type { EngineInterface, Register } from 'claude-code' @@ -37,64 +37,127 @@ const WORK_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'Bash' // fits a small model's prompt comfortably. const EXCERPT_CHARS = 60_000 +// The longest a compaction waits for the catch-up before going on without it. +const DISTILL_TIMEOUT_MS = 60_000 + +// After a failed read (not a missing or old CLI), wait this long before +// trying again, so a broken journal costs one spawn a minute, not one a request. +const RETRY_AFTER_MS = 60_000 + // What the mod knows about the session it runs in. A module variable: it // lives as long as the process, which is as long as the session. const mod = { isOff: false, hasWarned: false, + // The directory the session started in. The journal's MCP server keeps + // the directory it was started in too, so both resolve the same project + // even after /cd or a worktree move. + cwd: '', state: null as JournalState | null, session: '', + retryAt: 0, + // The task the system prompt names. It changes on task_create, on closing + // it, or with the session — never because the agent logged to another + // task — so the prompt (and the cache behind it) stays put. + pinned: null as string | null, turnsSinceEntry: 0, workSinceEntry: 0, } -async function cli($: EngineInterface, argv: string[]): Promise<string | null> { +type CliResult = + | { kind: 'ok'; stdout: string } + | { kind: 'missing' } + | { kind: 'old' } + | { kind: 'failed'; detail: string } + +async function cli($: EngineInterface, argv: string[]): Promise<CliResult> { + let run try { - const run = await $.process.run(['task-journal', ...argv], { - cwd: await $.session.root(), - timeoutMs: 10_000, + run = await $.process.run(['task-journal', ...argv], { + cwd: mod.cwd || (await $.session.cwd()), + timeoutMs: 30_000, }) + } catch (err) { + const detail = String(err) - return run.exitCode === 0 ? run.stdout : null - } catch { - return null + return /ENOENT|not found|No such file/i.test(detail) ? { kind: 'missing' } : { kind: 'failed', detail } } + + if (run.exitCode === 0) return { kind: 'ok', stdout: run.stdout } + if (/unrecognized subcommand|unexpected argument/i.test(run.stderr)) return { kind: 'old' } + + return { kind: 'failed', detail: run.stderr.trim().split('\n')[0] ?? `exit ${run.exitCode}` } +} + +function warnOnce($: EngineInterface, result: CliResult): void { + if (mod.hasWarned || result.kind === 'ok') return + mod.hasWarned = true + + const text = + result.kind === 'missing' + ? 'task-journal: the `task-journal` CLI is not on PATH, so the journal mod is off. Install it with `cargo install task-journal-cli task-journal-mcp`.' + : result.kind === 'old' + ? 'task-journal: the installed `task-journal` CLI is older than this plugin (0.30+ needed), so the journal mod is off. Update it with `cargo install task-journal-cli task-journal-mcp --force`.' + : `task-journal: reading the journal failed (${result.detail}); the mod will retry in a minute.` + $.ui.log(text) } -// Reads the state of the session the engine names now. The id it was read -// for is kept even when the read fails, so a missing CLI costs one spawn -// per session, not one per request. +// Reads the state of the session the engine names now. A missing or old CLI +// is final for the session; any other failure is retried after a pause. async function refresh($: EngineInterface): Promise<void> { const session = await $.session.id() - const out = await cli($, ['state', '--session', session]) - const state = out === null ? null : parseState(out) - - mod.session = session - mod.state = state + const prefer = mod.session === session && mod.pinned !== null ? ['--prefer', mod.pinned] : [] + const result = await cli($, ['state', '--session', session, ...prefer]) + const state = result.kind === 'ok' ? parseState(result.stdout) : null if (state === null) { - if (!mod.hasWarned) { - mod.hasWarned = true - $.ui.log( - 'task-journal: the task-journal CLI 0.30+ was not found on PATH, so the journal mod is off. Install it with `cargo install task-journal-cli task-journal-mcp --force`.', - ) + warnOnce($, result) + mod.state = null + if (result.kind === 'failed') { + mod.retryAt = (await $.clock.now()) + RETRY_AFTER_MS + } else { + mod.session = session } return } + if (mod.session !== session) mod.pinned = null + mod.session = session + mod.state = state + mod.pinned = state.active?.task_id ?? null $.ui.status(statusText(state)) } // The session id can change under the mod: a /clear starts a new one with // no session.start, and a resumed session may only take its id after -// session.start ran. Re-read whenever the id moved. +// session.start ran. Re-read whenever the id moved, or a retry is due. async function current($: EngineInterface): Promise<JournalState | null> { - if (mod.session !== (await $.session.id())) await refresh($) + const session = await $.session.id() + const isRetryDue = mod.state === null && mod.retryAt > 0 && (await $.clock.now()) >= mod.retryAt + if (mod.session !== session || isRetryDue) { + mod.retryAt = 0 + await refresh($) + } return mod.state } +// Resolves to undefined once `ms` passed. The wait costs the hook nothing +// while the raced call is in flight. +async function within<T>($: EngineInterface, ms: number, call: Promise<T>): Promise<T | undefined> { + let timer: { cancel: () => void } | undefined + const timeout = new Promise<undefined>(resolve => { + timer = $.clock.after(ms, () => resolve(undefined)) + }) + + try { + return await Promise.race([call, timeout]) + } finally { + timer?.cancel() + } +} + // Asks what the conversation decided that the journal lacks. A fork reuses // the session's own cached transcript; a process that has sent nothing yet // (a session resumed straight into /compact) has nothing to fork, and then @@ -104,17 +167,17 @@ async function distill($: EngineInterface, state: JournalState, messages: readon if (task === null) return const ask = distillPrompt(task) - let reply = await $.model.fork({ prompt: ask }) - if (!reply.isAnswered && reply.reason === 'nothing-to-fork') { + let reply = await within($, DISTILL_TIMEOUT_MS, $.model.fork({ prompt: ask })) + if (reply !== undefined && !reply.isAnswered && reply.reason === 'nothing-to-fork') { const conversation = transcriptExcerpt(messages, EXCERPT_CHARS) reply = await $.model.complete({ model: 'haiku', prompt: `<conversation>\n${conversation}\n</conversation>\n\n${ask}`, maxTokens: 1024, - timeoutMs: 60_000, + timeoutMs: DISTILL_TIMEOUT_MS, }) } - if (!reply.isAnswered) return + if (reply === undefined || !reply.isAnswered) return const session = await $.session.id() for (const event of parseDistill(reply.text, task.recent)) { @@ -142,6 +205,7 @@ export const register: Register = (on, options) => { return next(e) } + mod.cwd = e.cwd await refresh($) if (mod.state !== null) await $.env.set('TJ_MOD_ACTIVE', '1') @@ -151,6 +215,8 @@ export const register: Register = (on, options) => { on('session.end', async ($, e, next) => { mod.state = null mod.session = '' + mod.pinned = null + mod.retryAt = 0 mod.turnsSinceEntry = 0 mod.workSinceEntry = 0 @@ -203,6 +269,10 @@ export const register: Register = (on, options) => { const ran = await next(call) if (ran.deny !== undefined || ran.isError === true || !WRITE_TOOLS.has(name)) return ran + // A new task, or closing the named one, moves the pin; logging to + // another task does not. + if (name === 'task_create' || (name === 'task_close' && args.task_id === mod.pinned)) mod.pinned = null + mod.turnsSinceEntry = 0 mod.workSinceEntry = 0 await refresh($) @@ -217,7 +287,9 @@ export const register: Register = (on, options) => { const state = await current($) if (state === null || state.active === null) return next(e) - if (distillOnCompact) { + // A precompute builds a summary ahead of time that may never be used; + // the catch-up waits for a compaction that happens. + if (distillOnCompact && e.trigger !== 'precompute') { try { await distill($, state, e.messages) } catch { From 484ea71913a3bde9845a0707bee3bc466b7622dc Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:21:49 +0400 Subject: [PATCH 087/104] fix(hook): keep push-recall and /rewind when the mod is active --- crates/tj-cli/src/main.rs | 20 ++++++----- crates/tj-cli/tests/cli.rs | 74 +++++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d9b01d8..b229efc 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -2438,14 +2438,13 @@ fn real_main() -> Result<()> { }; // The Claude Code mod captures in-process and marks the hooks it - // starts; the classic capture stands down instead of doing the - // same work twice. Resume packs and model switches stay ours. - if mod_active() - && matches!( - kind.as_str(), - "UserPromptSubmit" | "PostToolUse" | "Stop" | "PreCompact" | "SessionEnd" - ) - { + // starts; the classic capture stands down where the mod does the + // same work: the Stop / PreCompact / SessionEnd transcript + // catch-up (and the compaction marker) here, per-message queueing + // and auto-open further down. Push-recall, the `/rewind` + // correction, resume packs and model switches stay ours. + let mod_on = mod_active(); + if mod_on && matches!(kind.as_str(), "Stop" | "PreCompact" | "SessionEnd") { return Ok(()); } @@ -3004,6 +3003,11 @@ runs in the background and won't block you; it only fills gaps and never closes return Ok(()); } + // The mod classifies prompts and tool calls itself (see above). + if mod_on && matches!(kind.as_str(), "UserPromptSubmit" | "PostToolUse") { + return Ok(()); + } + // A tool call arrives as its full input + response JSON; cap it // so one big file read or command output can't flood the queue. let text: String = if kind == "PostToolUse" { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a68d49f..bb00dc7 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -2249,7 +2249,8 @@ fn hook_leaves_a_trace(kind: &str, mod_value: Option<&str>) -> bool { } /// The Claude Code mod captures in-process and sets TJ_MOD_ACTIVE for the -/// hooks it starts; the classic capture must then stand down. +/// hooks it starts; the classic capture must then stand down. (Nothing here +/// echoes a prior entry, so push-recall stays quiet too.) #[test] fn mod_active_silences_the_classic_capture_hooks() { for kind in [ @@ -2273,6 +2274,77 @@ fn mod_active_keeps_session_start_and_model_switch() { } } +/// The mod takes over capture, not push-recall: a tool call that echoes a +/// prior rejection still gets the recall on both paths, and nothing is queued. +#[test] +fn mod_active_keeps_push_recall() { + let (dir, _task_id) = seed_axum_rejection(); + + for tool in ["Bash", "mcp__x__do"] { + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": tool, + "tool_input": { "command": "let's switch the server to axum" }, + "tool_response": { "output": "" } + }); + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .env("TJ_MOD_ACTIVE", "1") + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin(payload.to_string()) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); + assert!(out.contains("previously rejected"), "{tool}: {out}"); + } + + let pending = std::fs::read_dir(dir.path().join("task-journal").join("pending")) + .map(|d| d.count()) + .unwrap_or(0); + assert_eq!(pending, 0, "the mod queues tool calls itself"); +} + +/// `/rewind` is the user rolling back a path — the mod doesn't record that, +/// so the correction event still lands. +#[test] +fn mod_active_keeps_the_rewind_correction() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + let hash = tj_core::project_hash::from_path(proj.path()).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Rewind host"]) + .assert() + .success(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_DISABLE_CLASSIFY_SPAWN", "1") + .env("TJ_MOD_ACTIVE", "1") + .current_dir(proj.path()) + .args(["ingest-hook", "--backend", "heuristic"]) + .write_stdin( + serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "prompt": "/rewind go back to plan A", + }) + .to_string(), + ) + .assert() + .success(); + + let journal = project_events(xdg.path(), &hash); + assert_eq!(session_ids_of(&journal, "correction").len(), 1, "{journal}"); +} + #[test] fn mod_active_silences_the_nudge() { for (mod_value, silent) in [(Some("1"), true), (Some("0"), false), (None, false)] { From 97694b3b8b4338ae6af536ad6030a226edb16e94 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:26:58 +0400 Subject: [PATCH 088/104] fix(plugin): keep an empty hooks record so pre-mod Claude Code accepts hooks.json --- plugin/hooks/hooks.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin/hooks/hooks.json b/plugin/hooks/hooks.json index 9f45101..b614c24 100644 --- a/plugin/hooks/hooks.json +++ b/plugin/hooks/hooks.json @@ -1,3 +1,4 @@ { - "modules": ["./register.ts"] + "modules": ["./register.ts"], + "hooks": {} } From 76db3c86df26769be1e501ed6d556d78390b6288 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:27:20 +0400 Subject: [PATCH 089/104] docs: what the mod replaces, old Claude Code, and restarting sessions after an upgrade --- INSTALL.md | 30 +++++++++++++++++++++++------- README.md | 7 +++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 39b4235..cd63c58 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -65,13 +65,18 @@ Two options live in `/config` (or `claude plugin configure task-journal`): `distill_on_compact` (default on). The mod calls the `task-journal` CLI, so the binaries must be installed and -current (0.30+). If they aren't, it says so once and stays off. - -**Classic hooks next to the mod.** If you ran `task-journal install-hooks` with -`--auto-capture` before, you can keep it: the mod tells those hooks it is active -(`TJ_MOD_ACTIVE=1`) and they stand down, except the session-start resume pack and -the model-switch record. To remove them anyway: -`task-journal install-hooks --scope user --uninstall`. +current (0.30+). If they aren't, it says so once and stays off. On Claude Code +older than 2.1.287 the mod doesn't load; the MCP tools and the skill work as +before. + +**Classic hooks next to the mod.** If you ran `task-journal install-hooks` +before, you can keep it. The mod tells those hooks it is active +(`TJ_MOD_ACTIVE=1`), and they skip what the mod replaces — the reminder, +per-message classification (`--auto-capture`) and the transcript catch-ups at +stop, compaction and `/clear` (the mod catches up once, right before each +compaction). The rest keeps running: the session-start resume +pack, the model-switch record, push-recall and `/rewind` corrections. To remove +them anyway: `task-journal install-hooks --scope user --uninstall`. ### Older Claude Code, or without the plugin @@ -141,6 +146,17 @@ it with `--project-dir <path>` if the client launches it elsewhere. A client tha knows its session id can pass `session_id` to the write tools so each session keeps its own active task. +## Upgrading + +```bash +cargo install task-journal-cli task-journal-mcp --force +``` + +Then restart every open Claude Code and Codex session, so no old MCP server +keeps writing next to the new one. The first command after the upgrade +re-indexes the project once (a few seconds for a large journal); later calls +are incremental. + ## Verify ```bash diff --git a/README.md b/README.md index 08c2dc8..f6db340 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ that tie it back to the code. ## How it works -- **The mod (Claude Code 2.1.287+).** A plugin of function hooks that runs inside Claude Code and calls the `task-journal` CLI. It keeps the session's active task in the system prompt (a compaction can't drop it), gives each session its own active task, shows the task in the status line, reminds the agent to log only after several prompts without an entry, and — right before a compaction — asks the model what was never logged and records it as `suggested` events. It needs no setup; when it runs, the classic capture hooks stand down so nothing is recorded twice. Codex and older Claude Code keep using the hooks below. +- **The mod (Claude Code 2.1.287+).** A plugin of function hooks that runs inside Claude Code and calls the `task-journal` CLI. It keeps the session's active task in the system prompt (a compaction can't drop it), gives each session its own active task, shows the task in the status line, reminds the agent to log only after several prompts without an entry, and — right before a compaction — asks the model what was never logged and records it as `suggested` events. It needs no setup. When it runs, the classic hooks skip what it replaces (the reminder, per-message classification and the transcript catch-ups) and keep the rest (resume packs, push-recall, `/rewind`). Codex and Claude Code older than 2.1.287 keep using the hooks below; there the mod simply doesn't load. - **Self-tagging is the primary path (recommended).** You — the agent in the live session — record reasoning directly via the five MCP tools: open a task with a `goal`, append a typed `decision` / `finding` / `rejection` / `evidence` event at the moment of commitment, and `task_close` with a written `outcome`. This is free (it rides the interactive session), language-agnostic, and higher-fidelity than any after-the-fact classifier. The bundled `task-journal` skill drives this automatically. See [MCP tools](#mcp-tools). - **Auto-capture is an opt-in backstop — OFF by default (v0.14.0).** A fresh `install-hooks` wires only a cheap, read-only SessionStart resume hook: no per-message classifier runs, no `claude -p` is ever spawned, nothing is charged. Self-tagging is the capture mechanism. Opt in with `install-hooks --auto-capture` and Claude Code hooks also run every prompt, tool call, and reply through a two-stage classifier that lands typed events on its own: Stage 1 is a fast in-process heuristic (obvious EN+RU phrasing, zero cost); Stage 2 falls back to an LLM only when the heuristic is uncertain (and only if you pick `--backend agent-sdk` / `api`). Even opted in it is a safety net under your explicit self-tagging, not the main mechanism, and it misses real reasoning — especially non-English prose. - **Artifact extraction.** Each event scans its text for commit hashes, PR URLs, file paths, issue IDs, and branch names. Aggregated artifacts are how Task Journal links related tasks: when you start a new task touching the same issue or file, the prior task is surfaced automatically. @@ -319,12 +319,15 @@ Plugin install: /plugin update task-journal@task-journal ``` -Then refresh the binaries (the plugin doesn't bundle them): +Then refresh the binaries (the plugin doesn't bundle them) and restart every +open Claude Code and Codex session: ```bash cargo install task-journal-cli task-journal-mcp --force ``` +The first command after an upgrade may re-index the project once. + Restart Claude Code and verify: ```bash From 91e5fe4520f3022bb8ffdc1358bf83d1460cbd52 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:28:58 +0400 Subject: [PATCH 090/104] fix(worker): use an OS lock and claim pending entries atomically --- Cargo.lock | 2 +- crates/tj-cli/Cargo.toml | 4 +- crates/tj-cli/src/main.rs | 179 ++++++++++++------------------ crates/tj-cli/tests/cli.rs | 222 ++++++++++++++++++++++++++----------- 4 files changed, 230 insertions(+), 177 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a60fdf..50afee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2524,7 +2524,7 @@ dependencies = [ "chrono", "clap", "crossterm", - "libc", + "fd-lock", "mockito", "predicates", "ratatui", diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index 7ad5c3d..a76aa86 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -36,9 +36,7 @@ chrono = { workspace = true } ratatui = { workspace = true } crossterm = { workspace = true } tempfile = { workspace = true } - -[target.'cfg(unix)'.dependencies] -libc = "0.2" +fd-lock = { workspace = true } [dev-dependencies] assert_fs = { workspace = true } diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index b229efc..72f03e9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -5786,100 +5786,33 @@ fn spawn_classify_worker(backend: &str) -> anyhow::Result<()> { /// Characters of a PostToolUse chunk (tool input + response) that get queued. const POST_TOOL_USE_TEXT_MAX: usize = 2000; -/// How long an empty (pid not yet written) worker lockfile counts as held. -const LOCK_PID_GRACE: std::time::Duration = std::time::Duration::from_secs(5); - -/// File-lock guard for the classify-worker. Holds the lockfile until -/// dropped; ensures cleanup on panic. One worker per project_hash. -struct WorkerLock { - path: std::path::PathBuf, -} - -impl WorkerLock { - /// Try to acquire the lock. Returns Ok(Some(_)) on success, Ok(None) - /// if another live worker holds it, Err on filesystem failure. - fn try_acquire(project_hash: &str) -> anyhow::Result<Option<Self>> { - let dir = tj_core::paths::state_dir()?; - std::fs::create_dir_all(&dir)?; - let path = dir.join(format!("classifier-{project_hash}.lock")); - - loop { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&path) - { - Ok(mut f) => { - use std::io::Write; - let _ = writeln!(f, "{}", std::process::id()); - return Ok(Some(Self { path })); - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Inspect existing lockfile. If PID is alive → another - // worker is running; back off. If dead → remove stale - // file and retry. - let body = std::fs::read_to_string(&path).unwrap_or_default(); - match body.trim().parse::<u32>() { - Ok(pid) if pid_is_alive(pid) => return Ok(None), - Ok(_) => {} - Err(_) => { - // No PID yet: the holder may sit between - // `create_new` and the pid write. Only a lock - // that stayed empty past the grace is stale. - let fresh = std::fs::metadata(&path) - .and_then(|m| m.modified()) - .map(|t| t.elapsed().unwrap_or_default() < LOCK_PID_GRACE) - .unwrap_or(false); - if fresh { - return Ok(None); - } - } - } - let _ = std::fs::remove_file(&path); - continue; - } - Err(e) => return Err(e.into()), - } - } - } -} - -impl Drop for WorkerLock { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } -} - -#[cfg(unix)] -fn pid_is_alive(pid: u32) -> bool { - // kill(pid, 0) probes existence without sending a signal. - // SAFETY: libc::kill is a thin syscall wrapper, no aliasing concerns. - if unsafe { libc::kill(pid as libc::pid_t, 0) } == 0 { - return true; - } - - // EPERM: the process exists but belongs to another user. - std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -#[cfg(not(unix))] -fn pid_is_alive(_pid: u32) -> bool { - // Conservative on non-Unix: assume alive so we don't double-spawn. - // The lockfile gets cleaned up on Drop in the normal exit path. - true -} - /// classify-worker: drain pending v2 entries by running the real /// classifier. v1 entries (legacy text+error shape) are left for -/// `pending retry`. Holds a project-scoped file lock so only one +/// `pending retry`. Holds a project-scoped OS lock so only one /// worker per project runs at a time. fn run_classify_worker(backend: &str) -> Result<()> { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; - let lock = match WorkerLock::try_acquire(&project_hash)? { - Some(l) => l, - None => return Ok(()), // another worker is running + // An advisory lock held for the worker's lifetime. The OS releases it + // when the process exits or dies, so a leftover file never blocks the + // next worker and needs no stale-pid check. The file is never deleted: + // a worker could lock the unlinked file while another creates and + // locks a fresh one. + let state_dir = tj_core::paths::state_dir()?; + std::fs::create_dir_all(&state_dir)?; + let lock_file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(state_dir.join(format!("classifier-{project_hash}.lock")))?; + let mut lock = fd_lock::RwLock::new(lock_file); + let _held = match lock.try_write() { + Ok(guard) => guard, + // Another worker of this project is running. + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(e) => return Err(e.into()), }; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); @@ -5889,10 +5822,28 @@ fn run_classify_worker(backend: &str) -> Result<()> { .ok_or_else(|| anyhow::anyhow!("events_dir has no grandparent"))? .join("pending"); if !pending.exists() { - drop(lock); return Ok(()); } + // A worker that died mid-entry left it claimed. We hold the lock, so no + // other worker of this project is on it: put it back in the queue. + for claimed in std::fs::read_dir(&pending)?.flatten().map(|e| e.path()) { + let Some(name) = claimed + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".processing")) + else { + continue; + }; + let ours = std::fs::read_to_string(&claimed) + .ok() + .and_then(|b| serde_json::from_str::<serde_json::Value>(&b).ok()) + .is_some_and(|v| v["project_hash"] == project_hash.as_str()); + if ours { + let _ = std::fs::rename(&claimed, pending.join(name)); + } + } + // Snapshot entries up front so concurrent re-queues don't loop us. // Only this project's entries: the lock is per project, so a worker // of another project may be draining the same directory right now. @@ -5907,7 +5858,6 @@ fn run_classify_worker(backend: &str) -> Result<()> { } } - drop(lock); Ok(()) } @@ -5946,22 +5896,37 @@ fn process_pending_entry( // Inherit the session id queued on the v2 chunk (additive; absent → None). let chunk_session_id = tj_core::session_id::session_id_from_payload(&v); - let classifier = build_classifier(backend)?; - let outcome = classify_chunk( - classifier.as_ref(), - events_path, - project_hash, - &kind, - &text, - chunk_session_id.as_deref(), - )?; - if let ChunkOutcome::Unplaced(err) = outcome { - // Persist as legacy v1 pending entry so `pending retry` - // surfaces it; remove the v2 source. - persist_pending(events_path, project_hash, &kind, &text, &err)?; + // Claim the entry before the classifier call: a rename succeeds for one + // process only, so an entry is never classified (and recorded) twice. + let claimed = path.with_extension("json.processing"); + if std::fs::rename(path, &claimed).is_err() { + return Ok(()); // someone else took it + } + + let classified = (|| -> anyhow::Result<()> { + let classifier = build_classifier(backend)?; + let outcome = classify_chunk( + classifier.as_ref(), + events_path, + project_hash, + &kind, + &text, + chunk_session_id.as_deref(), + )?; + if let ChunkOutcome::Unplaced(err) = outcome { + // Persist as legacy v1 pending entry so `pending retry` + // surfaces it; remove the v2 source. + persist_pending(events_path, project_hash, &kind, &text, &err)?; + } + Ok(()) + })(); + if classified.is_err() { + // Back in the queue under its own name for the next worker. + let _ = std::fs::rename(&claimed, path); + return classified; } - std::fs::remove_file(path)?; + std::fs::remove_file(&claimed)?; Ok(()) } @@ -6832,14 +6797,6 @@ mod inline_tests { ); } - /// PID 1 always exists; for a non-root user `kill(1, 0)` fails with EPERM, - /// which means "alive, not ours" — never "dead". - #[cfg(unix)] - #[test] - fn pid_is_alive_treats_eperm_as_alive() { - assert!(pid_is_alive(1)); - } - #[test] fn persist_pending_v2_omits_session_id_when_none() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index bb00dc7..eea95a9 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -3850,28 +3850,11 @@ fn classify_worker_handles_classifier_failure_cleanly() { .args(["classify-worker", "--backend", "hybrid"]) .assert() .success(); - - // Lockfile must not be left behind. - let state = dir.path().join("task-journal").join("state"); - if state.exists() { - for e in std::fs::read_dir(&state).unwrap() { - let p = e.unwrap().path(); - assert!( - !p.file_name() - .and_then(|n| n.to_str()) - .unwrap_or("") - .ends_with(".lock"), - "lockfile must be removed after worker exits, found: {p:?}" - ); - } - } } -/// Lockfile prevents concurrent workers in the same project. We can't -/// easily race two real spawns deterministically in a unit test, so -/// instead simulate a held lock by writing a lockfile with our own -/// (live) PID, then run classify-worker and assert it exits cleanly -/// without draining the queue. +/// One worker per project, enforced by an OS lock on the lock file, not by +/// what the file says: while another process holds the lock, a worker +/// leaves the queue alone — even when the file names a dead pid. #[test] fn classify_worker_respects_existing_lock() { let dir = assert_fs::TempDir::new().unwrap(); @@ -3903,13 +3886,19 @@ fn classify_worker_respects_existing_lock() { ) .unwrap(); - // Hand-roll a lockfile with this process's (live) PID. The - // worker should see the live PID and bail without touching the - // pending entry. + // Hold the lock from this process, over a file naming a dead pid. let state = dir.path().join("task-journal").join("state"); std::fs::create_dir_all(&state).unwrap(); let lock_path = state.join(format!("classifier-{project_hash}.lock")); - std::fs::write(&lock_path, format!("{}\n", std::process::id())).unwrap(); + std::fs::write(&lock_path, format!("{}\n", i32::MAX)).unwrap(); + let mut lock = fd_lock::RwLock::new( + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(), + ); + let _held = lock.try_write().unwrap(); Command::cargo_bin("task-journal") .unwrap() @@ -3932,12 +3921,12 @@ fn classify_worker_respects_existing_lock() { ); } -/// The lock is created empty and the pid is written a moment later. A -/// second worker that reads it in between must treat it as held, not as a -/// stale lock to delete and steal. An empty lock that has sat for a while -/// is a crashed worker's leftover and is taken over. +/// A lock file nobody holds is a finished or crashed worker's leftover — +/// whatever it says (empty, or a pid that is alive again under another +/// program) the next worker takes over: the OS released the lock with the +/// process. #[test] -fn classify_worker_treats_fresh_empty_lock_as_held() { +fn classify_worker_takes_over_a_leftover_lock_file() { let dir = assert_fs::TempDir::new().unwrap(); let cwd = std::env::current_dir().unwrap(); let project_hash = tj_core::project_hash::from_path(&cwd).expect("compute project hash"); @@ -3950,54 +3939,163 @@ fn classify_worker_treats_fresh_empty_lock_as_held() { .join("events") .join(format!("{project_hash}.jsonl")); std::fs::create_dir_all(events_path.parent().unwrap()).unwrap(); - let entry = pending.join("01emptylock.json"); - std::fs::write( - &entry, - serde_json::json!({ - "schema": "v2", - "kind": "PostToolUse", - "text": "empty lock marker", - "project_hash": project_hash, - "events_path": events_path.to_string_lossy(), - "backend": "heuristic", - "queued_at": "2026-05-08T00:00:00Z", - }) - .to_string(), - ) - .unwrap(); - let state = dir.path().join("task-journal").join("state"); std::fs::create_dir_all(&state).unwrap(); let lock_path = state.join(format!("classifier-{project_hash}.lock")); - std::fs::write(&lock_path, "").unwrap(); - let worker = || { + for leftover in [String::new(), format!("{}\n", std::process::id())] { + let entry = pending.join("01leftoverlock.json"); + std::fs::write( + &entry, + serde_json::json!({ + "schema": "v2", + "kind": "PostToolUse", + "text": "leftover lock marker", + "project_hash": project_hash, + "events_path": events_path.to_string_lossy(), + "backend": "heuristic", + "queued_at": "2026-05-08T00:00:00Z", + }) + .to_string(), + ) + .unwrap(); + std::fs::write(&lock_path, &leftover).unwrap(); + Command::cargo_bin("task-journal") .unwrap() .env("XDG_DATA_HOME", dir.path()) .args(["classify-worker", "--backend", "heuristic"]) .assert() .success(); + + assert!(!entry.exists(), "lock file {leftover:?} is taken over"); + } +} + +/// Workers started at once on one queue — over a crashed worker's leftover +/// lock — record each entry exactly once: no duplicate classifier calls, no +/// duplicate events. +#[test] +fn classify_workers_started_together_record_each_entry_once() { + const ENTRIES: usize = 12; + const WORKERS: usize = 8; + + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Worker race host"]) + .assert() + .success(); + + let pending = dir.path().join("task-journal").join("pending"); + for i in 0..ENTRIES { + write_pending_entry( + &pending, + &format!("{hash}.01JB{i:022}.json"), + Some(&hash), + true, + "UserPromptSubmit", + &format!("We decided to use store-{i:02} for the cache"), + ); + } + let state = dir.path().join("task-journal").join("state"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write( + state.join(format!("classifier-{hash}.lock")), + format!("{}\n", i32::MAX), + ) + .unwrap(); + + let start = std::sync::Barrier::new(WORKERS); + std::thread::scope(|s| { + for _ in 0..WORKERS { + s.spawn(|| { + let mut cmd = Command::cargo_bin("task-journal").unwrap(); + cmd.env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]); + start.wait(); + cmd.assert().success(); + }); + } + }); + + // The race can leave an entry to the next worker, never record it twice. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + + let journal = project_events(dir.path(), &hash); + for i in 0..ENTRIES { + let marker = format!("store-{i:02}"); + let n = journal.lines().filter(|l| l.contains(&marker)).count(); + assert_eq!(n, 1, "{marker} recorded {n} times:\n{journal}"); + } +} + +/// A worker claims an entry (`<name>.processing`) before classifying it. A +/// failed entry goes back under its own name; one a crashed worker left +/// claimed is put back in the queue by the next worker and recorded. +#[test] +fn classify_worker_requeues_failed_and_orphaned_claims() { + let dir = assert_fs::TempDir::new().unwrap(); + let workdir = dir.path().join("proj"); + std::fs::create_dir_all(&workdir).unwrap(); + let hash = tj_core::project_hash::from_path(&workdir).unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["create", "Claim host"]) + .assert() + .success(); + let pending = dir.path().join("task-journal").join("pending"); + let worker = |backend: &str| { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .current_dir(&workdir) + .args(["classify-worker", "--backend", backend]) + .assert() + .success(); }; - worker(); - assert!( - entry.exists(), - "a fresh empty lock is held — entry must stay" + let queued = write_pending_entry( + &pending, + &format!("{hash}.01JC0000000000000000000000.json"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use store-failed for the cache", ); - assert!(lock_path.exists(), "a fresh empty lock must not be deleted"); + worker("no-such-backend"); + assert!(queued.exists(), "a failed entry is back under its own name"); - let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60); - std::fs::File::options() - .write(true) - .open(&lock_path) - .unwrap() - .set_modified(old) - .unwrap(); + let orphan = write_pending_entry( + &pending, + &format!("{hash}.01JC0000000000000000000001.json.processing"), + Some(&hash), + true, + "UserPromptSubmit", + "We decided to use store-orphan for the cache", + ); + worker("heuristic"); - worker(); - assert!(!entry.exists(), "a stale empty lock is taken over"); - assert!(!lock_path.exists(), "the new holder releases the lock"); + assert!(!queued.exists() && !orphan.exists()); + let journal = project_events(dir.path(), &hash); + for marker in ["store-failed", "store-orphan"] { + let n = journal.lines().filter(|l| l.contains(marker)).count(); + assert_eq!(n, 1, "{marker} recorded {n} times:\n{journal}"); + } } // ---------------- pending/ is global: every consumer is project-scoped ---------------- From 6a973404b795a0d85cb61560dc08fa51469e1395 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:32:25 +0400 Subject: [PATCH 091/104] fix(migrate): re-key queued chunks and recall log, drop stale WAL, back up replaced journal --- crates/tj-cli/src/main.rs | 58 ++++++++++++++++ crates/tj-cli/tests/cli.rs | 135 ++++++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 72f03e9..3320d80 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -186,6 +186,10 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool state_dir.join(format!("{from_hash}.sqlite-shm")), state_dir.join(format!("{to_hash}.sqlite-shm")), ), + ( + state_dir.join(format!("{from_hash}.recall-shown")), + state_dir.join(format!("{to_hash}.recall-shown")), + ), ( metrics_dir.join(format!("{from_hash}.jsonl")), metrics_dir.join(format!("{to_hash}.jsonl")), @@ -214,6 +218,29 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool .with_context(|| format!("checkpoint WAL of {src_state_path:?}"))?; } + // --force: the JSONL is the source of truth, so keep the journal being + // replaced. A copy, so a failed move still leaves the destination intact. + let (src_jsonl, dst_jsonl) = &pairs[0]; + let mut backup = None; + if src_jsonl.exists() && dst_jsonl.metadata().is_ok_and(|m| m.len() > 0) { + let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ"); + let bak = dst_jsonl.with_file_name(format!("{to_hash}.jsonl.bak-{stamp}")); + std::fs::copy(dst_jsonl, &bak).with_context(|| format!("back up {dst_jsonl:?}"))?; + backup = Some(bak); + } + + // --force: a destination `-wal` / `-shm` belongs to the database being + // replaced. Left next to the one moving in, SQLite would replay it into + // it; the source's own sidecars, if any, are moved in below. + if src_state_path.exists() { + for suffix in ["-wal", "-shm"] { + let stale = state_dir.join(format!("{to_hash}.sqlite{suffix}")); + if stale.exists() { + std::fs::remove_file(&stale).with_context(|| format!("remove {stale:?}"))?; + } + } + } + let mut moved: Vec<String> = Vec::new(); for (src, dst) in &pairs { if !src.exists() { @@ -229,6 +256,34 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool moved.push(dst.display().to_string()); } + // Queued chunks carry the hash in their name and, for the worker's + // ownership check, in their `project_hash`. + let pending_dir = events_dir.with_file_name("pending"); + if pending_dir.exists() { + let prefix = format!("{from_hash}."); + for src in std::fs::read_dir(&pending_dir)?.flatten().map(|e| e.path()) { + let Some(rest) = src + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_prefix(&prefix)) + else { + continue; + }; + let dst = pending_dir.join(format!("{to_hash}.{rest}")); + + let body = std::fs::read_to_string(&src)?; + if let Ok(mut v) = serde_json::from_str::<serde_json::Value>(&body) { + if v["project_hash"] == from_hash.as_str() { + v["project_hash"] = to_hash.clone().into(); + std::fs::write(&src, serde_json::to_string_pretty(&v)?)?; + } + } + + std::fs::rename(&src, &dst).with_context(|| format!("rename {src:?} -> {dst:?}"))?; + moved.push(dst.display().to_string()); + } + } + // Re-key the project_hash columns inside the (now renamed) SQLite. let new_state_path = state_dir.join(format!("{to_hash}.sqlite")); if new_state_path.exists() { @@ -259,6 +314,9 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool } println!(" project_hash {from_hash} -> {to_hash}"); } + if let Some(bak) = backup { + println!("kept the replaced journal as {}", bak.display()); + } Ok(()) } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index eea95a9..c77b45d 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1124,10 +1124,12 @@ fn migrate_project_force_keeps_destination_when_the_move_fails() { ); } +/// The JSONL is the source of truth: `--force` keeps the journal it replaces +/// as `<dst>.jsonl.bak-<timestamp>` and says where it went. #[test] fn migrate_project_force_replaces_existing_destination() { let xdg = assert_fs::TempDir::new().unwrap(); - let (proj_a, proj_b, _, _) = two_projects(); + let (proj_a, proj_b, _, to_hash) = two_projects(); for (proj, title) in [(&proj_a, "Source task"), (&proj_b, "Overwritten task")] { Command::cargo_bin("task-journal") @@ -1139,7 +1141,23 @@ fn migrate_project_force_replaces_existing_destination() { .success(); } - migrate(xdg.path(), proj_a.path(), proj_b.path(), true); + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + "--force", + ]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let out = String::from_utf8(out).unwrap(); Command::cargo_bin("task-journal") .unwrap() @@ -1149,6 +1167,119 @@ fn migrate_project_force_replaces_existing_destination() { .assert() .success() .stdout(contains("Source task").and(contains("Overwritten task").not())); + + let events = xdg.path().join("task-journal").join("events"); + let backup = std::fs::read_dir(&events) + .unwrap() + .map(|e| e.unwrap().path()) + .find(|p| { + p.file_name() + .unwrap() + .to_string_lossy() + .starts_with(&format!("{to_hash}.jsonl.bak-")) + }) + .unwrap_or_else(|| panic!("no backup of the replaced journal; stdout: {out}")); + assert!(std::fs::read_to_string(&backup) + .unwrap() + .contains("Overwritten task")); + assert!(out.contains(&backup.display().to_string()), "{out}"); +} + +/// A destination WAL left behind by the database `--force` replaces must not +/// be replayed into the one moving in. +#[test] +fn migrate_project_force_drops_the_replaced_databases_wal() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let state = xdg.path().join("task-journal").join("state"); + + for (proj, title) in [(&proj_a, "Source task"), (&proj_b, "Overwritten task")] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", title]) + .assert() + .success(); + } + drop(tj_core::db::open(state.join(format!("{from_hash}.sqlite"))).unwrap()); + + // A crash leftover: the destination's WAL with a write never folded in. + let dst_db = state.join(format!("{to_hash}.sqlite")); + let dst_wal = state.join(format!("{to_hash}.sqlite-wal")); + { + let conn = tj_core::db::open(&dst_db).unwrap(); + conn.execute_batch("PRAGMA wal_autocheckpoint=0;").unwrap(); + conn.execute( + "INSERT INTO dream_state(project_hash, last_dream_at, updated_at) VALUES ('stale-wal', 't', 't')", + [], + ) + .unwrap(); + let wal = std::fs::read(&dst_wal).unwrap(); + drop(conn); + std::fs::write(&dst_wal, wal).unwrap(); + } + + migrate(xdg.path(), proj_a.path(), proj_b.path(), true); + + let conn = rusqlite::Connection::open(&dst_db).unwrap(); + let check: String = conn + .query_row("PRAGMA integrity_check", [], |r| r.get(0)) + .unwrap(); + assert_eq!(check, "ok"); + let stale: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dream_state WHERE project_hash = 'stale-wal'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(stale, 0, "the replaced database's WAL was replayed"); +} + +/// Queued chunks and the push-recall log are keyed by the project hash too. +/// After a move the new project's worker classifies what the old one queued. +#[test] +fn migrate_project_rekeys_pending_entries_and_the_recall_log() { + let xdg = assert_fs::TempDir::new().unwrap(); + let (proj_a, proj_b, from_hash, to_hash) = two_projects(); + let pending = xdg.path().join("task-journal").join("pending"); + let state = xdg.path().join("task-journal").join("state"); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Queue host"]) + .assert() + .success(); + let queued = write_pending_entry( + &pending, + &format!("{from_hash}.01JD0000000000000000000000.json"), + Some(&from_hash), + true, + "UserPromptSubmit", + "We decided to use store-moved for the cache", + ); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write(state.join(format!("{from_hash}.recall-shown")), "s1 e1\n").unwrap(); + + migrate(xdg.path(), proj_a.path(), proj_b.path(), false); + + assert!(!queued.exists(), "entry left under the old hash"); + assert_eq!( + std::fs::read_to_string(state.join(format!("{to_hash}.recall-shown"))).unwrap(), + "s1 e1\n" + ); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_b.path()) + .args(["classify-worker", "--backend", "heuristic"]) + .assert() + .success(); + let journal = project_events(xdg.path(), &to_hash); + assert!(journal.contains("store-moved"), "{journal}"); } #[test] From abaf96c195b748b676963b2027b7e483119bfcff Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:44:54 +0400 Subject: [PATCH 092/104] fix(db): replay the log in linear time and in resumable chunks --- crates/tj-core/src/artifacts.rs | 58 ++--- crates/tj-core/src/db.rs | 431 ++++++++++++++++++++++---------- 2 files changed, 329 insertions(+), 160 deletions(-) diff --git a/crates/tj-core/src/artifacts.rs b/crates/tj-core/src/artifacts.rs index d5bf53e..83cfa22 100644 --- a/crates/tj-core/src/artifacts.rs +++ b/crates/tj-core/src/artifacts.rs @@ -11,6 +11,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; +use std::sync::LazyLock; /// Structured artifacts collected from one or many events. All vectors /// are deduplicated (case-sensitive) by the `merge` constructor — the @@ -102,8 +103,10 @@ pub fn extract(text: &str) -> Artifacts { // Word boundary on \b avoids matching inside longer non-hex tokens // (e.g. ULIDs are base32, but adjacent digits + letters could // technically pass — the boundary keeps matches clean). + static COMMIT_HASH: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"\b[0-9a-f]{7,40}\b").unwrap()); static_re( - r"\b[0-9a-f]{7,40}\b", + &COMMIT_HASH, |m| { // Reject if all-digits (could be a year, an ID, a port). // A real abbreviated commit always has at least one letter. @@ -116,31 +119,28 @@ pub fn extract(text: &str) -> Artifacts { ); // GitHub / GitLab PR URLs. - static_re( - r"https?://[A-Za-z0-9.\-]+/[A-Za-z0-9_./\-]+/(?:pull|merge_requests)/\d+", - |m| a.pr_urls.push(m.to_string()), - text, - ); + static PR_URL: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"https?://[A-Za-z0-9.\-]+/[A-Za-z0-9_./\-]+/(?:pull|merge_requests)/\d+") + .unwrap() + }); + static_re(&PR_URL, |m| a.pr_urls.push(m.to_string()), text); // Short PR references: "PR #51", "PR#51", "pull request #51". Anchored to // the PR / "pull request" keyword so a bare "#3" in prose (step #3, issue // #3) is NOT captured. Normalised to "PR #<n>" so it dedupes cleanly and // renders next to full URLs under the same `PRs:` group. - if let Ok(re) = Regex::new(r"(?i)\b(?:PR|pull request)\s*#(\d+)\b") { - for cap in re.captures_iter(text) { - if let Some(m) = cap.get(1) { - a.pr_urls.push(format!("PR #{}", m.as_str())); - } + static PR_REF: LazyLock<Regex> = + LazyLock::new(|| Regex::new(r"(?i)\b(?:PR|pull request)\s*#(\d+)\b").unwrap()); + for cap in PR_REF.captures_iter(text) { + if let Some(m) = cap.get(1) { + a.pr_urls.push(format!("PR #{}", m.as_str())); } } // Ticket IDs: ABC-123. At least 2 letters to avoid matching version // strings like v1-2 and minimum 1 digit. - static_re( - r"\b[A-Z]{2,}-\d+\b", - |m| a.linked_issues.push(m.to_string()), - text, - ); + static TICKET: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\b[A-Z]{2,}-\d+\b").unwrap()); + static_re(&TICKET, |m| a.linked_issues.push(m.to_string()), text); // File paths — heuristic: path-like tokens with at least one slash // (and an extension) OR a leading ./ . Path segments allow a @@ -148,8 +148,11 @@ pub fn extract(text: &str) -> Artifacts { // etc are captured as artifacts. Tight enough to skip prose, loose // enough to catch the common cases (src/foo.rs, ./bar.ts, // crates/tj-core/src/db.rs). + static FILE_PATH: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"(?:\./|\.?[A-Za-z0-9_\-]+/)+[A-Za-z0-9_.\-]+\.[A-Za-z0-9]{1,8}\b").unwrap() + }); static_re( - r"(?:\./|\.?[A-Za-z0-9_\-]+/)+[A-Za-z0-9_.\-]+\.[A-Za-z0-9]{1,8}\b", + &FILE_PATH, |m| { // Reject version-dir lookalikes: a numeric-only "extension" // (foo/1.0.0, …/superpowers/5.1.0) is a version number, not a @@ -170,13 +173,12 @@ pub fn extract(text: &str) -> Artifacts { // branch. The bare-`branch <name>` form is intentionally dropped — // it caused too many false positives in journal events that // mention the word "branch" without naming one. - if let Ok(re) = - Regex::new(r"\bgit\s+(?:checkout\s+-b|switch\s+-c|branch)\s+([A-Za-z0-9._/\-]+)") - { - for cap in re.captures_iter(text) { - if let Some(m) = cap.get(1) { - a.branch_names.push(m.as_str().to_string()); - } + static BRANCH: LazyLock<Regex> = LazyLock::new(|| { + Regex::new(r"\bgit\s+(?:checkout\s+-b|switch\s+-c|branch)\s+([A-Za-z0-9._/\-]+)").unwrap() + }); + for cap in BRANCH.captures_iter(text) { + if let Some(m) = cap.get(1) { + a.branch_names.push(m.as_str().to_string()); } } @@ -194,11 +196,9 @@ fn dedup(v: &mut Vec<String>) { v.retain(|x| seen.insert(x.clone())); } -fn static_re(pat: &str, mut f: impl FnMut(&str), text: &str) { - if let Ok(re) = Regex::new(pat) { - for m in re.find_iter(text) { - f(m.as_str()); - } +fn static_re(re: &Regex, mut f: impl FnMut(&str), text: &str) { + for m in re.find_iter(text) { + f(m.as_str()); } } diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 2e898b6..25a0100 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -424,55 +424,44 @@ pub fn list_all_projects(state_dir: impl AsRef<Path>) -> anyhow::Result<Vec<Stri Ok(out) } +/// Replay the whole log from scratch: `search_fts` is cleared once, then every +/// event is indexed again. Commits in chunks like [`ingest_new_events`]. pub fn rebuild_state( conn: &Connection, jsonl_path: impl AsRef<Path>, project_hash: &str, ) -> anyhow::Result<usize> { - let f = match std::fs::File::open(&jsonl_path) { - Ok(f) => f, - // A fresh project has no events log on disk yet — there is simply nothing - // to read, which is not an error (this is what crashed task_create the - // first time the journal was touched in a new worktree). - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(e) => { - return Err(anyhow::Error::new(e).context(format!("open {:?}", jsonl_path.as_ref()))) - } - }; - let reader = std::io::BufReader::new(f); + replay(conn, jsonl_path.as_ref(), project_hash, true) +} - let tx = conn.unchecked_transaction()?; - let mut count = 0; - let mut last_event_id: Option<String> = None; - for (i, line) in reader.lines().enumerate() { - let line = line.with_context(|| format!("read line {i}"))?; - if line.trim().is_empty() { - continue; - } - // Malformed JSONL lines are skipped with a warning so that one bad - // event cannot abort an otherwise-recoverable rebuild. SQL errors - // still propagate — those indicate schema/integrity problems. - let event: Event = match serde_json::from_str(&line) { - Ok(e) => e, - Err(err) => { - tracing::warn!( - line_number = i + 1, - error = %err, - "skipping malformed JSONL line in rebuild_state" - ); - continue; +/// The events of a JSONL log in order. Malformed lines are skipped with a +/// warning so that one bad event cannot abort an otherwise-recoverable replay; +/// read errors still propagate. +fn log_events(f: std::fs::File) -> impl Iterator<Item = anyhow::Result<Event>> { + std::io::BufReader::new(f) + .lines() + .enumerate() + .filter_map(|(i, line)| { + let line = match line.with_context(|| format!("read line {i}")) { + Ok(line) => line, + Err(e) => return Some(Err(e)), + }; + if line.trim().is_empty() { + return None; } - }; - upsert_task_from_event(&tx, &event, project_hash)?; - index_event(&tx, &event)?; - last_event_id = Some(event.event_id.clone()); - count += 1; - } - if let Some(eid) = last_event_id.as_deref() { - record_last_indexed(&tx, project_hash, eid)?; - } - tx.commit()?; - Ok(count) + + match serde_json::from_str(&line) { + Ok(event) => Some(Ok(event)), + Err(err) => { + tracing::warn!( + line_number = i + 1, + error = %err, + "skipping malformed JSONL line" + ); + None + } + } + }) } /// Returns whether a task with this id has been recorded in the derived @@ -880,47 +869,54 @@ pub fn task_artifacts( Ok(acc) } -/// Look up the most recent `event_id` we've ingested for this project. -/// Returns `None` when the project has never been indexed (first call, -/// or migration v002 just landed on an existing 0.1.x DB). -fn last_indexed_event_id(conn: &Connection, project_hash: &str) -> anyhow::Result<Option<String>> { - let mut stmt = - conn.prepare("SELECT last_indexed_event_id FROM index_state WHERE project_hash = ?1")?; - let mut rows = stmt.query(rusqlite::params![project_hash])?; - if let Some(row) = rows.next()? { - Ok(Some(row.get::<_, String>(0)?)) - } else { - Ok(None) - } +/// How far the log is indexed for a project: the `index_state` row as +/// `(last_indexed_event_id, updated_at)`. `None` when the project has never +/// been indexed (first call, or a migration cleared the marker). +type IndexMark = (String, String); + +fn index_mark(conn: &Connection, project_hash: &str) -> anyhow::Result<Option<IndexMark>> { + Ok(conn + .query_row( + "SELECT last_indexed_event_id, updated_at FROM index_state WHERE project_hash = ?1", + rusqlite::params![project_hash], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?) } fn record_last_indexed( conn: &Connection, project_hash: &str, event_id: &str, -) -> anyhow::Result<()> { +) -> anyhow::Result<IndexMark> { + let mark = ( + event_id.to_string(), + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + ); conn.execute( "INSERT INTO index_state(project_hash, last_indexed_event_id, updated_at) VALUES (?1, ?2, ?3) ON CONFLICT(project_hash) DO UPDATE SET last_indexed_event_id = excluded.last_indexed_event_id, updated_at = excluded.updated_at", - rusqlite::params![ - project_hash, - event_id, - chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) - ], + rusqlite::params![project_hash, mark.0, mark.1], )?; - Ok(()) + + Ok(mark) } +/// Events a replay indexes per transaction. Each commit moves the marker, so +/// a replay killed partway (a hook timeout) resumes after its last chunk +/// instead of starting over, and other writers wait one chunk at most. +const REPLAY_CHUNK: usize = 500; + /// Read only the tail of the JSONL log since the last call. The cheap path /// for hot loops (every MCP tool invocation): scan to the marker, ingest /// the rest, update the marker. /// -/// Falls back to a full [`rebuild_state`] in two cases: -/// - No marker yet for this project (first call after migration v002 or -/// on a brand-new install). +/// Falls back to a full replay (see [`rebuild_state`]) in two cases: +/// - No marker yet for this project (first call after a migration cleared +/// it, or a brand-new install). /// - The stored marker is not present in the JSONL (corrupted / truncated /// file). A `tracing::warn!` is emitted so the operator notices. pub fn ingest_new_events( @@ -928,74 +924,117 @@ pub fn ingest_new_events( jsonl_path: impl AsRef<Path>, project_hash: &str, ) -> anyhow::Result<usize> { - let marker = match last_indexed_event_id(conn, project_hash)? { - Some(id) => id, - None => return rebuild_state(conn, jsonl_path, project_hash), - }; - - let f = match std::fs::File::open(&jsonl_path) { - Ok(f) => f, - // A fresh project has no events log on disk yet — there is simply nothing - // to read, which is not an error (this is what crashed task_create the - // first time the journal was touched in a new worktree). - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(e) => { - return Err(anyhow::Error::new(e).context(format!("open {:?}", jsonl_path.as_ref()))) - } - }; - let reader = std::io::BufReader::new(f); + replay(conn, jsonl_path.as_ref(), project_hash, false) +} - // First pass: confirm the marker still exists in the file. If it does - // not, the JSONL has been rewritten under us — we can't trust the - // marker, so we fall back to a full rebuild. - let tx = conn.unchecked_transaction()?; - let mut found_marker = false; +/// Index the log after the marker — or all of it when `from_scratch` or there +/// is no usable marker, clearing `search_fts` first — in chunks of +/// [`REPLAY_CHUNK`] events. Returns how many events it indexed. +fn replay( + conn: &Connection, + jsonl_path: &Path, + project_hash: &str, + mut from_scratch: bool, +) -> anyhow::Result<usize> { let mut count = 0; - let mut last_event_id: Option<String> = None; - for (i, line) in reader.lines().enumerate() { - let line = line.with_context(|| format!("read line {i}"))?; - if line.trim().is_empty() { - continue; - } - let event: Event = match serde_json::from_str(&line) { - Ok(e) => e, - Err(err) => { - tracing::warn!( - line_number = i + 1, - error = %err, - "skipping malformed JSONL line in ingest_new_events" - ); - continue; - } + + 'pass: loop { + let f = match std::fs::File::open(jsonl_path) { + Ok(f) => f, + // A fresh project has no events log on disk yet — there is simply nothing + // to read, which is not an error (this is what crashed task_create the + // first time the journal was touched in a new worktree). + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(count), + Err(e) => return Err(anyhow::Error::new(e).context(format!("open {jsonl_path:?}"))), }; - if !found_marker { - if event.event_id == marker { - found_marker = true; + let mut events = log_events(f); + let mut committed: Option<IndexMark> = None; + + loop { + // IMMEDIATE: the marker is read under the write lock, so two + // processes never index the same chunk. + let tx = rusqlite::Transaction::new_unchecked( + conn, + rusqlite::TransactionBehavior::Immediate, + )?; + let mark = index_mark(&tx, project_hash)?; + + if committed.is_none() { + match mark.filter(|_| !from_scratch) { + None => clear_search_fts(&tx, project_hash)?, + Some((marker, _)) => { + let mut found = false; + for event in events.by_ref() { + if event?.event_id == marker { + found = true; + break; + } + } + if !found { + tracing::warn!( + project_hash = project_hash, + marker = marker.as_str(), + "last_indexed_event_id not found in JSONL — falling back to full rebuild" + ); + from_scratch = true; + continue 'pass; + } + } + } + } else if mark != committed { + // Another process indexed past our last chunk: go on from its marker. + from_scratch = false; + continue 'pass; + } + + let mut in_chunk = 0; + let mut last_event_id = None; + for event in events.by_ref().take(REPLAY_CHUNK) { + let event = event?; + upsert_task_from_event(&tx, &event, project_hash)?; + index_event(&tx, &event)?; + last_event_id = Some(event.event_id); + in_chunk += 1; + } + + if let Some(eid) = last_event_id { + committed = Some(record_last_indexed(&tx, project_hash, &eid)?); + } + tx.commit()?; + count += in_chunk; + + if in_chunk < REPLAY_CHUNK { + return Ok(count); } - continue; } - upsert_task_from_event(&tx, &event, project_hash)?; - index_event(&tx, &event)?; - last_event_id = Some(event.event_id.clone()); - count += 1; - } - - if !found_marker { - // Discard the (empty) tx and rebuild from scratch. - drop(tx); - tracing::warn!( - project_hash = project_hash, - marker = marker.as_str(), - "last_indexed_event_id not found in JSONL — falling back to full rebuild" - ); - return rebuild_state(conn, jsonl_path, project_hash); } +} + +/// Drop this project's rows from `search_fts` before a replay re-inserts +/// them: one scan, where deleting per event scanned the table every time. +fn clear_search_fts(conn: &Connection, project_hash: &str) -> anyhow::Result<()> { + conn.execute( + "DELETE FROM search_fts + WHERE task_id NOT IN (SELECT task_id FROM tasks WHERE project_hash <> ?1)", + rusqlite::params![project_hash], + )?; + + Ok(()) +} - if let Some(eid) = last_event_id.as_deref() { - record_last_indexed(&tx, project_hash, eid)?; +/// The `search_fts` rowid of an event: a stable FNV-1a hash of its id. +/// `event_id` is UNINDEXED, so finding an event's row by it scans the whole +/// table; by rowid, re-indexing an event replaces its row in O(log n). +// ponytail: two ids hashing alike (odds ~n²/2^64) would share one search row; +// a mapping table removes that if it ever matters. +fn fts_rowid(event_id: &str) -> i64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in event_id.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); } - tx.commit()?; - Ok(count) + + (hash >> 1) as i64 } /// Machine-written bookkeeping, not reasoning: the PreCompact boundary marker @@ -1062,14 +1101,18 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { is_bookkeeping(event) ], )?; - // search_fts has no PK; clear then insert to keep idempotent across rebuild_state replays. + // search_fts has no PK; replacing by the event's own rowid keeps it + // idempotent across replays. conn.execute( - "DELETE FROM search_fts WHERE event_id=?1", - rusqlite::params![event.event_id], - )?; - conn.execute( - "INSERT INTO search_fts(task_id, event_id, text, type) VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![event.task_id, event.event_id, event.text, type_str], + "INSERT OR REPLACE INTO search_fts(rowid, task_id, event_id, text, type) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + fts_rowid(&event.event_id), + event.task_id, + event.event_id, + event.text, + type_str + ], )?; if event.event_type == EventType::Decision { @@ -2248,6 +2291,132 @@ mod tests { } } + /// A log of `n` events: an open every 30 events, findings in between. + fn write_synthetic_log(path: &Path, n: usize) -> Vec<crate::event::Event> { + let mut events = Vec::with_capacity(n); + let mut task = String::new(); + for i in 0..n { + if i % 30 == 0 { + task = format!("tj-syn{i}"); + events.push(make_open_event(&task, &task)); + } else { + let mut e = make_text_event(&format!( + "step {i}: replay chunk marker in crates/tj-core/src/db.rs, commit {i:07x}ab" + )); + e.task_id = task.clone(); + events.push(e); + } + } + + let mut f = std::fs::File::create(path).unwrap(); + for e in &events { + write_event_line(&mut f, e); + } + + events + } + + fn count_rows(conn: &Connection, table: &str) -> i64 { + conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + /// Seconds a replay of an already indexed `n`-event log takes once + /// `index_state` is cleared, as the v009–v011 upgrade migrations do. + fn upgrade_replay_seconds(n: usize) -> f64 { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + write_synthetic_log(&jsonl, n); + let conn = open(d.path().join("s.sqlite")).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch("DELETE FROM index_state").unwrap(); + + let started = std::time::Instant::now(); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), n); + let secs = started.elapsed().as_secs_f64(); + + assert_eq!(count_rows(&conn, "search_fts"), n as i64); + secs + } + + #[test] + fn a_full_replay_grows_linearly_with_the_log() { + // Best of two runs each, to keep scheduler noise out of the ratio. + let small = upgrade_replay_seconds(1500).min(upgrade_replay_seconds(1500)); + let large = upgrade_replay_seconds(3000).min(upgrade_replay_seconds(3000)); + + // Linear work doubles; the old per-event FTS scan quadrupled it. + assert!( + large < small * 3.0, + "replay of 2x events took {:.2}x as long ({small:.3}s -> {large:.3}s)", + large / small + ); + } + + #[test] + fn a_replay_killed_partway_resumes_after_its_last_committed_chunk() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + let events = write_synthetic_log(&jsonl, 1200); + let conn = open(d.path().join("s.sqlite")).unwrap(); + + // The process dies while indexing an event of the second chunk. + conn.execute_batch(&format!( + "CREATE TRIGGER kill BEFORE INSERT ON events_index WHEN NEW.event_id = '{}' + BEGIN SELECT RAISE(ABORT, 'killed'); END;", + events[700].event_id + )) + .unwrap(); + assert!(ingest_new_events(&conn, &jsonl, ph).is_err()); + conn.execute_batch("DROP TRIGGER kill").unwrap(); + + let committed = count_rows(&conn, "events_index"); + assert!( + committed > 0 && committed <= 700, + "the chunks before the kill stay committed, got {committed}" + ); + + let resumed = ingest_new_events(&conn, &jsonl, ph).unwrap(); + assert_eq!( + resumed as i64, + 1200 - committed, + "the resumed replay must not redo committed chunks" + ); + assert_eq!(count_rows(&conn, "events_index"), 1200); + assert_eq!(count_rows(&conn, "search_fts"), 1200); + } + + #[test] + fn concurrent_replays_share_the_work_without_locking_errors_or_duplicates() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + write_synthetic_log(&jsonl, 2000); + drop(open(&db).unwrap()); + + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let (jsonl, db, barrier) = (jsonl.clone(), db.clone(), barrier.clone()); + std::thread::spawn(move || { + let conn = open(&db).unwrap(); + barrier.wait(); + ingest_new_events(&conn, &jsonl, "feedfacefeedface").map(|_| ()) + }) + }) + .collect(); + for h in handles { + let res = h.join().unwrap(); + assert!(res.is_ok(), "{:#}", res.unwrap_err()); + } + + let conn = open(&db).unwrap(); + assert_eq!(count_rows(&conn, "events_index"), 2000); + assert_eq!(count_rows(&conn, "search_fts"), 2000); + } + #[test] fn rebuild_state_skips_malformed_jsonl_lines() { use std::io::Write; From e563e69c11909cc85c76b10c5fc59cc3bbe9e605 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:46:57 +0400 Subject: [PATCH 093/104] fix(db): replay the log when an older binary rewrote the index --- crates/tj-cli/src/main.rs | 8 ++- crates/tj-core/src/db.rs | 125 ++++++++++++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index d9b01d8..5ec2865 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -233,7 +233,13 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool let new_state_path = state_dir.join(format!("{to_hash}.sqlite")); if new_state_path.exists() { let conn = tj_core::db::open(&new_state_path)?; - for table in ["tasks", "index_state", "embeddings", "dream_state"] { + for table in [ + "tasks", + "index_state", + "projection_state", + "embeddings", + "dream_state", + ] { conn.execute( &format!("UPDATE {table} SET project_hash = ?1 WHERE project_hash = ?2"), rusqlite::params![to_hash, from_hash], diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 25a0100..028b715 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -184,6 +184,20 @@ const MIGRATION_012: &str = r#" DELETE FROM task_pack_cache; "#; +/// v0.30.0 projection marker — `projection_state` holds the `index_state` row +/// as this version last wrote it. An older binary still running (or a +/// downgrade) ingests or rebuilds without the 0.30 columns and moves +/// `index_state`; the mismatch makes the next ingest replay the log to +/// re-derive them. It starts empty, so the first ingest after the upgrade +/// replays the log once. +const MIGRATION_013: &str = r#" +CREATE TABLE IF NOT EXISTS projection_state ( + project_hash TEXT PRIMARY KEY, + last_indexed_event_id TEXT NOT NULL, + updated_at TEXT NOT NULL +); +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. const MIGRATIONS: &[Migration] = &[ @@ -235,6 +249,10 @@ const MIGRATIONS: &[Migration] = &[ version: 12, sql: MIGRATION_012, }, + Migration { + version: 13, + sql: MIGRATION_013, + }, ]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -874,10 +892,18 @@ pub fn task_artifacts( /// been indexed (first call, or a migration cleared the marker). type IndexMark = (String, String); -fn index_mark(conn: &Connection, project_hash: &str) -> anyhow::Result<Option<IndexMark>> { +/// The mark stored in `table`: `index_state`, which every version writes, or +/// `projection_state`, the copy only this version writes. +fn read_mark( + conn: &Connection, + table: &str, + project_hash: &str, +) -> anyhow::Result<Option<IndexMark>> { Ok(conn .query_row( - "SELECT last_indexed_event_id, updated_at FROM index_state WHERE project_hash = ?1", + &format!( + "SELECT last_indexed_event_id, updated_at FROM {table} WHERE project_hash = ?1" + ), rusqlite::params![project_hash], |r| Ok((r.get(0)?, r.get(1)?)), ) @@ -893,14 +919,18 @@ fn record_last_indexed( event_id.to_string(), chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), ); - conn.execute( - "INSERT INTO index_state(project_hash, last_indexed_event_id, updated_at) - VALUES (?1, ?2, ?3) - ON CONFLICT(project_hash) DO UPDATE SET - last_indexed_event_id = excluded.last_indexed_event_id, - updated_at = excluded.updated_at", - rusqlite::params![project_hash, mark.0, mark.1], - )?; + for table in ["index_state", "projection_state"] { + conn.execute( + &format!( + "INSERT INTO {table}(project_hash, last_indexed_event_id, updated_at) + VALUES (?1, ?2, ?3) + ON CONFLICT(project_hash) DO UPDATE SET + last_indexed_event_id = excluded.last_indexed_event_id, + updated_at = excluded.updated_at" + ), + rusqlite::params![project_hash, mark.0, mark.1], + )?; + } Ok(mark) } @@ -914,9 +944,11 @@ const REPLAY_CHUNK: usize = 500; /// for hot loops (every MCP tool invocation): scan to the marker, ingest /// the rest, update the marker. /// -/// Falls back to a full replay (see [`rebuild_state`]) in two cases: +/// Falls back to a full replay (see [`rebuild_state`]) in three cases: /// - No marker yet for this project (first call after a migration cleared /// it, or a brand-new install). +/// - The marker is not the one this version last wrote: an older binary +/// ingested or rebuilt without the columns it does not know. /// - The stored marker is not present in the JSONL (corrupted / truncated /// file). A `tracing::warn!` is emitted so the operator notices. pub fn ingest_new_events( @@ -957,10 +989,18 @@ fn replay( conn, rusqlite::TransactionBehavior::Immediate, )?; - let mark = index_mark(&tx, project_hash)?; + let mark = read_mark(&tx, "index_state", project_hash)?; if committed.is_none() { - match mark.filter(|_| !from_scratch) { + let ours = mark == read_mark(&tx, "projection_state", project_hash)?; + if mark.is_some() && !ours { + tracing::warn!( + project_hash = project_hash, + "index_state was written by another version — replaying the log" + ); + } + + match mark.filter(|_| !from_scratch && ours) { None => clear_search_fts(&tx, project_hash)?, Some((marker, _)) => { let mut found = false; @@ -2868,6 +2908,65 @@ mod tests { assert_eq!(flagged, expected); } + #[test] + fn a_rebuild_by_an_older_binary_is_repaired_on_the_next_ingest() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let ph = "feedfacefeedface"; + + let open_ev = make_open_event("tj-old", "Old binary"); + let mut wrong = session_event("tj-old", "sess-1", "2026-01-01T00:00:01.000Z"); + wrong.event_type = crate::event::EventType::Decision; + let mut corr = make_text_event("Not this"); + corr.task_id = "tj-old".into(); + corr.event_type = crate::event::EventType::Correction; + corr.corrects = Some(wrong.event_id.clone()); + let amend = amend_event("tj-old", serde_json::json!({"goal": "g"})); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for e in [&open_ev, &wrong, &corr, &amend] { + write_event_line(&mut f, e); + } + drop(f); + + let conn = open(d.path().join("s.sqlite")).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + // A 0.29 server cannot parse the trailing amend, so it misses the + // marker and rebuilds: its rows lack the 0.30 columns, and it records + // the last event it could read. + conn.execute_batch( + "UPDATE events_index SET session_id = NULL, corrected_by = NULL, bookkeeping = 0;", + ) + .unwrap(); + conn.execute( + "UPDATE index_state SET last_indexed_event_id = ?1, updated_at = '2026-01-02T00:00:00.000Z'", + rusqlite::params![corr.event_id], + ) + .unwrap(); + + ingest_new_events(&conn, &jsonl, ph).unwrap(); + + assert_eq!( + active_task_for_session(&conn, ph, "sess-1") + .unwrap() + .as_deref(), + Some("tj-old") + ); + let corrected_by: Option<String> = conn + .query_row( + "SELECT corrected_by FROM events_index WHERE event_id = ?1", + rusqlite::params![wrong.event_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(corrected_by.as_deref(), Some(corr.event_id.as_str())); + assert_eq!( + ingest_new_events(&conn, &jsonl, ph).unwrap(), + 0, + "once repaired, the projection is in sync again" + ); + } + #[test] fn open_meta_goal_and_external_are_restored_by_a_rebuild() { let d = TempDir::new().unwrap(); From 9b7601454dd188cec7964d36155d33199c2c6ab7 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:48:34 +0400 Subject: [PATCH 094/104] fix(db): bind a session only to tasks it wrote to deliberately --- crates/tj-core/src/db.rs | 87 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 028b715..14747eb 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -189,13 +189,15 @@ DELETE FROM task_pack_cache; /// downgrade) ingests or rebuilds without the 0.30 columns and moves /// `index_state`; the mismatch makes the next ingest replay the log to /// re-derive them. It starts empty, so the first ingest after the upgrade -/// replays the log once. +/// replays the log once — which also fills `author`, the event's writer, so +/// only deliberate writes bind a session to a task. const MIGRATION_013: &str = r#" CREATE TABLE IF NOT EXISTS projection_state ( project_hash TEXT PRIMARY KEY, last_indexed_event_id TEXT NOT NULL, updated_at TEXT NOT NULL ); +ALTER TABLE events_index ADD COLUMN author TEXT; "#; /// All schema migrations in version order. Append new entries here; never @@ -602,6 +604,10 @@ pub fn task_id_by_external(conn: &Connection, reference: &str) -> anyhow::Result /// The open task of `project_hash` whose most recent event carries /// `meta.session_id == session_id` — the task a live agent session is /// working on. `None` when the session has no event on an open task. +/// +/// Only deliberate writes count: hooks and the classifier fall back to the +/// newest open task and stamp the current session, so their events (and +/// bookkeeping) would bind a fresh session to another session's task. pub fn active_task_for_session( conn: &Connection, project_hash: &str, @@ -612,6 +618,7 @@ pub fn active_task_for_session( "SELECT ei.task_id FROM events_index ei JOIN tasks t ON t.task_id = ei.task_id WHERE ei.session_id = ?2 AND t.project_hash = ?1 AND t.status = 'open' + AND ei.author IN ('user', 'agent') AND ei.bookkeeping = 0 ORDER BY ei.timestamp DESC LIMIT 1", rusqlite::params![project_hash, session_id], |r| r.get::<_, String>(0), @@ -1132,13 +1139,17 @@ pub fn index_event(conn: &Connection, event: &Event) -> anyhow::Result<()> { Some(serde_json::to_string(&artifacts)?) }; let session_id = event.meta.get("session_id").and_then(|v| v.as_str()); + let author_str = serde_json::to_value(event.author)? + .as_str() + .unwrap() + .to_string(); conn.execute( - "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id, bookkeeping) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + "INSERT OR REPLACE INTO events_index(event_id, task_id, type, timestamp, confidence, status, artifacts, session_id, bookkeeping, author) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", rusqlite::params![ event.event_id, event.task_id, type_str, event.timestamp, event.confidence, status_str, artifacts_json, session_id, - is_bookkeeping(event) + is_bookkeeping(event), author_str ], )?; // search_fts has no PK; replacing by the event's own rowid keeps it @@ -2757,6 +2768,37 @@ mod tests { assert_eq!(active_task_for_session(&conn, ph, "s3").unwrap(), None); } + #[test] + fn hook_and_classifier_writes_never_bind_a_session_to_a_task() { + use crate::event::{Author, EventType}; + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + let ph = "feedfacefeedface"; + upsert_task_from_event(&conn, &make_open_event("tj-t1", "T1"), ph).unwrap(); + + // Session S1 works on T1 through the agent. + let mut work = session_event("tj-t1", "s1", "2026-01-01T00:00:01.000Z"); + work.author = Author::Agent; + // A fresh session S2: its PostModelSwitch note and a Stop catch-up + // classifier event fall back to the newest open task, T1. + let mut switch = session_event("tj-t1", "s2", "2026-01-01T00:00:02.000Z"); + switch.author = Author::Classifier; + switch.event_type = EventType::Constraint; + switch.meta = serde_json::json!({"session_id": "s2", "kind": "model_switch"}); + let mut caught_up = session_event("tj-t1", "s2", "2026-01-01T00:00:03.000Z"); + caught_up.author = Author::Classifier; + for e in [&work, &switch, &caught_up] { + upsert_task_from_event(&conn, e, ph).unwrap(); + index_event(&conn, e).unwrap(); + } + + assert_eq!(active_task_for_session(&conn, ph, "s2").unwrap(), None); + assert_eq!( + active_task_for_session(&conn, ph, "s1").unwrap().as_deref(), + Some("tj-t1") + ); + } + #[test] fn upgrading_an_existing_db_backfills_session_ids_on_the_next_ingest() { let d = TempDir::new().unwrap(); @@ -2967,6 +3009,43 @@ mod tests { ); } + #[test] + fn upgrading_from_0_29_replays_the_log_exactly_once() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let ph = "feedfacefeedface"; + let events = write_synthetic_log(&jsonl, 40); + + // A 0.29 database (schema v008) that has indexed the whole log. + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, ph).unwrap(); + conn.execute_batch( + "DROP INDEX idx_events_session_time; + ALTER TABLE events_index DROP COLUMN session_id; + ALTER TABLE events_index DROP COLUMN corrected_by; + ALTER TABLE events_index DROP COLUMN bookkeeping; + ALTER TABLE events_index DROP COLUMN author; + DROP TABLE projection_state; + DELETE FROM schema_migrations WHERE version >= 9;", + ) + .unwrap(); + drop(conn); + + let conn = open(&db).unwrap(); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), 40); + assert_eq!(ingest_new_events(&conn, &jsonl, ph).unwrap(), 0); + + let authored: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events_index WHERE author = 'user'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(authored, events.len() as i64); + } + #[test] fn open_meta_goal_and_external_are_restored_by_a_rebuild() { let d = TempDir::new().unwrap(); From 6a86aa834e4bc48178c83f544d241b7c14f6f967 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:54:26 +0400 Subject: [PATCH 095/104] fix(core): state leaves corrected and bookkeeping entries out of counts and recent --- crates/tj-core/src/session_state.rs | 48 +++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/tj-core/src/session_state.rs b/crates/tj-core/src/session_state.rs index a59f803..d3c4b6c 100644 --- a/crates/tj-core/src/session_state.rs +++ b/crates/tj-core/src/session_state.rs @@ -94,8 +94,13 @@ fn active_task(conn: &Connection, task_id: &str) -> anyhow::Result<ActiveTask> { )?; let mut counts = BTreeMap::new(); - let mut stmt = - conn.prepare("SELECT type, COUNT(*) FROM events_index WHERE task_id = ?1 GROUP BY type")?; + // Corrected entries and bookkeeping (compaction markers, model switches) + // are not part of what the task holds. + let mut stmt = conn.prepare( + "SELECT type, COUNT(*) FROM events_index + WHERE task_id = ?1 AND corrected_by IS NULL AND bookkeeping = 0 + GROUP BY type", + )?; for row in stmt.query_map([task_id], |r| { Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)) })? { @@ -107,6 +112,7 @@ fn active_task(conn: &Connection, task_id: &str) -> anyhow::Result<ActiveTask> { "SELECT e.type, f.text FROM events_index e JOIN search_fts f ON f.event_id = e.event_id WHERE e.task_id = ?1 AND e.type NOT IN ('open', 'amend') + AND e.corrected_by IS NULL AND e.bookkeeping = 0 ORDER BY e.timestamp DESC, e.event_id DESC LIMIT ?2", )?; let mut recent = stmt @@ -194,6 +200,44 @@ mod tests { assert_eq!(unknown.active.unwrap().task_id, "tj-sub"); } + #[test] + fn corrected_entries_are_left_out_of_counts_and_recent() { + let (_dir, conn) = conn(); + write(&conn, "tj-a", EventType::Open, "a", Some("s1")); + let mut wrong = Event::new( + "tj-a", + EventType::Decision, + Author::Agent, + Source::Chat, + "Use X".into(), + ); + crate::session_id::stamp_session_id(&mut wrong.meta, Some("s1")); + crate::db::upsert_task_from_event(&conn, &wrong, "p").unwrap(); + crate::db::index_event(&conn, &wrong).unwrap(); + let mut fix = Event::new( + "tj-a", + EventType::Correction, + Author::Agent, + Source::Chat, + "Not X, Y".into(), + ); + fix.corrects = Some(wrong.event_id.clone()); + crate::db::upsert_task_from_event(&conn, &fix, "p").unwrap(); + crate::db::index_event(&conn, &fix).unwrap(); + + let active = session_state(&conn, "p", Some("s1"), None) + .unwrap() + .active + .unwrap(); + assert_eq!( + active.counts.get("decision"), + None, + "the corrected decision is not counted" + ); + assert!(active.recent.iter().all(|r| r.text != "Use X")); + assert!(active.recent.iter().any(|r| r.text == "Not X, Y")); + } + #[test] fn a_closed_task_is_no_longer_active() { let (_dir, conn) = conn(); From b5ab64e37de8d36bba6e3eda7e933df13277d955 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:59:16 +0400 Subject: [PATCH 096/104] fix(core): do not treat an empty .git directory as a project root --- crates/tj-core/src/project_hash.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/tj-core/src/project_hash.rs b/crates/tj-core/src/project_hash.rs index bd687b7..80aae36 100644 --- a/crates/tj-core/src/project_hash.rs +++ b/crates/tj-core/src/project_hash.rs @@ -14,14 +14,17 @@ use std::path::{Path, PathBuf}; /// 2. `.git` (file or directory) — covers normal checkouts and /// worktrees alike (a worktree's root holds a `.git` *file* /// pointing at the real gitdir, but its presence still marks the -/// boundary). +/// boundary). Like git itself, a `.git` directory counts only when it +/// holds `HEAD`: Codex's sandbox (bubblewrap) briefly mounts an empty +/// `.git` into a writable root such as `/tmp` while a command runs. /// /// Falls back to `start` if no marker is found, preserving prior /// behaviour for non-git scratch directories. pub fn project_root(start: &Path) -> PathBuf { let mut cur = start; loop { - if cur.join(".task-journal").is_dir() || cur.join(".git").exists() { + let git = cur.join(".git"); + if cur.join(".task-journal").is_dir() || git.is_file() || git.join("HEAD").exists() { return cur.to_path_buf(); } match cur.parent() { @@ -103,6 +106,7 @@ mod tests { // repo/ with .git inside; repo/src/foo/ should normalise to repo/. let repo = TempDir::new().unwrap(); std::fs::create_dir(repo.path().join(".git")).unwrap(); + std::fs::write(repo.path().join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); let sub = repo.path().join("src").join("foo"); std::fs::create_dir_all(&sub).unwrap(); @@ -120,6 +124,7 @@ mod tests { // (explicit opt-out of the parent's journal). let repo = TempDir::new().unwrap(); std::fs::create_dir(repo.path().join(".git")).unwrap(); + std::fs::write(repo.path().join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); let sub = repo.path().join("sub"); std::fs::create_dir(&sub).unwrap(); std::fs::create_dir(sub.join(".task-journal")).unwrap(); @@ -148,4 +153,18 @@ mod tests { "worktree subdir must normalise to worktree root via .git file" ); } + + #[test] + fn empty_dot_git_dir_is_not_a_boundary() { + // Codex's bubblewrap sandbox briefly mounts an empty `.git` into a + // writable root such as `/tmp` while a command runs. Git does not take + // it for a repository, and the project hash must not flip meanwhile. + let base = TempDir::new().unwrap(); + let proj = base.path().join("proj"); + std::fs::create_dir(&proj).unwrap(); + let before = from_path(&proj).unwrap(); + + std::fs::create_dir(base.path().join(".git")).unwrap(); + assert_eq!(from_path(&proj).unwrap(), before); + } } From 464ad70d83e74883be1e2362e2d24d95ec0686e2 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 14:59:16 +0400 Subject: [PATCH 097/104] test(session): set mtimes explicitly in the newest-first discovery test --- crates/tj-core/src/session/discovery.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/crates/tj-core/src/session/discovery.rs b/crates/tj-core/src/session/discovery.rs index 89bd975..ea3140c 100644 --- a/crates/tj-core/src/session/discovery.rs +++ b/crates/tj-core/src/session/discovery.rs @@ -157,15 +157,19 @@ mod tests { fn list_sessions_sorted_by_mtime_newest_first() { let dir = tempfile::tempdir().unwrap(); - // Create files with different modification times. - let older = dir.path().join("older.jsonl"); - std::fs::write(&older, "{}").unwrap(); - - // Sleep briefly to ensure different mtime. - std::thread::sleep(std::time::Duration::from_millis(50)); - - let newer = dir.path().join("newer.jsonl"); - std::fs::write(&newer, "{}").unwrap(); + // Set the mtimes explicitly: a sleep between two writes is not enough + // when the wall clock steps back (seen on WSL), and it slows the test. + let base = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000); + for (name, secs) in [("older.jsonl", 0), ("newer.jsonl", 60)] { + let path = dir.path().join(name); + std::fs::write(&path, "{}").unwrap(); + std::fs::File::options() + .write(true) + .open(&path) + .unwrap() + .set_modified(base + std::time::Duration::from_secs(secs)) + .unwrap(); + } let sessions = list_sessions(dir.path()).unwrap(); assert_eq!(sessions.len(), 2); From 542f13622d9a083ba56d8415a348d957120933f3 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:02:05 +0400 Subject: [PATCH 098/104] fix(search): break last_event_at ties by insertion order when listing tasks --- crates/tj-cli/src/main.rs | 2 +- crates/tj-mcp/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index ce8e50d..ec6b28e 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3868,7 +3868,7 @@ fn run_search( let mut stmt = conn.prepare( "SELECT task_id FROM tasks \ WHERE ?1 IS NULL OR task_id IN (SELECT task_id FROM events_index WHERE type = ?1) \ - ORDER BY last_event_at DESC LIMIT ?2", + ORDER BY last_event_at DESC, rowid DESC LIMIT ?2", )?; let ids = stmt .query_map(rusqlite::params![event_type, limit as i64], |r| { diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 9902995..f5bbb61 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -567,7 +567,7 @@ impl TaskJournalServer { WHERE (?1 IS NULL OR status = ?1) \ AND (?2 IS NULL OR task_id IN \ (SELECT task_id FROM events_index WHERE type = ?2)) \ - ORDER BY last_event_at DESC LIMIT 50", + ORDER BY last_event_at DESC, rowid DESC LIMIT 50", )?; let hits = stmt .query_map(rusqlite::params![status, event_type], task_search_hit)? From 195ddee61170d8d3b0cec74d751297c768574e36 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:02:05 +0400 Subject: [PATCH 099/104] fix(memory): keep suggested events out of the global memory index --- crates/tj-core/src/memory.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/tj-core/src/memory.rs b/crates/tj-core/src/memory.rs index 340c898..e4caa10 100644 --- a/crates/tj-core/src/memory.rs +++ b/crates/tj-core/src/memory.rs @@ -86,7 +86,7 @@ pub fn sync_from_project( let sql = format!( "SELECT e.event_id, e.task_id, f.type, e.tier, f.text, e.model, e.dim, e.vec, e.created_at, CASE WHEN d.superseded_by IS NOT NULL THEN 1 ELSE 0 END, - COALESCE(ei.bookkeeping, 0) OR ei.corrected_by IS NOT NULL + COALESCE(ei.bookkeeping, 0) OR ei.corrected_by IS NOT NULL OR ei.status = 'suggested' FROM embeddings e JOIN search_fts f ON f.event_id = e.event_id LEFT JOIN decisions d ON d.decision_id = e.event_id @@ -435,6 +435,22 @@ mod tests { .is_empty()); } + #[test] + fn sync_leaves_out_suggested_events() { + let d = tempfile::TempDir::new().unwrap(); + let proj = crate::db::open(d.path().join("p.sqlite")).unwrap(); + let global = open(d.path().join("memory.sqlite")).unwrap(); + let emb = crate::embed::HashEmbedder::new(64); + + let mut guess = finding("the classifier guessed a redis cache"); + guess.status = crate::event::EventStatus::Suggested; + crate::db::index_event(&proj, &guess).unwrap(); + crate::db::embed_pending(&proj, "ph", &emb, "t", 100).unwrap(); + + assert_eq!(sync_from_project(&global, &proj, "ph").unwrap(), 0); + assert_eq!(count(&global).unwrap(), 0); + } + #[test] fn preferences_store_dedup_and_list_in_order() { let d = tempfile::TempDir::new().unwrap(); From d286e44a59899ae058fb4f21dd4428e57dcd0161 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:02:05 +0400 Subject: [PATCH 100/104] test(cli): shim gh so close's harvest never reaches GitHub in tests --- crates/tj-cli/tests/cli.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 9636b35..a6204a9 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -29,19 +29,32 @@ impl Command { } } -/// The test process's PATH minus any directory that holds `claude` or `codex`. -/// Computed once: on WSL the PATH carries dozens of slow `/mnt/c` directories. +/// The test process's PATH minus any directory that holds `claude` or `codex`, +/// led by a directory whose `gh` always fails, so `close`'s artifact harvest +/// never asks GitHub about this repository. Computed once: on WSL the PATH +/// carries dozens of slow `/mnt/c` directories. fn path_without_llm_clis() -> &'static std::ffi::OsStr { static PATH: std::sync::OnceLock<std::ffi::OsString> = std::sync::OnceLock::new(); PATH.get_or_init(|| { + let shims = std::env::temp_dir().join(format!("tj-cli-test-shims-{}", std::process::id())); + std::fs::create_dir_all(&shims).unwrap(); + std::fs::write(shims.join("gh.cmd"), "@exit /b 1\r\n").unwrap(); + std::fs::write(shims.join("gh"), "#!/bin/sh\nexit 1\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(shims.join("gh"), std::fs::Permissions::from_mode(0o755)) + .unwrap(); + } + let path = std::env::var_os("PATH").unwrap_or_default(); - let keep = std::env::split_paths(&path).filter(|dir| { + let keep = std::iter::once(shims).chain(std::env::split_paths(&path).filter(|dir| { ["claude", "codex"].iter().all(|bin| { ["", ".exe", ".cmd"] .iter() .all(|ext| !dir.join(format!("{bin}{ext}")).exists()) }) - }); + })); std::env::join_paths(keep).unwrap() }) } From e888308e25e2b1f4bfe380e1b563e6b25439efae Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:02:22 +0400 Subject: [PATCH 101/104] docs: changelog for 0.30.0 --- CHANGELOG.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff6b3d..735b8f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,92 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.30.0] - 2026-10-04 + +Claude Code 2.1.287 added mods: plugins that run inside Claude Code and can +shape the system prompt, the compaction and the status line. This release ships +a Task Journal mod, makes every session (Claude Code and Codex alike) keep its +own active task, and fixes what a full audit of the journal turned up. + +### Added +- **Task Journal mod for Claude Code 2.1.287+**, bundled with the plugin + (`plugin/hooks/`). It keeps the session's active task in the system prompt, so + a compaction can no longer lose it, and asks the compaction summary to keep + the task id. Right before a compaction it asks the model what was decided but + never logged and records those as `suggested` events. It shows the task in + the status line, toasts each entry, and reminds the agent to log only after + several prompts without an entry. Options: `nudge_after_turns`, + `distill_on_compact`. While it runs, the classic hooks skip what it replaces + (the reminder, per-message classification, the transcript catch-ups) and keep + the rest (resume packs, push-recall, `/rewind`). On Claude Code older than + 2.1.287 the mod doesn't load and everything else works as before. +- **Per-session active task.** Events carry their session id and the index + keeps it, so two sessions in one project each resume their own task instead + of whichever task was touched last. Codex sessions bind the same way: the MCP + server reads the session id Codex sends with each tool call. Only deliberate + writes bind a session — a hook's model-switch note or a classifier's guess + landing on another task no longer pulls a fresh session onto it. +- `task-journal state [--session] [--prefer]`: the session's active task, + counts and latest entries as JSON (schema `tj-state/1`). +- `task-journal event --suggested --session --origin` for integrations, and an + optional `session_id` on the MCP write tools. +- `task_search` honours `status` and `project`; with an empty query it lists the + project's tasks, and its result gains a `tasks` field (title, status, goal). +- `doctor` reports whether `codex` is on PATH. + +### Fixed +- **Re-indexing is linear and resumable.** Rebuilding the index cost time + quadratic in the journal's size (a 7,000-event journal: ~30 s); it is now + linear (~1.5 s for the same journal), commits in chunks, and a run that is + cut off picks up where it stopped instead of starting over. +- **An older binary next to this one can't corrupt the index.** A 0.29 server + still running in an open session would rebuild the index without the new + columns; the next 0.30 call now notices and repairs it. +- **Goals and external links survive a rebuild.** They lived only in SQLite; a + rebuild from the log dropped them, Loom links included. They are now in the + journal (on the open event, or an `amend` event for later changes). +- **Corrections take effect.** A `correction` now hides the event it corrects + from every pack section and from `export-pr`. +- **Packs say what is unconfirmed.** Suggested decisions and rejections are + marked; compaction markers and model-switch records no longer appear as + decisions, in `export-pr`, in recall or in global memory; a compact pack keeps + its gap summary; a cached pack no longer shows stale linked-task status. +- **Auto-capture stays in its project.** The shared `pending/` queue was + processed by whichever project's worker ran first, so one project's chunks + could land in another's journal. Entries are now scoped to their project, + processed oldest first, and the classify worker holds an OS file lock, so two + workers can no longer pick up the same chunk. +- A crash mid-write no longer costs the next event too; schema migrations are + atomic and safe when two processes open a fresh database. +- `PostToolUse` chunks are capped and no longer turn tool output into bogus + `constraint` events; push-recall shows each hit once per session. +- `pending retry` really retries, and `ingest-hook` no longer deletes failed + chunks before they can be retried. +- The `codex` backend no longer triggers your own Codex hooks from inside a + `dream` or `complete` run. +- `dream` no longer skips sessions forever after a scoped or partly failed run, + drops events for task ids the model made up, and survives an unreadable + transcript. `backfill` detects imported sessions reliably and indexes them. +- FTS search no longer fails on punctuation or words like `AND`. +- An empty `.git` directory (as Codex's sandbox briefly mounts in `/tmp`) is no + longer taken for a repository root; like git, a `.git` directory needs `HEAD`. +- `task_close` harvests artifacts from the project dir; `event_add` and + `artifact_add` reject unknown task ids instead of writing orphan events. +- `migrate-project` carries the WAL, the pending queue and the push-recall log + along, and keeps a backup of a destination journal it replaces. +- `export`, `events list`, `rejected`, the TUI and the extractor got smaller + fixes (malformed lines, char-safe ids, key repeats, char counts). + +### Changed +- The removed `ClaudeBinaryRunner`, the unused dream backends and the + `FileChanged` / `watchPaths` / `TJ_ASYNC_REWAKE` paths are gone; they were never + reachable. +- `task-journal-core` is pinned once in the workspace; publishing waits for fmt, + clippy and tests to pass. +- An `amend` line is skipped (with a warning) by binaries older than 0.30.0. +- After upgrading, restart open Claude Code and Codex sessions; the first + command re-indexes the project once. + ## [0.29.0] - 2026-09-25 Three months of Claude Code releases (2.1.196 → 2.1.281) changed how hooks are From b93f48141a92516fceb27252eba50ece3d2e0277 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:06:27 +0400 Subject: [PATCH 102/104] chore: ignore the tsconfig Claude Code lays beside a mod it loads --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d10d988..5d60f9b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ # Types Claude Code lays beside a mod it loads from this folder plugin/.claude-plugin/types/ +plugin/tsconfig.json From 00bcece07581dfad64a80b412a5f7f00cb823055 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:20:33 +0400 Subject: [PATCH 103/104] build: update crossbeam-epoch, h2, rustls and anyhow past their advisories; document the rmcp stdio exemption --- .cargo/audit.toml | 9 +++++++++ Cargo.lock | 42 +++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 .cargo/audit.toml diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..2a62ec3 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,9 @@ +# cargo-audit configuration (read by `cargo audit` and the CI audit job). + +[advisories] +ignore = [ + # rmcp < 1.4: DNS rebinding in the Streamable HTTP *server* transport. + # task-journal-mcp only serves over stdio (transport-io), which the + # advisory states is not affected. Drop this once rmcp is upgraded to 1.x. + "RUSTSEC-2026-0189", +] diff --git a/Cargo.lock b/Cargo.lock index 50afee7..321513b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,7 +88,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -99,14 +99,14 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "assert-json-diff" @@ -360,7 +360,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -730,7 +730,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -781,7 +781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1026,9 +1026,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1402,7 +1402,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1699,7 +1699,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2165,14 +2165,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -2194,9 +2194,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -2420,7 +2420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2593,7 +2593,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3091,7 +3091,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From 94b6271dee0cd64a6360bbe5ca0adb852dcbc8be Mon Sep 17 00:00:00 2001 From: Mher Shahinyan <shahinyanm@gmail.com> Date: Sun, 4 Oct 2026 15:20:33 +0400 Subject: [PATCH 104/104] test(cli): count a claimed .processing entry as queued and keep the move-failure stand-in on unix --- crates/tj-cli/tests/cli.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a6204a9..139d00d 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -1105,6 +1105,10 @@ fn migrate_project_carries_uncheckpointed_wal_writes() { assert_eq!(rows, 1, "the WAL-only write was lost in migration"); } +// Unix only: the stand-in for a failing move (renaming a directory over a +// file) is refused on Unix but allowed on Windows, where the real failure is a +// sharing violation that a test can't stage portably. +#[cfg(unix)] #[test] fn migrate_project_force_keeps_destination_when_the_move_fails() { let xdg = assert_fs::TempDir::new().unwrap(); @@ -3938,7 +3942,14 @@ fn ingest_hook_returns_fast_in_async_mode() { let entries: Vec<_> = std::fs::read_dir(&pending) .unwrap() .filter_map(|e| e.ok()) - .filter(|e| e.path().extension().and_then(|s| s.to_str()) == Some("json")) + // A worker that already claimed the entry renamed it to `.processing`: + // still queued, just being classified. + .filter(|e| { + matches!( + e.path().extension().and_then(|s| s.to_str()), + Some("json" | "processing") + ) + }) .collect(); // Worker may have already drained; in that case at least the // worker should have left a v1 pending entry from /bin/false