diff --git a/bt-daemon/Cargo.lock b/bt-daemon/Cargo.lock index 18d1bf23..1488f310 100644 --- a/bt-daemon/Cargo.lock +++ b/bt-daemon/Cargo.lock @@ -239,7 +239,7 @@ dependencies = [ [[package]] name = "braintrust-sdk-rust" version = "0.1.0-alpha.2" -source = "git+https://github.com/braintrustdata/braintrust-sdk-rust?rev=019e428ab80d0da4c00dab383d4b0d1fdf4486fd#019e428ab80d0da4c00dab383d4b0d1fdf4486fd" +source = "git+https://github.com/braintrustdata/braintrust-sdk-rust?rev=6cb1f157d950862ced0fe0306bd8e2b0ecba8602#6cb1f157d950862ced0fe0306bd8e2b0ecba8602" dependencies = [ "anyhow", "arc-swap", @@ -282,6 +282,7 @@ dependencies = [ "reqwest", "rquickjs", "rquickjs-serde", + "rusqlite", "semver", "serde", "serde_json", @@ -594,6 +595,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -781,6 +794,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -792,6 +814,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1050,7 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1113,6 +1144,17 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1531,7 +1573,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e4f499ac5b943d97ee6dbc44f23c2c10426f420f7d2f1793d6318911b6608c" dependencies = [ - "hashbrown", + "hashbrown 0.17.1", "relative-path", "rquickjs-sys", ] @@ -1555,6 +1597,31 @@ dependencies = [ "cc", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.19", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1812,6 +1879,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2204,6 +2283,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/bt-daemon/Cargo.toml b/bt-daemon/Cargo.toml index b4ff59d1..8869b01c 100644 --- a/bt-daemon/Cargo.toml +++ b/bt-daemon/Cargo.toml @@ -18,7 +18,7 @@ required-features = ["cli"] [dependencies] # Keep this pinned exactly, matching the dependency policy used by `bt`. -braintrust-sdk-rust = { git = "https://github.com/braintrustdata/braintrust-sdk-rust", rev = "019e428ab80d0da4c00dab383d4b0d1fdf4486fd" } +braintrust-sdk-rust = { git = "https://github.com/braintrustdata/braintrust-sdk-rust", rev = "6cb1f157d950862ced0fe0306bd8e2b0ecba8602" } base64 = "0.22" anyhow = "1" async-trait = "0.1" @@ -28,7 +28,7 @@ fs2 = "0.4" regex = "1" rquickjs = "0.12.2" rquickjs-serde = "0.6.1" -serde = { version = "1", features = ["derive"] } +serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" semver = "1" sha2 = "0.10" @@ -39,6 +39,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "io-uti tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = { version = "1", features = ["v4", "v5"] } +rusqlite = { version = "0.40.2", features = ["bundled"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading"] } diff --git a/bt-daemon/README.md b/bt-daemon/README.md index 94447942..0a01e325 100644 --- a/bt-daemon/README.md +++ b/bt-daemon/README.md @@ -153,10 +153,11 @@ bt trace run --plugin ./local.mjs codex -- "summarize this change" bt trace import codex SESSION_ID --plugin ./sanitize-history.mjs ``` -The journal stores raw input events, not transformed spans. After daemon -recovery, replayed events therefore pass through the resumed session's current -route: ordinary sessions use the current globally configured plugins, while a -managed session continues using only that run's isolated plugins. +Capture writes complete native input to the NDJSON event WAL. Translation commits +route-neutral span operations and continuation state to a SQLite span ledger, +then collects the translated event prefix. Plugins and delivery consume only +the span ledger, using the resumed route's plugin configuration. Historical +import reads agent transcripts independently of these recovery caches. `context.operation` is `"insert"` or `"merge"`; `context.source` and `context.session_id` identify the translated event stream; and `context.env` @@ -169,14 +170,17 @@ span processor is constructed. Plugins execute in bounded, thread-local QuickJS runtimes with no filesystem or network host APIs. Modules must be self-contained and transforms must be stateless: module globals belong to a worker thread, not a session. Every configured plugin is mandatory. If one -fails, the affected source session and route pause delivery; other sessions -continue. The daemon sends a payload-free failure marker on the affected span -with `plugin failure: `, then keeps new events in that session's journal. +fails, delivery pauses only for that route and source session; other routes +and sessions continue. The daemon sends a failure marker on the affected span +with `plugin failure: `, then holds that route's undelivered operations in the span ledger. +Source translation continues while the route is paused. It checks the failed plugin's content digest once a second. When that file -changes, it reprocesses the paused session through the current plugin chain, -replaces the marker with the recovered span, and resumes delivery after the -replay reaches a durable checkpoint. A still failing edit leaves that session -paused. No daemon restart or new agent event is needed. +changes, it reprocesses the paused route from the failed operation through the +current plugin chain, replaces the marker with the ordinary translated span, +and resumes delivery after the replay reaches a durable checkpoint. The +successful span has no recovery label or marker-only error. A still failing +edit leaves that route paused. No daemon lifecycle command or new agent event +is needed. Plugin failures are deduplicated in the daemon's private local state, including the raw QuickJS exception and stack. Inspect them with: @@ -186,8 +190,9 @@ bt trace doctor codex ``` The doctor output reports the plugin path, full local exception, session and -span IDs, journal position, pending bytes, and recovery state. Raw exception -details remain local; the Braintrust marker only includes the plugin path. +span IDs, first unprocessed span ledger position, and active recovery state. Raw +exception details remain local; the Braintrust marker only includes the +plugin path. Managed-run diagnostics are copied out of their temporary daemon directory before it is removed. Plugins are trusted local code: although they have no host APIs, they can copy @@ -296,25 +301,24 @@ different profiles, organizations, projects, experiments, or parent spans. ## Recovery and storage -Capture requests return after the event is flushed to its journal. Daemon -workers handle authentication, translation, and delivery. Restart recovery -replays journaled events with deterministic span IDs, so repeated delivery -updates the same spans. - -Claude, Codex, and Cursor events reference daemon-owned transcript mirrors. Cursor -preserves separate immutable generations when its native transcript is rewritten. -Grok uses -separate updates and events mirrors. Replay does not depend on the original -transcript staying at its old path. - -The daemon streams journals and transcripts, applies queue backpressure, and -retires idle sessions. Retired sessions are rebuilt from their journals when -another event arrives. Journals and mirrors preserve conversation content; -credentials are excluded from journaled routing data. - -Recovery files are removed after seven days without modification. Cleanup runs -at startup and hourly while the daemon is running. This retention period is -currently fixed in `src/server.rs`. +Capture requests return after the native event is synced to its NDJSON WAL. +Translation atomically commits span operations, continuation state, and its +input watermark to a private SQLite database with WAL mode and full durability. +The translated input prefix can then be collected without affecting plugins, +authentication, delivery, or their recovery. + +Each delivery route has an independent span ledger cursor. After checked backend +acceptance, SQLite commits the cursor and collects operations through the slowest +registered consumer. Paused routes protect undelivered spans; translation failures +protect untranslated events. Open sessions and turns retain continuation state +and stable IDs for later merges, rather than retaining earlier operations. + +Restart and idle-session resumption restore translator continuation and replay +pending span operations. Historical import and attach work directly from agent +transcripts; previously collected hook details may be unavailable in those files. +Native payloads, span ledgers, continuation, and transcript mirrors can contain +conversation content and are private local state. Credentials are excluded. +Resolved incident history is bounded and does not prevent cache collection. ## Platform and integration coverage diff --git a/bt-daemon/docs/agent-event-wal-recovery-design.md b/bt-daemon/docs/agent-event-wal-recovery-design.md new file mode 100644 index 00000000..194a98a0 --- /dev/null +++ b/bt-daemon/docs/agent-event-wal-recovery-design.md @@ -0,0 +1,415 @@ +# Agent event WAL and failure recovery + +Status: implementation under review in [draft PR #132](https://github.com/braintrustdata/braintrust-coding-agent-plugins/pull/132), superseding the plugin-specific recovery model. There is no migration from unreleased PR #130 state. + +## Purpose and governing rules + +The daemon captures a complete, ordered record of every native event it +accepts from a coding-agent capture adapter until translation has durably transferred its required state and output to the next pipeline stage. The WAL is an immediate-term recovery cache, not a permanent archive. The translator reads that record +and produces Braintrust spans to the best of its ability. A translator's current +interests and schema knowledge must not decide which native fields or events +survive in the journal. + +The recovery system should stop only the work that cannot safely progress, +identify the reason, and resume from the exact failed position when that +reason may have changed. A plugin failure should stop its delivery route; a +translation failure should stop translation for its source session. Other +sessions, and routes that have no failure, should continue. + +These are the proposed invariants: + +1. A hook is acknowledged only after its native event is recorded in the + capture WAL. Auth, translation, plugin execution, and backend delivery are + downstream work. +2. The captured native event is retained without dropping fields, filtering + event names, or substituting a translator's reduced representation. + Compression and external payload storage are acceptable only when they + reconstruct the captured event losslessly. + `source` and a nonempty `session_id` are the required capture identity; + they select the session WAL partition. No other native payload shape is + required before append. +3. Unknown or intentionally unused events are journaled and consumed without + producing spans or pausing progress. A **recognized** event whose required + structure cannot be interpreted safely is a translation failure. +4. A failure never advances the affected progress cursor past the first + unprocessed event or undelivered span operation. Later native events still + enter the WAL. +5. Retry conditions are specific to the recorded failure. A check only makes + replay eligible; replay and acknowledged delivery determine whether the + failure has actually been resolved. +6. A recovered span has the same current fields it would have had on an + uninterrupted delivery. Failure indicators exist only while the failure is + active. The local diagnostic history can retain resolved incidents. +7. Credentials are resolved at delivery time and are never stored in the + native-event WAL or a persisted recovery condition. + +"Complete" here means complete for events delivered to the capture adapter. +The daemon cannot reconstruct events an agent never exposes. Captured events are collected after their output and continuation state have been committed. Historical import reads agent transcripts, with potentially reduced detail compared with live hooks. + +## Current seams that this design replaces + +The current daemon already appends an envelope before acknowledging `event.log`, +and it keeps a source-qualified session journal with route-specific delivery +checkpoints. It also records plugin failures and retries their route after the +failed plugin file changes. The proposed model preserves those useful +boundaries while replacing several constraints: + +| Current behavior | Required change | +| --- | --- | +| `capture_hook` extracts `session_id` before it can send an envelope; JS adapters also supply an envelope `session_id`. The daemon deserializes an `Envelope` before journaling it. | Keep `source` and nonempty `session_id` as required capture metadata so the event can enter its session partition. Do not require any further native event fields or translator-specific decoding before append. | +| The journal serializes `RedactedEnvelope.payload` as `serde_json::Value`. Pi journal compaction drops some native fields from provider, streaming, and terminal events. | Preserve the complete native payload. Replace lossy Pi compaction with lossless compression or storage of the original payload. | +| `configure_event` resolves auth before `session_for` creates an actor. `SessionConfig` combines routing and credentials. | Create source-session and route work from non-secret routing information. Give translators route-neutral context and resolve credentials only for delivery. | +| Some known-event typed decoders return `None` and their callers silently emit no operations. | Distinguish unknown or unused events from known events whose required fields fail decoding. | +| `PluginRecoveryState`, `PluginPause`, and plugin diagnostics each hold part of the recovery state. | Persist one mode-neutral work state and a separate typed failure cause. Derive doctor output from incidents; retain plugin diagnostics only as bounded history. | +| Journal events are synced before acknowledgement, but native records are collected by age. | Collect input only through durable translation progress. Collect span operations only through the slowest registered consumer cursor. Incident history does not protect already-consumed payloads. | + +Old journals remain readable during migration. Native fields that an old lossy +journal omitted cannot be reconstructed; the migration must not label those +records as complete historical captures. + +## Proposed data flow + +```mermaid +flowchart LR + Agent[Capture adapter: source and session ID] --> WAL[Per-session native event WAL] + WAL --> Translator[Source session translator] + Translator --> Spans[Derived logical span revisions] + Spans --> RouteA[Route A: overlay, plugins, auth, sink] + Spans --> RouteB[Route B: overlay, plugins, auth, sink] + RouteA --> Backend[Braintrust] + RouteB --> Backend + State[Durable cursors and failure state] -.-> WAL + State -.-> Translator + State -.-> RouteA + State -.-> RouteB +``` + +The pipeline has two recovery boundaries: + +1. **NDJSON event WAL:** untranslated native input, including malformed recognized events. Translation owns this stream and its input watermark. +2. **SQLite span ledger:** committed route-neutral create/merge operations and durable translator continuation. Plugins, credentials, backend delivery, and downstream recovery consume this ledger exclusively. + +A source session owns one translator. Each route owns its overlay, plugin chain, credentials, sink, and independent output cursor. A paused delivery route does not pause source translation. A source translation failure protects the unprocessed event suffix while previously produced output remains deliverable. + +Translator continuation is required for correctness after input collection. It retains open-span identities, message context, counters, native transcript offsets, and pending correlation state needed by later events. Session and turn spans can be created early and finalized later by merge; retaining every earlier create or merge is unnecessary. + +### SQLite storage and commit boundary + +Each `(source, session_id)` has a private `derived//spans.sqlite` database. SQLite uses WAL mode and `synchronous=FULL`. The storage schema contains: + +| Table | Purpose | +| --- | --- | +| `revisions` | Ordered output revisions with an independent monotonic sequence, identity, and input provenance. Both event output and checkpoint/finalization output are included. | +| `batches` | Bounded serialized create/merge batches, ordered within each revision. Stable span IDs and graph relationships remain in the JSON payload. Sink-only late-merge keys are preserved explicitly. | +| `continuation` | Versioned translator state and committed event input watermark. | +| `consumers` | Durable route registrations and acknowledged output sequence, plus input provenance for diagnostics. Actor retirement does not discard a paused consumer. | +| `aliases` | Maps captured route selection to its resolved attachment route so both names share one consumer cursor. | +| `controls` | Bounded finalization identity so repeated finalization at the same input boundary is idempotent even after payload collection. | + +Ignored input and controls without output commit continuation without creating an output revision. An input event produces bounded scratch batches. One SQLite transaction imports those batches, records the revision, saves the resulting continuation, and advances the input watermark. Scratch files are disposable and are not a recovery ledger. A crash before transaction commit leaves input available; a crash after commit permits input collection and downstream resumption without rereading it. If collection is interrupted, the remaining redundant input is skipped using the committed watermark. + +Output sequence numbers are distinct from event WAL offsets. Several output revisions, including checkpoint and finalization output, can have the same input boundary. A consumer advances only after all operations through its output sequence are accepted by checked delivery. Its exact failed operation is recorded within that revision. Consumer cursor advancement and deletion through the slowest cursor occur in a single SQLite transaction. Backend acceptance and a local SQLite commit cannot form one transaction; the existing destination projection ledger suppresses identical retransmission where possible. + +Delivery streams one revision and one bounded batch at a time. SQLite stores an operation ledger rather than a normalized or materialized graph. Local graph queries could add a derived index later; they are not needed for replay or merge semantics. + +A new historical consumer must reconstruct its requested history using transcript import before attaching to the available live suffix. A collected live-hook prefix is unavailable. Import and attach read transcripts directly and do not require either recovery ledger to contain historical data. + +### Capture and session partitioning + +The capture boundary should accept a native payload as opaque bytes or an +opaque structured value supplied by an in-process adapter, plus separately +captured non-secret metadata. `source` and a nonempty `session_id` are the +only required identity fields; the adapter may also supply its version, agent +version, receive time, route selection, managed-run ID, and process evidence. +Hook adapters extract the session ID from the native hook input using their +configured field. In-process JS adapters place the ID in the envelope; when +the agent event has no native ID, the adapter must assign one that consistently +groups that session's events. The current OpenCode adapter, for example, can +use a fallback ID. The daemon validates this identity before append and writes +directly to the `(source, session_id)` WAL partition. Missing identity is a +capture error, not a paused session: there is no trustworthy partition to +resume. Adapter-specific mistakes in assigning IDs need to be fixed at the +capture boundary. + +Hook adapters that receive JSON bytes should retain those bytes; adapters +that receive an object should serialize the complete object without +translator-specific projection. Neither route metadata nor daemon-generated +credentials should be mixed into the native payload. Append a framed, +checksummed record to the selected session partition before interpreting any +other native fields. The existing per-session journal files can be migrated +into that format. The physical segment layout should support bounded append +and streaming replay. + +Valid but unfamiliar native JSON is still a valid capture when the adapter +supplies `source` and `session_id`. An in-process adapter can preserve +syntactically invalid native bytes if it has that identity separately, then +translation pauses on a typed parse failure. A hook whose session ID must be +extracted from invalid JSON cannot meet the capture contract. A transport +frame that is absent or exceeds the wire size limit remains a capture error: +there is no complete event to append. + +The WAL record should carry a format version, monotonic capture sequence, +length, checksum, and enough metadata to interpret the native bytes. The reader +should stop at an incomplete final record after a crash without treating +earlier records as corrupt. Unknown WAL record versions must remain available +for a newer reader instead of being deleted. + +### Translation contract + +The translator receives ordered native events for one source session. It +returns an explicit disposition for each record: + +```rust +enum TranslationOutcome { + Produced(Vec), + Ignored { reason: IgnoreReason }, +} + +enum TranslationFailure { + UnsupportedShape { + source: String, + event: String, + source_version: Option, + translator_revision: String, + field_path: Option, + detail: String, + }, + Internal { translator_revision: String, detail: String }, +} +``` + +The `Result` boundary is important: + +- An event name the translator does not recognize is `Ignored::UnknownEvent`. + A known event that has no trace-bearing use is `Ignored::UnusedEvent`. + Neither blocks the stream. Both remain in the native WAL. +- A recognized event with unknown extra fields is generally supported. For a + field modeled as `Option`, absence is valid and represents `None`; when + the field is present, it must match the expected shape of `T`. A present + value with the wrong type or structure is `UnsupportedShape`, even when the + field is optional. Do not treat malformed data as though it were absent. +- A recognized event missing a required identity, relationship, lifecycle, or + content field returns `UnsupportedShape`. The source session pauses before + consuming that event. Later native events continue to be captured. +- Translator code errors are separate from native shape errors. Both are + associated with the revision of the running translator that failed. + +Each agent translator needs an explicit list of the event names it handles and +the fields that are genuinely required for each one. Existing `decode(...) +.unwrap_or_default()` and similar no-op paths should be audited one event at a +time; only required-shape failures change to errors. Do not add a blanket +strict decoder to arbitrary tool arguments, messages, provider payloads, or +metadata. A translator call must either commit its output and state transition +or fail without advancing the source translation cursor. If a failed call has +already mutated in-memory state, discard that translator instance and rebuild +it from the last committed position. + +An `UnsupportedShape` incident becomes eligible for replay only when a **new +running translator revision** is available. An installed binary changing on +disk does not change code in the current process. Version handover must start a +new daemon generation or perform an internal, journal-safe process handoff; no +user-facing restart command is required. A retry that fails under the new +revision records that revision and waits for another relevant change. + +### Route overlays and delivery + +Logical span revisions contain deterministic source-qualified span identities +and route-neutral content. A route overlay supplies destination-specific parent +attachment, tags, metadata, and project context before its plugins run. The +sink obtains an auth lease only when it is ready to submit output. A missing +login therefore creates an ordinary delivery failure for that route; it does +not prevent actor creation, native capture, or translation. + +Each route consumes the derived revision stream through its own cursor. A +plugin, credential, destination, rate-limit, or transport failure pauses that +route at the first undelivered operation. Other routes and source sessions +continue. The route does not retain an unbounded span queue in memory; it +re-reads durable revisions when it is eligible to retry. A route created after +a session has begun can backfill from the logical revision stream or rebuild it +from the native WAL while respecting its own destination and plugin chain. + +Translation failure is source-session scoped because every route needs that +translator's next logical revision. Route failures are delivery scoped. The +same recovery state model serves both scopes, but their progress cursors refer +to different streams. + +## Generic recovery state + +`PluginRecoveryState` should be replaced by a mode-neutral representation. A +persisted incident describes what blocked one work scope, not the status of a +particular plugin: + +```rust +enum WorkScope { + SourceSession { source: String, session_id: String }, + DeliveryRoute { source: String, session_id: String, route_id: RouteId }, +} + +enum WorkState { Paused, Reprocessing } + +struct Incident { + id: IncidentId, + scope: WorkScope, + first_unprocessed: Cursor, + cause: FailureCause, + local_error: String, + first_seen_ms: i64, + last_seen_ms: i64, + attempts: u64, + marker_span_id: Option, +} +``` + +The active state has no failure marker. Once work succeeds and its cursor is +durably advanced, move the incident to bounded local resolved history and set +the work state to `Active`. Doctor can show the resolution time without making +"recovered" a permanent processing mode or span field. The actor's in-memory +state is a cache of the persisted state, not a second authority. + +The cursor must identify the exact boundary. A source translation cursor is a +native record ID. A route delivery cursor is a logical revision ID and +operation ordinal within it. A native event can produce multiple operations; +checkpointing only at its event start would resend earlier successful +operations from that event. Plugin replay starts with the pristine revision at +the failed operation, reruns the ordered plugin chain, and does not resend +previously acknowledged operations. The route ID must remain stable when a +saved profile gains its canonical profile ID; credentials are not part of the +identity of a native event. + +### Failure causes and retry checks + +`FailureCause` is a serializable enum. Its variant holds only the evidence +needed to correlate a future check with the actual failure. It never stores a +credential or an agent payload. + +| Cause | Failure evidence | Retry candidate | +| --- | --- | --- | +| `InputShape` | Source, known event, agent version, translator revision, required-field error | A new running translator revision for that source. | +| `TranslatorFault` | Source and translator revision | A changed translator revision. Repeated failures under the same revision do not loop. | +| `PluginFile` | Canonical path, content digest or missing-file state, failing plugin position | That exact file's bytes or existence change. Edits to other plugins do not trigger replay. | +| `Credentials` | Non-secret auth selection and error class | Credential-store change if observable, or bounded re-resolution through the host provider. A successful lease makes replay eligible. | +| `Destination` or `Permission` | Route selection, backend status, and stable destination identity | The same destination becomes usable or its explicit route/auth selection changes; use bounded checks. Never silently redirect old records. | +| `RateLimited` | Backend retry time and destination | The server's `Retry-After` expires, then a bounded retry. | +| `LocalStorage` | SQLite lock, unavailable file, or disk write failure | Timed retry of the same transaction with the translator restored to its preceding continuation; committed cursors remain unchanged until success. | +| `Transport` | Endpoint identity and connection/timeout/server-error class | Capped exponential backoff with jitter; actual delivery confirms success. | +| `PermanentDelivery` | Typed rejection and route/sink revision | A relevant route or sink revision changes; doctor explains why time alone will not help. | + +Classification must happen at typed stage boundaries. Do not infer a cause by +matching text from `anyhow` or a JavaScript stack trace. `FailureCause` can +carry a public safe summary for a marker and a private full diagnostic for +doctor. Every variant implements a condition check that returns +`Unchanged`, `RetryCandidate`, or `CheckFailed`. `RetryCandidate` is not +`Recovered`: a plugin can change and still throw, a login can work while the +destination remains forbidden, and a reachable endpoint can still reject a +span. + +Plugin exceptions, load errors, and execution timeouts belong to `PluginFile` +when the failing invocation is attributable to that plugin. Keep the raised +execution limit from PR #130 (one second) as the initial policy, and record +the elapsed time and limit in doctor. A timeout alone is not evidence that +another plugin or the daemon revision should wake the held route. + +The scheduler checks only paused scopes, with per-cause intervals and bounded +work. A filesystem watcher can wake `PluginFile` checks promptly, backed by a +periodic digest comparison to avoid missed notifications. Auth, permission, +and transport checks use backoff and avoid one network probe per paused +session when many sessions share the same endpoint or profile. A candidate +starts a reprocessing attempt even when no new agent event arrives. + +### Replay and state transitions + +1. Capture keeps appending native records regardless of downstream state. +2. On a failure, persist the incident and its exact cursor before reporting + progress beyond that cursor. Do not send later output for the blocked scope. +3. A matching retry check moves the scope to `Reprocessing`. Snapshot the WAL + or derived-log tip so the attempt has a finite boundary. +4. For a source failure, restore committed translator continuation and resume the event WAL at its input watermark. For a downstream failure, read the SQLite span ledger from the consumer cursor and skip only the acknowledged operation prefix of the failed revision. Downstream recovery never rebuilds output from native events. +5. If replay fails again, keep the original unprocessed boundary unless an + acknowledged checkpoint moved it forward. Replace the incident cause with + the newly observed blocker and its evidence. A plugin fix followed by an + auth failure must become a `Credentials` incident, not a plugin retry loop. +6. After acknowledged delivery, persist the new cursor and resolve the + incident. Then process records captured after the snapshot tip. + +An actor may be retired from memory while paused, provided a durable scheduler +and cursor can recreate it for checks and replay. The whole daemon may idle +when there is a durable wakeup mechanism; neither actor lifetime nor new agent +traffic can be required for recovery. + +### Failure indicators in Braintrust + +The local incident is authoritative. A remote marker is optional because +translation may not have produced a span ID, or auth/the endpoint may be +unavailable. When the failed operation has a deterministic eventual span ID +and the backend is reachable, send a payload-free marker using that ID. For a +plugin failure its visible error remains `plugin failure: `; full +details stay in doctor. Record whether the marker was accepted, not merely +queued locally. + +If no marker was accepted, successful replay sends the normal span directly. +If a marker exists, successful replay sends the normally translated and +transformed span to the same ID and explicitly clears every marker-only field, +including `error`. It adds no recovery label, tag, or metadata. The backend's +audit history may still record the earlier marker write; the current span +should have the uninterrupted result's fields. Marker writes and replacement +must bypass terminal-span suppression without making ordinary replay emit +duplicate updates. + +## Durability, acknowledgement, and retention + +The capture writer calls `sync_data()` before acknowledging a hook. Event WAL framing preserves logical byte offsets across prefix collection and rejects truncated or checksummed-invalid records. A torn final record is truncated on the next writer open without discarding earlier complete records. + +Only the committed translation watermark permits event collection. Translation failures protect the failed event and its later input suffix. Plugin, auth, and network failures protect the span ledger suffix instead; they do not protect already-translated events. Resolved diagnostic history protects neither ledger. + +Only checked backend acceptance permits consumer cursor advancement. Collection deletes output through the minimum acknowledged sequence of all registered consumers, including paused consumers. Cursor updates and output deletion are transactional. Continuation is retained independently, so open outer spans can receive future merges after their old operations have been collected. + +Compatibility for existing released event journals and span identity namespaces remains. The unreleased PR #130 plugin recovery state is not a supported input format. No historical hook archive or migration of that plugin-specific state is provided. + +Native payloads can contain conversation content and secrets that a later +span plugin would remove before upload. WAL files, SQLite databases, and scratch files need local +access controls; doctor must expose failure causes and offsets without dumping +payloads. Credentials resolved by the host remain outside these records. + +## Doctor and operational behavior + +Doctor should report source and session, affected route or source-level scope, +the first blocked record/operation, the number of later records and bytes +captured, failure cause and full local error, last check and attempt, next +eligible check, marker status, and any resolved incident. It should explain +whether the stream is actively translating/delivering, paused, or reprocessing. +Status and flush should count paused work as pending. Healthy sessions and +routes continue without bulk replay or a daemon lifecycle command. + +The capture acknowledgement says only that the WAL accepted the event. It +does not claim that a translator recognized it or that the backend accepted +its spans. A journal append or sync failure remains a capture error returned +to the hook, because no recovery record exists for that event. + +## Implementation status and limits + +The daemon now appends session-scoped native records before translation, +translates each source session once into durable route-neutral revisions, and +lets each route apply its own overlay, plugin chain, acknowledgement ledger, +and sink. Missing route credentials no longer prevent actor creation or WAL +capture. Recognized malformed shapes create source-scoped incidents; unknown +and intentionally unused events remain in the WAL without blocking progress. + +Persisted incidents cover source translation and route delivery scopes. The +actor's in-memory pause cache contains only marker/progress state; the cause, +retry check, and first-unprocessed position come from the persisted incident. +Plugin retries compare the digest of the exact failing path. Marker replacement +uses the original span ID and clears marker-only fields from the current span. +Resolved incidents move to bounded local history; no recovered label remains +on successful spans. Native events are collected after atomic span production; span payloads are collected through the slowest consumer. Translator-owned transcript references remain separate from downstream recovery; downstream replay never consults them. Maintenance collects old mirrors only when remaining input, committed continuation, and current observation templates no longer reference them. + +Backend acceptance is checked through the opt-in SDK API in the stacked SDK +dependency. A crash after backend acceptance and before local acknowledgement +can still cause a retry; the destination-visible projection ledger suppresses +identical writes. This does not promise exactly-once external side effects. +The translator revision is the running daemon binary digest, so an upgrade +replays translation failures after the daemon starts with the new binary. + +The automated suite exercises WAL recovery, generic incidents, replay cursors, +per-route delivery, doctor output, and translator fixtures. Live remote span +replacement remains a manual integration check requiring a valid Braintrust +login and an endpoint reachable by the test daemon. diff --git a/bt-daemon/src/command_output.rs b/bt-daemon/src/command_output.rs index e9835efd..4e7ab777 100644 --- a/bt-daemon/src/command_output.rs +++ b/bt-daemon/src/command_output.rs @@ -157,6 +157,8 @@ pub struct DoctorCommandOutput { pub auth: AuthDiagnostic, pub daemon: DaemonDiagnostic, pub warnings: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub recovery_incidents: Vec, pub plugin_diagnostics: Vec, } @@ -274,35 +276,35 @@ impl TraceCommandOutput { for warning in &doctor.warnings { rendered.push_str(&format!("\nWarning: {warning}")); } + for incident in &doctor.recovery_incidents { + let (source, session_id) = incident.scope.source_session(); + rendered.push_str(&format!( + "\nRecovery: {:?}\nSource: {}\nSession: {}\nRecovery cursor ({:?}): {}\nOperation index: {}\nCause: {:?}\nDetail: {}\nAttempts: {}", + incident.state, + source, + session_id, + incident.scope, + incident.first_unprocessed, + incident.operation_index, + incident.cause, + incident.local_error, + incident.attempts, + )); + } for diagnostic in &doctor.plugin_diagnostics { - let state = match diagnostic.state { - Some(crate::plugin_diagnostics::PluginRecoveryState::Paused) => "paused", - Some(crate::plugin_diagnostics::PluginRecoveryState::Reprocessing) => "reprocessing", - Some(crate::plugin_diagnostics::PluginRecoveryState::Recovered) => "recovered", - None => "historical (current state unknown)", - }; rendered.push_str(&format!( - "\nSpan plugin: {}\nState: {}\nSource: {}\nSession: {}\nSpan: {}\nOperation: {}\nFailure journal offset: {}\nPending journal bytes: {}\nFirst failure: {}\nLast failure: {}\nCause: {}", + "\nSpan plugin failure history: {}\nSource: {}\nSession: {}\nSpan: {}\nOperation: {}\nFailure span ledger cursor: {}\nPending span revisions at last report: {}\nFirst failure: {}\nLast failure: {}\nCause: {}", diagnostic.plugin_path.display(), - state, diagnostic.source, diagnostic.session_id.as_deref().unwrap_or("unknown"), diagnostic.span_id.as_deref().unwrap_or("unknown"), diagnostic.operation.as_deref().unwrap_or("unknown"), - diagnostic.journal_start.unwrap_or(0), - diagnostic.pending_bytes.unwrap_or(0), + diagnostic.span_cursor.unwrap_or(0), + diagnostic.pending_revisions.unwrap_or(0), render_timestamp(diagnostic.first_seen_ms), render_timestamp(diagnostic.last_seen_ms), diagnostic.exception )); - if diagnostic.state == Some(crate::plugin_diagnostics::PluginRecoveryState::Recovered) { - if let Some(when) = diagnostic.state_changed_ms { - rendered.push_str(&format!("\nRecovered: {}", render_timestamp(when))); - } - } - if diagnostic.state == Some(crate::plugin_diagnostics::PluginRecoveryState::Paused) { - rendered.push_str("\nNext: edit the failing plugin; tracing will retry this session automatically."); - } } Ok(rendered) } @@ -554,6 +556,25 @@ mod tests { error: None, }, warnings: Vec::new(), + recovery_incidents: vec![crate::RecoveryIncident { + scope: crate::WorkScope::SourceSession { + source: "pi".into(), + session_id: "pi-session".into(), + }, + state: crate::WorkState::Paused, + first_unprocessed: 42, + operation_index: 0, + cause: crate::FailureCause::InputShape { + event: "tool_execution_start".into(), + translator_revision: "revision-a".into(), + }, + local_error: "missing toolCallId".into(), + marker_span_id: None, + first_seen_ms: 1, + last_seen_ms: 2, + attempts: 0, + resolved_at_ms: None, + }], plugin_diagnostics: vec![crate::PluginDiagnostic { source: "codex".into(), plugin_path: PathBuf::from("/tmp/redact.mjs"), @@ -565,8 +586,7 @@ mod tests { session_id: Some("session-1".into()), span_id: Some("span-1".into()), operation: Some("merge".into()), - state: Some(crate::plugin_diagnostics::PluginRecoveryState::Paused), - pending_bytes: Some(1024), + pending_revisions: Some(1024), ..Default::default() }], }); @@ -576,6 +596,14 @@ mod tests { assert_eq!(value["auth"]["source"], "saved_profile"); assert_eq!(value["daemon"]["status"], "running"); assert_eq!(value["daemon"]["auth"]["status"], "error"); + assert_eq!( + value["recovery_incidents"][0]["scope"]["session_id"], + "pi-session" + ); + assert_eq!( + value["recovery_incidents"][0]["cause"]["kind"], + "input_shape" + ); assert_eq!( value["daemon"]["session_errors"][0], "could not resolve Braintrust auth for codex" @@ -587,10 +615,9 @@ mod tests { "Error: raw secret\n at redact (redact.mjs:1)" ); let human = output.render(OutputFormat::Human).unwrap(); - assert!(human.contains("State: paused")); + assert!(human.contains("Span plugin failure history: /tmp/redact.mjs")); assert!(human.contains("Session: session-1")); assert!(human.contains("Span: span-1")); - assert!(human.contains("Pending journal bytes: 1024")); - assert!(human.contains("Next: edit the failing plugin")); + assert!(human.contains("Pending span revisions at last report: 1024")); } } diff --git a/bt-daemon/src/delivery_ledger.rs b/bt-daemon/src/delivery_ledger.rs index f635fa7b..0470e1fa 100644 --- a/bt-daemon/src/delivery_ledger.rs +++ b/bt-daemon/src/delivery_ledger.rs @@ -1,17 +1,16 @@ -//! Destination-scoped completed-span delivery ledger. +//! Destination-scoped checked-delivery and visible-span projection ledger. //! //! Hook journals prevent a daemon recovery from re-emitting already flushed //! observations. Imports are a second ingress, though: they synthesize the -//! same source session without using that journal. This ledger records which -//! completed span rows a destination has already accepted, so every ingress can -//! avoid re-reporting any later partial merge for a completed span while a new -//! destination still receives the full trace. +//! same source session without using that journal. This ledger records accepted +//! span images and completed spans, so retries can skip unchanged merges while +//! a new destination still receives the full trace. use crate::sink::Sink; use crate::translate::{SpanOp, SpanRow}; use crate::wire::SessionConfig; use sha2::{Digest, Sha256}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; #[derive(serde::Serialize, serde::Deserialize, Default)] @@ -20,6 +19,8 @@ struct LedgerFile { completed_span_ids: HashSet, #[serde(default, alias = "late_merge_span_ids")] late_merge_ids: HashSet, + #[serde(default)] + span_images: HashMap, } struct DeliveryLedger { @@ -28,6 +29,8 @@ struct DeliveryLedger { pending: HashSet, known_late_merges: HashSet, pending_late_merges: HashSet, + known_images: HashMap, + pending_images: HashMap, } impl DeliveryLedger { @@ -50,6 +53,10 @@ impl DeliveryLedger { crate::ids::session_storage_id(source, session_id), &fingerprint_id[..32] )); + crate::paths::ensure_private_dir(path.parent().expect("ledger path has a parent"))?; + if tokio::fs::metadata(&path).await.is_ok() { + crate::paths::restrict_file_to_owner(&path)?; + } let persisted = match tokio::fs::read(&path).await { Ok(bytes) => serde_json::from_slice::(&bytes)?, Err(error) if error.kind() == std::io::ErrorKind::NotFound => LedgerFile::default(), @@ -61,24 +68,42 @@ impl DeliveryLedger { pending: HashSet::new(), known_late_merges: persisted.late_merge_ids, pending_late_merges: HashSet::new(), + known_images: persisted.span_images, + pending_images: HashMap::new(), }) } fn filter(&self, ops: &[SpanOp]) -> Vec { - ops.iter() - .filter(|op| { - let row = match op { - SpanOp::Insert(row) | SpanOp::Merge(row) => row, - }; - if let Some(key) = &row.late_merge_key { - let id = late_merge_id(row, key); - !self.known_late_merges.contains(&id) && !self.pending_late_merges.contains(&id) - } else { - !self.known.contains(&row.span_id) && !self.pending.contains(&row.span_id) + let mut images = self.known_images.clone(); + images.extend(self.pending_images.clone()); + let mut terminal = self.known.clone(); + terminal.extend(self.pending.clone()); + let mut late = self.known_late_merges.clone(); + late.extend(self.pending_late_merges.clone()); + let mut filtered = Vec::new(); + for op in ops { + let row = match op { + SpanOp::Insert(row) | SpanOp::Merge(row) => row, + }; + if let Some(key) = &row.late_merge_key { + let id = late_merge_id(row, key); + if !late.insert(id) { + continue; } - }) - .cloned() - .collect() + } else if terminal.contains(&row.span_id) { + continue; + } + let next = next_image(images.get(&row.span_id), op); + if images.get(&row.span_id) == Some(&next) { + continue; + } + images.insert(row.span_id.clone(), next); + if row.end_ms.is_some() { + terminal.insert(row.span_id.clone()); + } + filtered.push(op.clone()); + } + filtered } fn record_emitted(&mut self, ops: &[SpanOp]) { @@ -92,29 +117,46 @@ impl DeliveryLedger { if let Some(key) = &row.late_merge_key { self.pending_late_merges.insert(late_merge_id(row, key)); } + let current = self + .pending_images + .get(&row.span_id) + .or_else(|| self.known_images.get(&row.span_id)); + self.pending_images + .insert(row.span_id.clone(), next_image(current, op)); } } async fn commit(&mut self) -> anyhow::Result<()> { - if self.pending.is_empty() && self.pending_late_merges.is_empty() { + if self.pending.is_empty() + && self.pending_late_merges.is_empty() + && self.pending_images.is_empty() + { return Ok(()); } self.known.extend(self.pending.drain()); self.known_late_merges .extend(self.pending_late_merges.drain()); + self.known_images.extend(self.pending_images.drain()); let parent = self.path.parent().expect("ledger path has a parent"); - tokio::fs::create_dir_all(parent).await?; + crate::paths::ensure_private_dir(parent)?; let temp = self .path .with_extension(format!("{}.tmp", uuid::Uuid::new_v4())); - tokio::fs::write( - &temp, - serde_json::to_vec(&LedgerFile { - completed_span_ids: self.known.clone(), - late_merge_ids: self.known_late_merges.clone(), - })?, - ) - .await?; + let bytes = serde_json::to_vec(&LedgerFile { + completed_span_ids: self.known.clone(), + late_merge_ids: self.known_late_merges.clone(), + span_images: self.known_images.clone(), + })?; + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(&temp).await?; + use tokio::io::AsyncWriteExt; + file.write_all(&bytes).await?; + file.flush().await?; + file.sync_data().await?; + drop(file); if let Err(first_error) = tokio::fs::rename(&temp, &self.path).await { let _ = tokio::fs::remove_file(&self.path).await; tokio::fs::rename(&temp, &self.path) @@ -125,10 +167,31 @@ impl DeliveryLedger { ) })?; } + crate::paths::restrict_file_to_owner(&self.path)?; Ok(()) } } +/// Project a row operation using the SDK's merge behavior: an insert replaces +/// the prior image, while a merge overwrites only fields present in the row. +fn next_image(previous: Option<&serde_json::Value>, op: &SpanOp) -> serde_json::Value { + let row = match op { + SpanOp::Insert(row) | SpanOp::Merge(row) => row, + }; + let next = serde_json::to_value(row).unwrap_or(serde_json::Value::Null); + if matches!(op, SpanOp::Insert(_)) { + return next; + } + let mut merged = previous + .and_then(serde_json::Value::as_object) + .cloned() + .unwrap_or_default(); + if let Some(changes) = next.as_object() { + merged.extend(changes.clone()); + } + serde_json::Value::Object(merged) +} + fn late_merge_id(row: &SpanRow, key: &str) -> String { format!("{}\u{1f}{key}", row.span_id) } @@ -342,7 +405,7 @@ mod tests { } #[tokio::test] - async fn a_completed_span_receives_each_distinct_late_merge_once() { + async fn identical_late_merges_are_suppressed_even_with_distinct_internal_keys() { let temp = tempfile::tempdir().unwrap(); let first = RecordingSink::default(); let mut first = LedgerSink::new( @@ -370,11 +433,11 @@ mod tests { .emit(&[late_merge("span-1", "output")]) .await .unwrap(), - 1 + 0 ); assert_eq!( second.emit(&[late_merge("span-1", "usage")]).await.unwrap(), - 1 + 0 ); assert_eq!( second diff --git a/bt-daemon/src/derived.rs b/bt-daemon/src/derived.rs new file mode 100644 index 00000000..a36d5ec7 --- /dev/null +++ b/bt-daemon/src/derived.rs @@ -0,0 +1,969 @@ +//! Durable, route-neutral span operation ledger produced from the event WAL. +//! One source session translates each native record once. Route actors stream +//! the committed revisions and apply their own attachment and metadata. + +use crate::journal::{self, JournalReader, JournalRecord}; +use crate::translate::{AgentTranslator, Registry, SessionCtx, SpanOp, SpanRow}; +use crate::wire::{Envelope, SessionConfig}; +use rusqlite::{params, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::io::AsyncWriteExt; + +#[derive(Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ControlKind { + Checkpoint, + Finalize, +} + +#[derive(Serialize, Deserialize)] +struct Batch { + ops: Vec, + /// SpanOp's late-merge key is sink-only and skipped by normal serde. + late_merge_keys: Vec>, +} + +impl Batch { + fn new(ops: Vec) -> Self { + let late_merge_keys = ops + .iter() + .map(|op| row(op).late_merge_key.clone()) + .collect(); + Self { + ops, + late_merge_keys, + } + } + + fn into_ops(mut self) -> Vec { + for (op, key) in self.ops.iter_mut().zip(self.late_merge_keys) { + row_mut(op).late_merge_key = key; + } + self.ops + } +} + +fn row(op: &SpanOp) -> &SpanRow { + match op { + SpanOp::Insert(row) | SpanOp::Merge(row) => row, + } +} + +fn row_mut(op: &mut SpanOp) -> &mut SpanRow { + match op { + SpanOp::Insert(row) | SpanOp::Merge(row) => row, + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct SpanRevision { + pub start: u64, + pub through: u64, + pub envelope: Option, + pub sequence: u64, + pub id: String, +} + +pub(crate) fn retryable_storage(error: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(error, _)) = error.downcast_ref::() + { + use rusqlite::ErrorCode::*; + return matches!( + error.code, + DatabaseBusy + | DatabaseLocked + | SystemIoFailure + | DiskFull + | CannotOpen + | ReadOnly + | PermissionDenied + ); + } + error.downcast_ref::().is_some() +} + +#[derive(Debug, thiserror::Error)] +#[error("event WAL offset {start}: {message}")] +pub(crate) struct InputFailurePosition { + pub start: u64, + pub message: String, +} + +struct State { + translator: Box, + native_through: u64, + poisoned: bool, +} + +pub struct SourceTranslation { + source: String, + session_id: String, + dir: PathBuf, + journal_path: PathBuf, + state: tokio::sync::Mutex, +} + +fn open_database(path: &Path) -> anyhow::Result { + let db = rusqlite::Connection::open(path)?; + let version: i64 = db.pragma_query_value(None, "user_version", |r| r.get(0))?; + anyhow::ensure!( + version <= 1, + "unsupported span ledger schema version {version}" + ); + db.busy_timeout(std::time::Duration::from_secs(5))?; + db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA auto_vacuum=INCREMENTAL; + CREATE TABLE IF NOT EXISTS continuation ( + singleton INTEGER PRIMARY KEY CHECK(singleton=1), through INTEGER NOT NULL, + version INTEGER NOT NULL, state TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS revisions ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT UNIQUE NOT NULL, start INTEGER NOT NULL, through INTEGER NOT NULL, + envelope TEXT, kind TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS revision_order ON revisions(through,kind); + CREATE TABLE IF NOT EXISTS batches ( + revision TEXT NOT NULL, ordinal INTEGER NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY(revision,ordinal)); + CREATE TABLE IF NOT EXISTS consumers ( + route TEXT PRIMARY KEY, through INTEGER NOT NULL DEFAULT 0, sequence INTEGER NOT NULL DEFAULT 0); + CREATE TABLE IF NOT EXISTS controls(kind TEXT PRIMARY KEY, through INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS aliases(captured TEXT PRIMARY KEY, effective TEXT NOT NULL);")?; + if version == 0 { + db.pragma_update(None, "user_version", 1)?; + } + crate::paths::restrict_file_to_owner(path)?; + Ok(db) +} + +fn route_key(route: &crate::wire::SessionRoute) -> anyhow::Result { + let mut route = route.clone(); + route.auth.profile_id = None; + route.auth.source = route.auth.effective_source(); + Ok(serde_json::to_string(&route)?) +} +fn effective_key( + db: &rusqlite::Connection, + route: &crate::wire::SessionRoute, +) -> anyhow::Result { + let key = route_key(route)?; + Ok(db + .query_row( + "SELECT effective FROM aliases WHERE captured=?1", + [&key], + |r| r.get(0), + ) + .optional()? + .unwrap_or(key)) +} + +pub(crate) fn checkpoints( + data_dir: &Path, + source: &str, + session_id: &str, +) -> anyhow::Result> { + let path = data_dir + .join("derived") + .join(crate::ids::session_storage_id(source, session_id)) + .join("spans.sqlite"); + if !path.exists() { + return Ok(Vec::new()); + } + let db = open_database(&path)?; + let mut stmt = db.prepare("SELECT route,through FROM consumers")?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?; + rows.map(|r| { + let (route, through) = r?; + Ok((serde_json::from_str(&route)?, through as u64)) + }) + .collect() +} +pub(crate) fn pending_output( + data_dir: &Path, + source: &str, + session_id: &str, + route: &crate::wire::SessionRoute, +) -> anyhow::Result { + let path = data_dir + .join("derived") + .join(crate::ids::session_storage_id(source, session_id)) + .join("spans.sqlite"); + if !path.exists() { + return Ok(false); + } + let db = open_database(&path)?; + Ok(db.query_row("SELECT EXISTS(SELECT 1 FROM revisions WHERE sequence>COALESCE((SELECT sequence FROM consumers WHERE route=?1),0))",[effective_key(&db,route)?],|r|r.get(0))?) +} + +impl SourceTranslation { + pub fn new( + source: &str, + session_id: &str, + translator_session_id: &str, + data_dir: &Path, + registry: &Registry, + ) -> anyhow::Result> { + let mut translator = + registry.create_checked_with_session_namespace(source, translator_session_id)?; + let dir = data_dir + .join("derived") + .join(crate::ids::session_storage_id(source, session_id)); + crate::paths::ensure_private_dir(&dir)?; + let db = open_database(&dir.join("spans.sqlite"))?; + let saved: Option<(i64, u32, String)> = db + .query_row( + "SELECT through,version,state FROM continuation WHERE singleton=1", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional()?; + let native_through = if let Some((through, version, snapshot)) = saved { + anyhow::ensure!(version == 1, "unsupported translator continuation version"); + translator.restore(serde_json::from_str(&snapshot)?)?; + through as u64 + } else { + 0 + }; + Ok(Arc::new(Self { + source: source.into(), + session_id: session_id.into(), + dir, + journal_path: journal::source_journal_path(data_dir, source, session_id), + state: tokio::sync::Mutex::new(State { + translator, + native_through, + poisoned: false, + }), + })) + } + + fn event_path(&self, through: u64) -> PathBuf { + self.dir.join(format!("event-{through:016x}")) + } + fn control_path(&self, kind: ControlKind, through: u64) -> PathBuf { + let kind = match kind { + ControlKind::Checkpoint => "checkpoint", + ControlKind::Finalize => "finalize", + }; + self.dir.join(format!("{kind}-{through:016x}")) + } + fn neutral_ctx(&self) -> SessionCtx { + SessionCtx { + session_id: self.session_id.clone(), + config: None, + } + } + + /// Produce committed revisions from captured input. Only this upstream stage opens the event WAL. + pub async fn ensure_event(&self, through: u64) -> anyhow::Result { + let mut state = self.state.lock().await; + anyhow::ensure!( + !state.poisoned, + "source translator needs a new daemon revision" + ); + for captured in journal::captured_routes(&self.journal_path)? { + if let Some(route) = captured.envelope.route { + self.register(&route, 0)?; + } + } + if state.native_through >= through { + return Ok(self.event_path(through)); + } + let mut reader = + JournalReader::open_from(&self.journal_path, state.native_through, through) + .await? + .ok_or_else(|| anyhow::anyhow!("native session WAL is missing"))?; + while let Some(entry) = reader.next_record().await? { + let (env, legacy_checkpoint) = match entry.record { + JournalRecord::Event(redacted) => { + let mut env = journal::envelope_from_redacted(redacted); + if let Some(route) = env.route.as_ref() { + self.register(route, 0)?; + } + if crate::translate::canonical_source_name(&env.source) == self.source { + env.source = self.source.clone(); + env.config = None; + (Some(env), None) + } else { + (None, None) + } + } + JournalRecord::DeliveryCheckpoint { route, through } => { + (None, Some((route, through))) + } + }; + let before = state.native_through; + let ctx = self.neutral_ctx(); + let rollback = state.translator.snapshot()?; + let result = async { + let first = if let Some(env) = &env { + state.translator.handle(env, &ctx)? + } else { + Vec::new() + }; + let scratch = self + .stage_batches(first, || state.translator.drain_pending(&ctx)) + .await?; + let envelope = env.as_ref().map(|env| { + let mut envelope = env.redacted(); + envelope.payload = Value::Null; + envelope.route = None; + envelope.capture = None; + envelope.managed_run_id = None; + envelope + }); + let snapshot = state.translator.snapshot()?; + self.commit( + &self.event_path(entry.through), + (before, entry.through), + envelope, + "event", + snapshot, + scratch, + ) + .await + } + .await; + if let Err(error) = result { + if retryable_storage(&error) { + state.translator.restore(rollback)?; + } else { + state.poisoned = true; + } + let message = error.to_string(); + return Err(error.context(InputFailurePosition { + start: before, + message, + })); + } + state.native_through = entry.through; + if let Some((route, through)) = legacy_checkpoint { + let db = open_database(&self.dir.join("spans.sqlite"))?; + let sequence = db.query_row( + "SELECT COALESCE(MAX(sequence),0) FROM revisions WHERE through<=?1", + [through as i64], + |r| r.get::<_, i64>(0), + )? as u64; + drop(db); + self.register(&route, 0)?; + self.acknowledge(&route, sequence).await?; + } + } + anyhow::ensure!( + state.native_through >= through, + "native WAL ended before requested revision {through}" + ); + Ok(self.event_path(through)) + } + + // Scratch output is uncommitted and disposable. Import it in one transaction with continuation + // state, without accumulating a potentially large transcript's operations in memory. + async fn stage_batches( + &self, + first: Vec, + mut next: impl FnMut() -> anyhow::Result>>, + ) -> anyhow::Result { + let path = self.dir.join(format!("{}.tmp", uuid::Uuid::new_v4())); + let result = async { + let mut file = crate::durable::create_private(&path).await?; + let mut batch = Some(first); + while let Some(ops) = batch { + let mut bytes = serde_json::to_vec(&Batch::new(ops))?; + bytes.push(b'\n'); + file.write_all(&bytes).await?; + batch = next()?; + } + file.flush().await?; + Ok::<_, anyhow::Error>(()) + } + .await; + if let Err(error) = result { + let _ = tokio::fs::remove_file(&path).await; + return Err(error); + } + Ok(path) + } + + async fn commit( + &self, + path: &Path, + window: (u64, u64), + envelope: Option, + kind: &str, + snapshot: Value, + scratch: PathBuf, + ) -> anyhow::Result<()> { + let (start, through) = window; + let db_path = self.dir.join("spans.sqlite"); + let id = path.file_name().unwrap().to_string_lossy().into_owned(); + let kind = kind.to_owned(); + let cleanup = scratch.clone(); + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let mut db = open_database(&db_path)?; + let tx = db.transaction()?; + let mut created = false; + let mut ordinal = 0i64; + for line in BufReader::new(std::fs::File::open(scratch)?).lines() { + let line = line?; + let batch: Batch = serde_json::from_str(&line)?; + if batch.ops.is_empty() { + continue; + } + if !created { + tx.execute("INSERT INTO revisions(id,start,through,envelope,kind) VALUES(?1,?2,?3,?4,?5)",params![id,start as i64,through as i64,envelope.as_ref().map(serde_json::to_string).transpose()?,kind])?; + created = true; + } + tx.execute("INSERT INTO batches VALUES(?1,?2,?3)",params![id,ordinal,line])?; + ordinal += 1; + } + tx.execute("INSERT INTO continuation VALUES(1,?1,1,?2) ON CONFLICT(singleton) DO UPDATE SET through=excluded.through,state=excluded.state", + params![through as i64,serde_json::to_string(&snapshot)?])?; + if id.starts_with("finalize-") { + tx.execute("INSERT INTO controls VALUES('finalize',?1) ON CONFLICT(kind) DO UPDATE SET through=excluded.through",[through as i64])?; + } + tx.commit()?; + Ok(()) + }).await?; + let _ = tokio::fs::remove_file(cleanup).await; + result + } + + pub async fn ensure_control(&self, kind: ControlKind, through: u64) -> anyhow::Result { + let mut state = self.state.lock().await; + anyhow::ensure!( + !state.poisoned, + "source translator needs a new daemon revision" + ); + anyhow::ensure!( + state.native_through >= through, + "source translator has not reached native offset {through}" + ); + let through = state.native_through; + let path = match kind { + ControlKind::Checkpoint => self + .dir + .join(format!("checkpoint-{}", uuid::Uuid::new_v4())), + ControlKind::Finalize => self.control_path(kind, through), + }; + let exists = matches!(kind, ControlKind::Finalize) && { + let db = open_database(&self.dir.join("spans.sqlite"))?; + db.query_row( + "SELECT 1 FROM controls WHERE kind='finalize' AND through=?1", + [through as i64], + |r| r.get::<_, u32>(0), + ) + .optional()? + .is_some() + }; + if exists { + return Ok(path); + } + anyhow::ensure!( + state.native_through >= through, + "source translator has not reached native offset {through}" + ); + let ctx = self.neutral_ctx(); + let rollback = state.translator.snapshot()?; + let result = async { + let first = match kind { + ControlKind::Checkpoint => state.translator.checkpoint(&ctx), + ControlKind::Finalize => state.translator.finalize(&ctx), + }?; + let scratch = self + .stage_batches(first, || state.translator.drain_pending(&ctx)) + .await?; + self.commit( + &path, + (through, state.native_through), + None, + "control", + state.translator.snapshot()?, + scratch, + ) + .await + } + .await; + if let Err(error) = result { + if retryable_storage(&error) { + state.translator.restore(rollback)?; + } else { + state.poisoned = true; + } + return Err(error); + } + Ok(path) + } + + pub(crate) async fn collect_input( + &self, + journal: &tokio::sync::Mutex, + ) -> anyhow::Result<()> { + let state = self.state.lock().await; + journal + .lock() + .await + .collect_through(state.native_through) + .await + } + + /// Automatic parent selection changes the effective route, not the identity of its consumer. + /// Bind the captured route to that resolved destination so it cannot leave a phantom cursor. + pub(crate) fn bind_route( + &self, + captured: &crate::wire::SessionRoute, + effective: &crate::wire::SessionRoute, + ) -> anyhow::Result<()> { + let captured = route_key(captured)?; + let effective = route_key(effective)?; + if captured == effective { + return Ok(()); + } + let mut db = open_database(&self.dir.join("spans.sqlite"))?; + let tx = db.transaction()?; + tx.execute("INSERT INTO aliases VALUES(?1,?2) ON CONFLICT(captured) DO UPDATE SET effective=excluded.effective",params![captured,effective])?; + tx.execute("INSERT INTO consumers(route,through,sequence) SELECT ?2,through,sequence FROM consumers WHERE route=?1 ON CONFLICT(route) DO UPDATE SET through=MIN(through,excluded.through),sequence=MIN(sequence,excluded.sequence)",params![captured,effective])?; + tx.execute("DELETE FROM consumers WHERE route=?1", [captured])?; + tx.commit()?; + Ok(()) + } + + /// Durable consumer registration survives actor retirement and daemon crashes. + pub(crate) fn register( + &self, + route: &crate::wire::SessionRoute, + through: u64, + ) -> anyhow::Result { + let db = open_database(&self.dir.join("spans.sqlite"))?; + let route = effective_key(&db, route)?; + db.execute( + "INSERT OR IGNORE INTO consumers(route,through) VALUES(?1,?2)", + params![route, through as i64], + )?; + Ok(db.query_row( + "SELECT through FROM consumers WHERE route=?1", + [route], + |r| r.get::<_, i64>(0), + )? as u64) + } + + pub(crate) async fn acknowledge( + &self, + route: &crate::wire::SessionRoute, + sequence: u64, + ) -> anyhow::Result { + let path = self.dir.join("spans.sqlite"); + let route = route_key(route)?; + tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let mut db=open_database(&path)?;let tx=db.transaction()?; + tx.execute("UPDATE consumers SET through=MAX(through,COALESCE(CASE WHEN ?2>=COALESCE((SELECT MAX(sequence) FROM revisions),0) THEN (SELECT through FROM continuation WHERE singleton=1) ELSE (SELECT through FROM revisions WHERE sequence=?2) END,through)),sequence=MAX(sequence,?2) WHERE route=?1",params![route,sequence as i64])?; + let through=tx.query_row("SELECT through FROM consumers WHERE route=?1",[route],|r|r.get::<_,i64>(0))?; + let floor:Option=tx.query_row("SELECT MIN(sequence) FROM consumers",[],|r|r.get(0))?; + if let Some(floor)=floor { + tx.execute("DELETE FROM batches WHERE revision IN(SELECT id FROM revisions WHERE sequence<=?1)",[floor])?; + tx.execute("DELETE FROM revisions WHERE sequence<=?1",[floor])?; + } + tx.commit()?; + // Freed pages are reusable immediately. Incremental vacuum bounds disk retained after a large backlog. + db.execute_batch("PRAGMA incremental_vacuum(128)")?; + Ok(through as u64) + }).await? + } + + pub(crate) async fn consumer_sequence( + &self, + route: &crate::wire::SessionRoute, + ) -> anyhow::Result { + let path = self.dir.join("spans.sqlite"); + let route = route_key(route)?; + tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let db = open_database(&path)?; + Ok(db.query_row( + "SELECT sequence FROM consumers WHERE route=?1", + [route], + |r| r.get::<_, i64>(0), + )? as u64) + }) + .await? + } + pub(crate) async fn tip(&self) -> anyhow::Result { + let path = self.dir.join("spans.sqlite"); + tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let db = open_database(&path)?; + Ok( + db.query_row("SELECT COALESCE(MAX(sequence),0) FROM revisions", [], |r| { + r.get::<_, i64>(0) + })? as u64, + ) + }) + .await? + } + /// Read one revision at a time so replay memory is bounded independently of session length. + pub(crate) async fn next_revision( + &self, + after: u64, + tip: u64, + ) -> anyhow::Result> { + let path = self.dir.join("spans.sqlite"); + tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let db=open_database(&path)?; + let value=db.query_row("SELECT start,through,envelope,id,sequence FROM revisions WHERE sequence>?1 AND sequence<=?2 ORDER BY sequence LIMIT 1", + params![after as i64,tip as i64],|r|Ok((r.get::<_,i64>(0)?,r.get::<_,i64>(1)?,r.get::<_,Option>(2)?,r.get::<_,String>(3)?,r.get::<_,i64>(4)?))).optional()?; + value.map(|(start,through,env,id,sequence)| Ok(SpanRevision { start:start as u64,through:through as u64, + envelope:env.map(|s|serde_json::from_str(&s)).transpose()?,id,sequence:sequence as u64 })).transpose() + }).await? + } + #[cfg(test)] + async fn revisions( + &self, + route: &crate::wire::SessionRoute, + _through: u64, + ) -> anyhow::Result> { + let mut after = self.consumer_sequence(route).await?; + let tip = self.tip().await?; + let mut result = Vec::new(); + while let Some(revision) = self.next_revision(after, tip).await? { + after = revision.sequence; + result.push(revision); + } + Ok(result) + } + pub(crate) fn revision_path_for_id(&self, id: &str) -> PathBuf { + self.dir.join(id) + } + pub fn route_translator(self: &Arc) -> Box { + Box::new(DerivedTranslator { + dir: self.dir.clone(), + pending: None, + sequence: 0, + }) + } +} + +struct DerivedTranslator { + dir: PathBuf, + pending: Option<(String, u64)>, + sequence: u64, +} +impl DerivedTranslator { + fn read_next(&mut self, ctx: &SessionCtx) -> anyhow::Result>> { + let Some((id, ordinal)) = self.pending.as_mut() else { + return Ok(None); + }; + let db = open_database(&self.dir.join("spans.sqlite"))?; + if *ordinal == 0 { + self.sequence = db + .query_row( + "SELECT sequence FROM revisions WHERE id=?1", + [id.as_str()], + |r| r.get::<_, i64>(0), + ) + .optional()? + .unwrap_or(0) as u64; + } + let payload: Option = db + .query_row( + "SELECT payload FROM batches WHERE revision=?1 AND ordinal=?2", + params![id.as_str(), *ordinal as i64], + |r| r.get(0), + ) + .optional()?; + let Some(payload) = payload else { + self.pending = None; + return Ok(None); + }; + *ordinal += 1; + let ops = serde_json::from_str::(&payload)?.into_ops(); + Ok(Some(self.overlay(ops, ctx.config.as_ref()))) + } + fn overlay(&mut self, mut ops: Vec, config: Option<&SessionConfig>) -> Vec { + let Some(config) = config else { + return ops; + }; + let (parent, external_root) = config.attached_span_ids(); + let effective_root = external_root.or_else(|| parent.clone()); + for op in &mut ops { + let is_insert = matches!(op, SpanOp::Insert(_)); + let row = row_mut(op); + let is_top_root = + row.span_id == row.root_span_id && row.parent_span_ids.is_empty() && is_insert; + let is_root = row.span_id == row.root_span_id && row.parent_span_ids.is_empty(); + if let Some(root) = &effective_root { + row.root_span_id = root.clone(); + } + if is_root { + if let Some(parent) = &parent { + row.parent_span_ids = vec![parent.clone()]; + } + } + if is_top_root { + if let Some(additional) = config + .additional_metadata + .as_ref() + .and_then(Value::as_object) + { + let metadata = row + .metadata + .get_or_insert_with(|| Value::Object(Map::new())); + if let Some(metadata) = metadata.as_object_mut() { + for (key, value) in additional { + if !key.starts_with("_bt_") { + metadata.entry(key.clone()).or_insert_with(|| value.clone()); + } + } + } + } + if let Some(project) = config.project_name() { + if let Some(metadata) = row.metadata.as_mut().and_then(Value::as_object_mut) { + if metadata.get("source").and_then(Value::as_str) == Some("codex") { + metadata + .entry("project") + .or_insert_with(|| Value::String(project.into())); + } + } + } + if !config.tags.is_empty() { + let tags = row.tags.get_or_insert_with(Vec::new); + for tag in &config.tags { + if !tags.contains(tag) { + tags.push(tag.clone()); + } + } + } + } + } + ops + } +} + +impl AgentTranslator for DerivedTranslator { + fn ledger_sequence(&self) -> Option { + Some(self.sequence) + } + fn set_revision_path(&mut self, path: &Path) { + let id = path.file_name().unwrap().to_string_lossy().into_owned(); + self.sequence = 0; + self.pending = Some((id, 0)); + } + + fn handle(&mut self, _event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { + self.read_next(ctx).map(|batch| batch.unwrap_or_default()) + } + + fn drain_pending(&mut self, ctx: &SessionCtx) -> anyhow::Result>> { + self.read_next(ctx) + } + + fn checkpoint(&mut self, ctx: &SessionCtx) -> anyhow::Result> { + self.read_next(ctx).map(|batch| batch.unwrap_or_default()) + } + + fn finalize(&mut self, ctx: &SessionCtx) -> anyhow::Result> { + self.read_next(ctx).map(|batch| batch.unwrap_or_default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wire::SessionRoute; + fn event(name: &str) -> Envelope { + Envelope { + source: "debug".into(), + source_version: None, + plugin_version: None, + session_id: "ledger".into(), + event: name.into(), + ts_ms: 1, + payload: serde_json::json!({"original":"payload"}), + route: Some(SessionRoute::default()), + config: None, + managed_run_id: None, + capture: None, + } + } + fn source(data: &Path) -> Arc { + SourceTranslation::new( + "debug", + "ledger", + "ledger", + data, + &Registry::default_agents(), + ) + .unwrap() + } + fn read_all(translator: &mut dyn AgentTranslator, path: &Path) -> (Vec, u64) { + translator.set_revision_path(path); + let ctx = SessionCtx { + session_id: "ledger".into(), + config: None, + }; + let mut result = translator.handle(&event("unused"), &ctx).unwrap(); + while let Some(batch) = translator.drain_pending(&ctx).unwrap() { + result.extend(batch); + } + (result, translator.ledger_sequence().unwrap()) + } + + #[tokio::test] + async fn translation_commit_is_atomic_and_failed_input_survives() { + let temp = tempfile::tempdir().unwrap(); + let producer = source(temp.path()); + let mut writer = journal::JournalWriter::open_path(&producer.journal_path) + .await + .unwrap(); + let through = writer.append(&event("first")).await.unwrap(); + let db = open_database(&producer.dir.join("spans.sqlite")).unwrap(); + db.execute_batch("CREATE TRIGGER fail_commit BEFORE INSERT ON continuation BEGIN SELECT RAISE(ABORT,'injected commit failure'); END;").unwrap(); + assert!(producer.ensure_event(through).await.is_err()); + assert_eq!( + db.query_row("SELECT COUNT(*) FROM revisions", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 0 + ); + assert_eq!( + db.query_row("SELECT COUNT(*) FROM batches", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 0 + ); + assert_eq!( + db.query_row("SELECT COUNT(*) FROM continuation", [], |r| r + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + assert!(std::fs::read_to_string(&producer.journal_path) + .unwrap() + .contains("original")); + db.execute_batch("DROP TRIGGER fail_commit").unwrap(); + drop(db); + drop(producer); + let restarted = source(temp.path()); + let path = restarted.ensure_event(through).await.unwrap(); + assert_eq!( + read_all(restarted.route_translator().as_mut(), &path) + .0 + .len(), + 2 + ); + } + + #[tokio::test] + async fn temporary_database_lock_retries_without_losing_translator_state() { + let temp = tempfile::tempdir().unwrap(); + let producer = source(temp.path()); + let mut writer = journal::JournalWriter::open_path(&producer.journal_path) + .await + .unwrap(); + let through = writer.append(&event("first")).await.unwrap(); + let db = open_database(&producer.dir.join("spans.sqlite")).unwrap(); + db.execute_batch("BEGIN IMMEDIATE").unwrap(); + let error = producer.ensure_event(through).await.unwrap_err(); + assert!(retryable_storage(&error), "{error:#}"); + db.execute_batch("ROLLBACK").unwrap(); + let path = producer.ensure_event(through).await.unwrap(); + let (ops, _) = read_all(producer.route_translator().as_mut(), &path); + assert_eq!(ops.len(), 2); + assert_eq!(row(&ops[1]).metadata.as_ref().unwrap()["seq"], 0); + } + + #[tokio::test] + async fn collected_input_is_not_needed_for_delivery_or_translator_continuation() { + let temp = tempfile::tempdir().unwrap(); + let producer = source(temp.path()); + let writer = tokio::sync::Mutex::new( + journal::JournalWriter::open_path(&producer.journal_path) + .await + .unwrap(), + ); + let route = SessionRoute::default(); + producer.register(&route, 0).unwrap(); + let first = writer.lock().await.append(&event("first")).await.unwrap(); + let path = producer.ensure_event(first).await.unwrap(); + producer.collect_input(&writer).await.unwrap(); + assert!(!std::fs::read_to_string(&producer.journal_path) + .unwrap() + .contains("original")); + drop(producer); + let restarted = source(temp.path()); + let mut consumer = restarted.route_translator(); + let (ops, sequence) = read_all(consumer.as_mut(), &path); + assert_eq!(ops.len(), 2, "pending spans survive collection and restart"); + restarted.acknowledge(&route, sequence).await.unwrap(); + assert!(restarted.revisions(&route, first).await.unwrap().is_empty()); + let second = writer.lock().await.append(&event("second")).await.unwrap(); + let second_path = restarted.ensure_event(second).await.unwrap(); + let (ops, _) = read_all(consumer.as_mut(), &second_path); + assert_eq!( + ops.len(), + 1, + "continuation must not recreate the session root" + ); + assert_eq!(row(&ops[0]).metadata.as_ref().unwrap()["seq"], 1); + } + + #[tokio::test] + async fn slowest_consumer_protects_event_and_control_operations() { + let temp = tempfile::tempdir().unwrap(); + let producer = source(temp.path()); + let mut writer = journal::JournalWriter::open_path(&producer.journal_path) + .await + .unwrap(); + let route = SessionRoute::default(); + let mut blocked = route.clone(); + blocked.tags.push("blocked".into()); + producer.register(&route, 0).unwrap(); + producer.register(&blocked, 0).unwrap(); + let through = writer.append(&event("first")).await.unwrap(); + let path = producer.ensure_event(through).await.unwrap(); + let (_, first_seq) = read_all(producer.route_translator().as_mut(), &path); + let control = producer.control_path(ControlKind::Checkpoint, through); + let merge = SpanOp::Merge(SpanRow { + span_id: "root".into(), + root_span_id: "root".into(), + end_ms: Some(2), + ..Default::default() + }); + let scratch = producer + .stage_batches(vec![merge], || Ok(None)) + .await + .unwrap(); + let snapshot = producer.state.lock().await.translator.snapshot().unwrap(); + producer + .commit( + &control, + (through, through), + None, + "control", + snapshot, + scratch, + ) + .await + .unwrap(); + let (_, control_seq) = read_all(producer.route_translator().as_mut(), &control); + assert!(control_seq > first_seq); + producer.acknowledge(&route, control_seq).await.unwrap(); + assert_eq!( + producer.revisions(&blocked, through).await.unwrap().len(), + 2 + ); + producer.acknowledge(&blocked, first_seq).await.unwrap(); + assert_eq!( + producer.revisions(&blocked, through).await.unwrap().len(), + 1, + "control remains despite the same input offset" + ); + producer.acknowledge(&blocked, control_seq).await.unwrap(); + let db = open_database(&producer.dir.join("spans.sqlite")).unwrap(); + assert_eq!( + db.query_row("SELECT COUNT(*) FROM batches", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 0 + ); + assert_eq!( + db.query_row("SELECT COUNT(*) FROM revisions", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 0 + ); + } +} diff --git a/bt-daemon/src/dispatch.rs b/bt-daemon/src/dispatch.rs index 885a6191..715cbdea 100644 --- a/bt-daemon/src/dispatch.rs +++ b/bt-daemon/src/dispatch.rs @@ -25,6 +25,21 @@ use tokio::sync::{mpsc, oneshot}; /// never data. const QUEUE_CAPACITY: usize = 1024; +/// An actor exists from routing information before credentials are resolved. +/// Its initial translation replay cannot submit output yet. +struct UnconfiguredSink; + +#[async_trait::async_trait] +impl crate::sink::Sink for UnconfiguredSink { + async fn emit(&mut self, _ops: &[crate::translate::SpanOp]) -> anyhow::Result { + anyhow::bail!("delivery attempted before route authentication") + } + + async fn flush(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} + #[derive(Default)] pub struct Counters { pub queued: AtomicU64, @@ -34,6 +49,7 @@ pub struct Counters { enum SessionMsg { Event(Box, u64), Configure(Box, oneshot::Sender<()>), + AuthFailure(String, u64), Barrier(oneshot::Sender<()>), Flush(oneshot::Sender), Finalize(oneshot::Sender), @@ -42,7 +58,6 @@ enum SessionMsg { /// Where to rebuild a session's translator state from, streamed at startup. pub struct ReplayPlan { - pub journal_path: PathBuf, /// Replay stops here — the journal's length when this session was /// created, so the event creating it is not replayed and then delivered /// a second time from the queue. @@ -55,7 +70,6 @@ pub struct ReplayPlan { pub(crate) struct SessionOptions { pub session_id: String, - pub translator_session_id: String, pub source: String, pub plugin_version: Option, pub replay: Option, @@ -66,6 +80,8 @@ pub(crate) struct SessionOptions { pub data_dir: PathBuf, pub journal: Arc>, pub correlation_changed: Arc, + pub auth_provider: Option>, + pub derived: Arc, } /// Handle to one live session: its queue plus observable counters/state. @@ -75,16 +91,13 @@ pub struct Session { pub counters: Arc, pub last_error: Arc>>, pub permalink: Arc>>, - pause: Arc>>, + pause: Arc>>, last_activity: Mutex, } -#[derive(Clone)] -struct PluginPause { - path: PathBuf, - digest: Option, - span_id: String, - journal_start: u64, +#[derive(Clone, Default)] +struct PausedWork { + marker_span_id: Option, marker_emitted: bool, marker_cleared: bool, } @@ -93,12 +106,11 @@ impl Session { /// Spawn a session's actor task and return its handle. pub fn spawn( options: SessionOptions, - translators: Arc, + _translators: Arc, sink_factory: Arc, ) -> Arc { let SessionOptions { session_id, - translator_session_id, source, plugin_version, replay, @@ -109,6 +121,8 @@ impl Session { data_dir, journal, correlation_changed, + auth_provider, + derived, } = options; let (tx, rx) = mpsc::channel(QUEUE_CAPACITY); let counters = Arc::new(Counters::default()); @@ -118,10 +132,8 @@ impl Session { let actor = SessionActor { session_id: session_id.clone(), - translator_session_id, source: source.clone(), plugin_version, - translators, sink_factory, counters: counters.clone(), last_error: last_error.clone(), @@ -135,6 +147,9 @@ impl Session { data_dir, journal, correlation_changed, + auth_provider, + derived, + delivered_sequence: AtomicU64::new(0), }; tokio::spawn(actor.run(rx)); @@ -180,7 +195,7 @@ impl Session { self.last_activity.lock().unwrap().elapsed() } - pub fn has_paused_plugin(&self) -> bool { + pub fn has_paused_work(&self) -> bool { self.pause.lock().unwrap().is_some() } @@ -234,6 +249,13 @@ impl Session { .map_err(|_| anyhow::anyhow!("session actor dropped configuration reply")) } + pub async fn auth_failure(&self, error: String, first_unprocessed: u64) -> anyhow::Result<()> { + self.tx + .send(SessionMsg::AuthFailure(error, first_unprocessed)) + .await + .map_err(|_| anyhow::anyhow!("session actor is gone")) + } + /// Drain, flush, and stop the actor (used on daemon shutdown and when an /// idle session is retired). pub async fn shutdown(&self) { @@ -470,15 +492,13 @@ async fn capture_grok_transcript_pass( struct SessionActor { session_id: String, - translator_session_id: String, source: String, plugin_version: Option, - translators: Arc, sink_factory: Arc, counters: Arc, last_error: Arc>>, permalink: Arc>>, - pause: Arc>>, + pause: Arc>>, replay: Option, config: crate::wire::SessionConfig, correlation_key: String, @@ -487,6 +507,9 @@ struct SessionActor { data_dir: PathBuf, journal: Arc>, correlation_changed: Arc, + auth_provider: Option>, + derived: Arc, + delivered_sequence: AtomicU64, } #[derive(Clone, Copy)] @@ -495,7 +518,6 @@ enum BatchMode { Replay, Checkpoint, TerminalFinalize, - ShutdownFinalize, } impl BatchMode { @@ -507,7 +529,7 @@ impl BatchMode { "translate checkpoint failed", "sink emit (checkpoint) failed", ), - Self::TerminalFinalize | Self::ShutdownFinalize => ( + Self::TerminalFinalize => ( "translate finalization failed", "sink emit (finalization) failed", ), @@ -515,90 +537,52 @@ impl BatchMode { } fn observes_correlation(self) -> bool { - !matches!(self, Self::ShutdownFinalize) + true } } impl SessionActor { async fn run(self, mut rx: mpsc::Receiver) { - let mut translator = match self - .translators - .create_checked_with_session_namespace(&self.source, &self.translator_session_id) - { - Ok(translator) => translator, - Err(error) => { - self.set_error(format!("translator init failed: {error}")); - return; - } - }; - let sink = match self.sink_factory.create( - &self.session_id, - &self.source, - self.plugin_version.as_deref(), - ) { - Ok(s) => s, - Err(e) => { - self.set_error(format!("sink init failed: {e}")); - // Still drain the queue so the daemon's counters settle and - // callers waiting on flush don't hang. - while let Some(msg) = rx.recv().await { - match msg { - SessionMsg::Event(_, _) => { - self.counters.queued.fetch_sub(1, Ordering::Relaxed); - } - SessionMsg::Configure(_, r) => { - let _ = r.send(()); - } - SessionMsg::Barrier(r) => { - let _ = r.send(()); - } - SessionMsg::Flush(r) => { - let _ = r.send(0); - } - SessionMsg::Finalize(r) => { - let _ = r.send(0); - } - SessionMsg::Shutdown(r) => { - let _ = r.send(()); - break; - } - } - } - return; - } - }; + let mut translator = self.derived.route_translator(); + let sink: Box = Box::new(UnconfiguredSink); + let mut auth_ready = false; let mut sink: Box = Box::new( - LedgerSink::new( - sink, - &self.data_dir, - &self.source, - &self.session_id, - Some(&self.config), - ) - .await, + LedgerSink::new(sink, &self.data_dir, &self.source, &self.session_id, None).await, ); let mut ctx = SessionCtx { session_id: self.session_id.clone(), config: Some(self.config.clone()), }; - sink.configure(&self.config); - self.refresh_permalink(sink.as_ref()); - if let Ok(Some(active)) = crate::plugin_diagnostics::active_pipeline( - &self.data_dir, - &self.source, - &self.session_id, - &self.route, - ) { - if let Some(span_id) = active.span_id { - *self.pause.lock().unwrap() = Some(PluginPause { - path: active.plugin_path, - digest: active.plugin_digest, - span_id, - journal_start: active.journal_start.unwrap_or(0), - marker_emitted: true, - marker_cleared: false, - }); - } + let recovery_scope = + crate::recovery::WorkScope::delivery(&self.source, &self.session_id, &self.route).ok(); + if let Some(incident) = recovery_scope.as_ref().and_then(|scope| { + crate::recovery::active(&self.data_dir, scope) + .ok() + .flatten() + }) { + *self.pause.lock().unwrap() = Some(PausedWork { + marker_span_id: incident.marker_span_id.clone(), + marker_emitted: incident.marker_span_id.is_some(), + marker_cleared: false, + }); + } + let source_scope = crate::recovery::WorkScope::SourceSession { + source: self.source.clone(), + session_id: self.session_id.clone(), + }; + if let Ok(Some(incident)) = crate::recovery::active(&self.data_dir, &source_scope) { + let _ = incident; + *self.pause.lock().unwrap() = Some(PausedWork::default()); + } + if self.pause.lock().unwrap().is_none() + && recovery_scope.as_ref().is_some_and(|scope| { + crate::recovery::active(&self.data_dir, scope) + .ok() + .flatten() + .is_some() + }) + { + *self.pause.lock().unwrap() = Some(PausedWork::default()); } // Rebuild translator state before accepting the first new event. // Stable span ids make this both crash recovery and a complete copy @@ -612,28 +596,29 @@ impl SessionActor { // Keep the journal conservative for this actor generation rather than // checkpointing past an event that may need recovery delivery. let mut checkpointable = self - .replay_into(&mut translator, &mut sink, &ctx, self.replay.as_ref()) + .prepare_and_replay( + &mut translator, + &mut sink, + &ctx, + self.replay.as_ref(), + auth_ready, + ) .await; - if checkpointable && self.pause.lock().unwrap().is_some() { + if auth_ready && checkpointable && self.pause.lock().unwrap().is_some() { let tip = self.replay.as_ref().map_or(0, |plan| plan.through); let marker_cleared = self .pause .lock() .unwrap() .as_ref() - .is_some_and(|pause| pause.marker_cleared); - if marker_cleared && self.flush_sink(&mut sink).await { + .is_some_and(|state| !state.marker_emitted || state.marker_cleared); + if marker_cleared && self.flush_sink(&mut sink, acknowledged_through).await { self.record_delivery_checkpoint(&mut sink, tip, &mut acknowledged_through) .await; if acknowledged_through >= tip { - let _ = crate::plugin_diagnostics::update_pipeline( - &self.data_dir, - &self.source, - &self.session_id, - &self.route, - crate::plugin_diagnostics::PluginRecoveryState::Recovered, - tip, - ); + if let Some(scope) = &recovery_scope { + let _ = crate::recovery::resolve(&self.data_dir, scope); + } *self.pause.lock().unwrap() = None; *self.last_error.lock().unwrap() = None; } @@ -648,24 +633,84 @@ impl SessionActor { let msg = tokio::select! { msg = rx.recv() => match msg { Some(msg) => msg, None => break }, _ = retry_tick.tick(), if self.pause.lock().unwrap().is_some() => { - let paused = self.pause.lock().unwrap().clone().expect("paused"); - let changed = crate::plugin_diagnostics::plugin_digest(&paused.path); - if changed.is_none() || changed == paused.digest { - continue; + let route_incident = recovery_scope.as_ref().and_then(|scope| crate::recovery::active(&self.data_dir, scope).ok().flatten()); + let source_incident = crate::recovery::active(&self.data_dir, &source_scope).ok().flatten(); + let now = crate::recovery::now_ms(); + let eligible = |incident: &crate::recovery::Incident| match &incident.cause { + crate::recovery::FailureCause::Credentials { retry_after_ms, .. } => now >= *retry_after_ms, + cause => cause.check_local(crate::recovery::translator_revision(), now) + == crate::recovery::CheckResult::RetryCandidate, + }; + let retry_source = source_incident.as_ref().is_some_and(eligible); + let (mut scope, mut incident) = if retry_source { + (Some(&source_scope), source_incident.clone()) + } else if route_incident.as_ref().is_some_and(eligible) { + (recovery_scope.as_ref(), route_incident.clone()) + } else if source_incident.is_none() && route_incident.is_none() { + // Another route may have recovered the shared source. + // Drain its newly committed output even without a native event. + (recovery_scope.as_ref(), None) + } else { continue; }; + if retry_source { + let tip=self.journal.lock().await.position(); + if let Err(error)=self.derived.ensure_event(tip).await { + self.record_translation_failure(&error,Some((0,tip))); + self.set_error(format!("span production retry failed: {error}"));continue; + } + if let Err(error)=self.derived.collect_input(&self.journal).await {self.set_error(format!("event WAL collection failed: {error}"));} + let _=crate::recovery::resolve(&self.data_dir,&source_scope); + if let Some(route_incident)=&route_incident { + if !matches!(route_incident.cause,crate::recovery::FailureCause::Credentials{..}) && + route_incident.cause.check_local(crate::recovery::translator_revision(),now)!=crate::recovery::CheckResult::RetryCandidate { + continue; + } + } + scope=recovery_scope.as_ref(); + incident=route_incident.clone(); } - if let Err(error) = crate::plugin_diagnostics::update_pipeline( - &self.data_dir, &self.source, &self.session_id, &self.route, - crate::plugin_diagnostics::PluginRecoveryState::Reprocessing, - last_event_through, - ) { - self.set_error(format!("failed to record plugin recovery: {error}")); + if !auth_ready || incident.as_ref().is_some_and(|incident| matches!(incident.cause, crate::recovery::FailureCause::Credentials { .. })) { + let Some(provider) = &self.auth_provider else { + self.set_error("credential retry unavailable without an auth provider".into()); + continue; + }; + let lease = crate::server::resolve_route_auth( + provider.as_ref(), + &self.route.auth, + crate::server::AuthResolveReason::Initial, + self.route.auth.org_name.as_deref(), + ).await; + match lease { + Ok(lease) => { + ctx.config = Some(self.route.with_auth(lease.auth)); + auth_ready = true; + } + Err(error) => { + if let Some(scope) = scope { + let _ = crate::recovery::pause( + &self.data_dir, scope.clone(), + incident.as_ref().map_or(0, |incident| incident.first_unprocessed), + incident.as_ref().map_or(0, |incident| incident.operation_index), + crate::recovery::FailureCause::Credentials { + selection: self.route.auth.clone(), + retry_after_ms: now + 2_000, + }, + error.to_string(), + incident.as_ref().and_then(|incident| incident.marker_span_id.clone()), + ); + } + self.set_error(format!("credential retry failed: {error}")); + continue; + } + } } - let Ok(next_translator) = self.translators.create_checked_with_session_namespace( - &self.source, &self.translator_session_id, - ) else { - self.set_error("plugin recovery could not create translator".into()); - continue; - }; + let changed = incident.as_ref().and_then(|incident| match &incident.cause { crate::recovery::FailureCause::PluginFile { path, .. } => Some(crate::recovery::plugin_digest(path)), _ => None }); + if let Some(scope) = scope { + if let Err(error) = crate::recovery::begin_retry(&self.data_dir, scope) { + self.set_error(format!("failed to record recovery attempt: {error}")); + continue; + } + } + let next_translator = self.derived.route_translator(); let Ok(next_sink) = self.sink_factory.create( &self.session_id, &self.source, self.plugin_version.as_deref(), ) else { @@ -679,30 +724,30 @@ impl SessionActor { if let Some(config) = &ctx.config { next_sink.configure(config); } let tip = self.journal.lock().await.position(); let plan = ReplayPlan { - journal_path: crate::journal::source_journal_path( - &self.data_dir, &self.source, &self.session_id, - ), through: tip, acknowledged_through, }; let mut next_translator = next_translator; - let replay_ok = self.replay_into( - &mut next_translator, &mut next_sink, &ctx, Some(&plan), + let replay_ok = self.prepare_and_replay( + &mut next_translator, &mut next_sink, &ctx, Some(&plan), true, ).await; let flushed = replay_ok && self.checkpoint_and_flush( &mut next_translator, &mut next_sink, &ctx, Some((acknowledged_through, tip)), ).await; - let stable = crate::plugin_diagnostics::plugin_digest(&paused.path) == changed; + let stable = incident.as_ref().is_none_or(|incident| match &incident.cause { + crate::recovery::FailureCause::PluginFile { path, .. } => crate::recovery::plugin_digest(path) == changed.flatten(), + crate::recovery::FailureCause::InputShape { translator_revision, .. } | crate::recovery::FailureCause::TranslatorFault { translator_revision } => translator_revision != crate::recovery::translator_revision(), + _ => true, + }); let marker_cleared = self.pause.lock().unwrap().as_ref() .is_some_and(|current| !current.marker_emitted || current.marker_cleared); if flushed && stable && marker_cleared { self.record_delivery_checkpoint(&mut next_sink, tip, &mut acknowledged_through).await; if acknowledged_through >= tip { - let _ = crate::plugin_diagnostics::update_pipeline( - &self.data_dir, &self.source, &self.session_id, &self.route, - crate::plugin_diagnostics::PluginRecoveryState::Recovered, tip, - ); + if let Some(scope) = scope { + let _ = crate::recovery::resolve(&self.data_dir, scope); + } *self.pause.lock().unwrap() = None; *self.last_error.lock().unwrap() = None; translator = next_translator; @@ -716,50 +761,75 @@ impl SessionActor { } // A new plugin failure records its new digest. A sink // failure retains the old one so a later tick can retry. - let _ = crate::plugin_diagnostics::update_pipeline( - &self.data_dir, &self.source, &self.session_id, &self.route, - crate::plugin_diagnostics::PluginRecoveryState::Paused, tip, - ); continue; } }; match msg { SessionMsg::Event(env, journal_through) => { - if journal_through <= replayed_through || self.pause.lock().unwrap().is_some() { + let paused = self.pause.lock().unwrap().clone(); + let source_failure = crate::recovery::active(&self.data_dir, &source_scope) + .ok() + .flatten(); + let translation_paused = source_failure.is_some_and(|incident| { + matches!( + incident.cause, + crate::recovery::FailureCause::InputShape { .. } + | crate::recovery::FailureCause::TranslatorFault { .. } + | crate::recovery::FailureCause::LocalStorage { .. } + ) + }); + let skip_for_pause = translation_paused; + if journal_through <= replayed_through || skip_for_pause { last_event_through = last_event_through.max(journal_through); self.counters.queued.fetch_sub(1, Ordering::Relaxed); - if self.pause.lock().unwrap().is_some() { - let _ = crate::plugin_diagnostics::update_pipeline( - &self.data_dir, - &self.source, - &self.session_id, - &self.route, - crate::plugin_diagnostics::PluginRecoveryState::Paused, - last_event_through, - ); - } continue; } let event_start = last_event_through; let event_window = Some((event_start, journal_through)); last_event_through = journal_through; let correlation_barrier = is_tool_lifecycle_event(&env.event); - if let Some(cfg) = &env.config { + if let Some(cfg) = env.config.as_ref().filter(|_| auth_ready) { sink.configure(cfg); ctx.config = Some(cfg.clone()); self.refresh_permalink(sink.as_ref()); } - let translated = translator.handle(&env, &ctx); - let (correlation_changed, delivered) = self - .emit_translator_batches( - &mut translator, - &mut sink, - &ctx, - translated, - BatchMode::Live, - event_window, - ) - .await; + let produced = self.derived.ensure_event(journal_through).await; + let (correlation_changed, delivered) = match produced { + Ok(_) => { + if let Err(error) = self.derived.collect_input(&self.journal).await { + self.set_error(format!("event WAL collection failed: {error}")); + } + if auth_ready && paused.is_some() { + self.counters.queued.fetch_sub(1, Ordering::Relaxed); + continue; + } + self.consume_into( + &mut translator, + &mut sink, + &ctx, + auth_ready, + BatchMode::Live, + ) + .await + } + Err(error) => { + self.record_translation_failure(&error, event_window); + self.set_error(format!("span production failed: {error}")); + if let Err(error) = self.derived.collect_input(&self.journal).await { + self.set_error(format!("event WAL collection failed: {error}")); + } + let (changed, _) = self + .consume_into( + &mut translator, + &mut sink, + &ctx, + auth_ready, + BatchMode::Live, + ) + .await; + (changed, false) + } + }; if correlation_changed || correlation_barrier { if let Err(error) = crate::server::persist_active_parent_snapshot( &self.data_dir, @@ -785,22 +855,155 @@ impl SessionActor { ) .await; } - if delivered && checkpointable { + if delivered && checkpointable && auth_ready { pending_through = pending_through.max(journal_through); - } else { + } else if !delivered { checkpointable = false; } } SessionMsg::Configure(config, reply) => { - sink.configure(&config); - ctx.config = Some(*config); - self.refresh_permalink(sink.as_ref()); + let config = *config; + let became_ready = !auth_ready; + auth_ready = true; + ctx.config = Some(config.clone()); + let route_retry_allowed = recovery_scope + .as_ref() + .and_then(|scope| { + crate::recovery::active(&self.data_dir, scope) + .ok() + .flatten() + }) + .is_none_or(|incident| { + matches!( + incident.cause, + crate::recovery::FailureCause::Credentials { .. } + ) || incident.cause.check_local( + crate::recovery::translator_revision(), + crate::recovery::now_ms(), + ) == crate::recovery::CheckResult::RetryCandidate + }); + if became_ready && route_retry_allowed { + if let Ok(next_sink) = self.sink_factory.create( + &self.session_id, + &self.source, + self.plugin_version.as_deref(), + ) { + let mut next_translator = self.derived.route_translator(); + let mut next_sink: Box = Box::new( + LedgerSink::new( + next_sink, + &self.data_dir, + &self.source, + &self.session_id, + Some(&config), + ) + .await, + ); + next_sink.configure(&config); + let tip = last_event_through; + let plan = ReplayPlan { + through: tip, + acknowledged_through, + }; + let replay_ok = self + .prepare_and_replay( + &mut next_translator, + &mut next_sink, + &ctx, + Some(&plan), + true, + ) + .await; + if replay_ok { + if let Some(scope) = &recovery_scope { + if crate::recovery::active(&self.data_dir, scope) + .ok() + .flatten() + .is_some() + { + self.record_delivery_checkpoint( + &mut next_sink, + tip, + &mut acknowledged_through, + ) + .await; + if acknowledged_through >= tip { + let _ = crate::recovery::resolve(&self.data_dir, scope); + *self.pause.lock().unwrap() = None; + *self.last_error.lock().unwrap() = None; + } + } + } + translator = next_translator; + sink = next_sink; + checkpointable = true; + pending_through = tip; + replayed_through = tip; + } + } + } else if auth_ready { + sink.configure(&config); + self.refresh_permalink(sink.as_ref()); + } let _ = reply.send(()); } + SessionMsg::AuthFailure(error, _capture_cursor) => { + let first_unprocessed = self + .derived + .consumer_sequence(&self.route) + .await + .unwrap_or(0); + auth_ready = false; + if let Ok(scope) = crate::recovery::WorkScope::delivery( + &self.source, + &self.session_id, + &self.route, + ) { + let prior = crate::recovery::active(&self.data_dir, &scope) + .ok() + .flatten(); + if !prior.as_ref().is_some_and(|incident| { + matches!( + incident.cause, + crate::recovery::FailureCause::PluginFile { .. } + ) + }) { + let _ = crate::recovery::pause( + &self.data_dir, + scope, + first_unprocessed, + 0, + crate::recovery::FailureCause::Credentials { + selection: self.route.auth.clone(), + retry_after_ms: crate::recovery::now_ms() + 1_000, + }, + error.clone(), + prior.and_then(|incident| incident.marker_span_id), + ); + *self.pause.lock().unwrap() = Some(PausedWork::default()); + } + } + self.set_error(format!("session auth failed: {error}")); + } SessionMsg::Barrier(reply) => { let _ = reply.send(()); } SessionMsg::Flush(reply) => { + if self.source_paused() && !self.delivery_paused() { + let (_, delivered) = self + .consume_into(&mut translator, &mut sink, &ctx, true, BatchMode::Replay) + .await; + if delivered { + self.record_delivery_checkpoint( + &mut sink, + pending_through, + &mut acknowledged_through, + ) + .await; + } + let _ = reply.send(self.counters.queued.load(Ordering::Relaxed).max(1)); + continue; + } if self.pause.lock().unwrap().is_some() { let _ = reply.send(self.counters.queued.load(Ordering::Relaxed).max(1)); continue; @@ -821,7 +1024,12 @@ impl SessionActor { ) .await; } - let _ = reply.send(self.counters.queued.load(Ordering::Relaxed)); + let pending = self.counters.queued.load(Ordering::Relaxed); + let _ = reply.send(if self.pause.lock().unwrap().is_some() { + pending.max(1) + } else { + pending + }); } SessionMsg::Finalize(reply) => { if self.pause.lock().unwrap().is_some() { @@ -845,7 +1053,12 @@ impl SessionActor { ) .await; } - let _ = reply.send(self.counters.queued.load(Ordering::Relaxed)); + let pending = self.counters.queued.load(Ordering::Relaxed); + let _ = reply.send(if self.pause.lock().unwrap().is_some() { + pending.max(1) + } else { + pending + }); } SessionMsg::Shutdown(reply) => { if self.pause.lock().unwrap().is_some() { @@ -853,11 +1066,10 @@ impl SessionActor { break; } checkpointable &= self - .finalize_and_flush( + .checkpoint_and_flush( &mut translator, &mut sink, &ctx, - BatchMode::ShutdownFinalize, Some((acknowledged_through, last_event_through)), ) .await; @@ -892,12 +1104,32 @@ impl SessionActor { let mut next = match first { Ok(ops) => Some(ops), Err(e) => { + self.record_translation_failure(&e, event_window); self.set_error(format!("{translate_error}: {e}")); return (false, false); } }; + let event_window = translator + .ledger_sequence() + .filter(|seq| *seq > 0) + .map(|seq| (seq - 1, seq)) + .or(event_window); let mut correlation_changed = false; let mut delivered = true; + let mut operation_index = 0u32; + let resume_at = event_window + .and_then(|(start, _)| { + crate::recovery::WorkScope::delivery(&self.source, &self.session_id, &self.route) + .ok() + .and_then(|scope| { + crate::recovery::active(&self.data_dir, &scope) + .ok() + .flatten() + }) + .filter(|incident| incident.first_unprocessed == start) + .map(|incident| incident.operation_index) + }) + .unwrap_or(0); while let Some(ops) = next { if !ops.is_empty() { if mode.observes_correlation() { @@ -919,6 +1151,11 @@ impl SessionActor { .unwrap_or_default(); let mut processed = Vec::with_capacity(ops.len()); for op in &ops { + let current_index = operation_index; + operation_index = operation_index.saturating_add(1); + if current_index < resume_at { + continue; + } match crate::span_processor::process( plugin_paths, op, @@ -929,12 +1166,27 @@ impl SessionActor { Ok(result) => { if let Some(failure) = result.failure { delivered = false; + let prefix_accepted = processed.is_empty() + || match sink.emit(&processed).await { + Ok(_) => match sink.flush().await { + Ok(()) => true, + Err(error) => { + self.set_error(format!("{emit_error}: {error}")); + false + } + }, + Err(error) => { + self.set_error(format!("{emit_error}: {error}")); + false + } + }; self.record_plugin_failure( sink, op, &failure.path, &failure.message, event_window, + if prefix_accepted { current_index } else { 0 }, ) .await; break; @@ -946,12 +1198,27 @@ impl SessionActor { Err(error) => { delivered = false; if let Some(plugin) = plugin_paths.first() { + let prefix_accepted = processed.is_empty() + || match sink.emit(&processed).await { + Ok(_) => match sink.flush().await { + Ok(()) => true, + Err(error) => { + self.set_error(format!("{emit_error}: {error}")); + false + } + }, + Err(error) => { + self.set_error(format!("{emit_error}: {error}")); + false + } + }; self.record_plugin_failure( sink, op, plugin, &error.to_string(), event_window, + if prefix_accepted { current_index } else { 0 }, ) .await; } @@ -964,11 +1231,8 @@ impl SessionActor { } if delivered && !processed.is_empty() { let paused = self.pause.lock().unwrap().clone(); - let recovery_span = paused.and_then(|pause| { - event_window - .filter(|(start, _)| *start >= pause.journal_start) - .map(|_| pause.span_id) - }); + let recovery_span = paused + .and_then(|pause| pause.marker_span_id.filter(|_| !pause.marker_cleared)); let result = if let Some(recovery_span) = recovery_span { let mut emitted = 0; let mut result = Ok(()); @@ -987,8 +1251,8 @@ impl SessionActor { Ok(n) => { emitted += n; if replacing { - if let Some(paused) = self.pause.lock().unwrap().as_mut() { - paused.marker_cleared = true; + if let Some(state) = self.pause.lock().unwrap().as_mut() { + state.marker_cleared = true; } } } @@ -1008,6 +1272,11 @@ impl SessionActor { } Err(e) => { delivered = false; + self.record_delivery_failure( + &e, + event_window, + operation_index.saturating_sub(processed.len() as u32), + ); self.set_error(format!("{emit_error}: {e}")); } } @@ -1019,12 +1288,19 @@ impl SessionActor { next = match translator.drain_pending(ctx) { Ok(next) => next, Err(e) => { + self.record_translation_failure(&e, event_window); self.set_error(format!("{translate_error}: {e}")); delivered = false; None } }; } + if delivered { + if let Some(sequence) = translator.ledger_sequence() { + self.delivered_sequence + .fetch_max(sequence, Ordering::Relaxed); + } + } (correlation_changed, delivered) } @@ -1035,12 +1311,13 @@ impl SessionActor { plugin: &std::path::Path, message: &str, event_window: Option<(u64, u64)>, + operation_index: u32, ) { let (row, operation) = match op { crate::translate::SpanOp::Insert(row) => (row, "insert"), crate::translate::SpanOp::Merge(row) => (row, "merge"), }; - let (start, through) = event_window.unwrap_or((0, self.journal.lock().await.position())); + let (start, through) = event_window.unwrap_or((0, self.derived.tip().await.unwrap_or(0))); let marker = crate::translate::SpanOp::Merge(crate::translate::SpanRow { span_id: row.span_id.clone(), root_span_id: row.root_span_id.clone(), @@ -1064,14 +1341,37 @@ impl SessionActor { false } }; - *self.pause.lock().unwrap() = Some(PluginPause { - path: plugin.to_path_buf(), - digest: crate::plugin_diagnostics::plugin_digest(plugin), - span_id: row.span_id.clone(), - journal_start: start, + *self.pause.lock().unwrap() = Some(PausedWork { + marker_span_id: marker_emitted.then(|| row.span_id.clone()), marker_emitted, marker_cleared: false, }); + match crate::recovery::WorkScope::delivery(&self.source, &self.session_id, &self.route) { + Ok(scope) => { + let plugin_index = self + .route + .span_plugins + .iter() + .position(|candidate| candidate == plugin) + .unwrap_or(0); + if let Err(error) = crate::recovery::pause( + &self.data_dir, + scope, + start, + operation_index, + crate::recovery::FailureCause::PluginFile { + path: plugin.to_path_buf(), + digest: crate::recovery::plugin_digest(plugin), + plugin_index, + }, + message.to_owned(), + marker_emitted.then(|| row.span_id.clone()), + ) { + self.set_error(format!("failed to persist recovery incident: {error}")); + } + } + Err(error) => self.set_error(format!("failed to identify recovery route: {error}")), + } if let Err(error) = crate::plugin_diagnostics::record_pipeline_failure( &self.data_dir, crate::plugin_diagnostics::PipelineFailure { @@ -1082,8 +1382,8 @@ impl SessionActor { operation, plugin_path: plugin, exception: message, - journal_start: start, - journal_through: through, + span_cursor: start, + span_through: through, }, ) { self.set_error(format!("failed to persist span plugin diagnostic: {error}")); @@ -1096,85 +1396,261 @@ impl SessionActor { )); } - /// Stream the journal through the translator, emitting each entry's spans - /// as they are produced. Nothing is accumulated across entries: peak - /// memory is one journal entry and the ops it yields, so recovering a - /// long session costs the same as running it. + fn record_translation_failure(&self, error: &anyhow::Error, event_window: Option<(u64, u64)>) { + let Some((mut start, _)) = event_window else { + return; + }; + if let Some(position) = error.downcast_ref::() { + start = position.start; + } + let revision = crate::recovery::translator_revision().to_owned(); + let cause = if crate::derived::retryable_storage(error) { + crate::recovery::FailureCause::LocalStorage { + retry_after_ms: crate::recovery::now_ms() + 1_000, + } + } else if let Some(shape) = error.downcast_ref::() { + crate::recovery::FailureCause::InputShape { + event: shape.event.clone(), + translator_revision: revision.clone(), + } + } else { + crate::recovery::FailureCause::TranslatorFault { + translator_revision: revision.clone(), + } + }; + let scope = crate::recovery::WorkScope::SourceSession { + source: self.source.clone(), + session_id: self.session_id.clone(), + }; + if let Err(persist_error) = crate::recovery::pause( + &self.data_dir, + scope, + start, + 0, + cause, + error.to_string(), + None, + ) { + self.set_error(format!( + "failed to persist translation incident: {persist_error}" + )); + } + if self.pause.lock().unwrap().is_none() { + *self.pause.lock().unwrap() = Some(PausedWork::default()); + } + } + + fn record_delivery_failure( + &self, + error: &anyhow::Error, + event_window: Option<(u64, u64)>, + operation_index: u32, + ) { + let Ok(scope) = + crate::recovery::WorkScope::delivery(&self.source, &self.session_id, &self.route) + else { + self.set_error("could not identify failed delivery route".into()); + return; + }; + let prior = crate::recovery::active(&self.data_dir, &scope) + .ok() + .flatten(); + let cause = crate::recovery::classify_delivery_error( + error, + &self.route, + self.config.auth.api_url.as_deref(), + prior.as_ref().map_or(0, |incident| incident.attempts), + ); + let first_unprocessed = event_window.map_or(0, |window| window.0); + if let Err(persist_error) = crate::recovery::pause( + &self.data_dir, + scope, + first_unprocessed, + operation_index, + cause, + error.to_string(), + prior.and_then(|incident| incident.marker_span_id), + ) { + self.set_error(format!( + "failed to persist delivery incident: {persist_error}" + )); + } + if self.pause.lock().unwrap().is_none() { + *self.pause.lock().unwrap() = Some(PausedWork::default()); + } + } + + /// Source translation and route delivery pause independently. + fn source_paused(&self) -> bool { + crate::recovery::active( + &self.data_dir, + &crate::recovery::WorkScope::SourceSession { + source: self.source.clone(), + session_id: self.session_id.clone(), + }, + ) + .ok() + .flatten() + .is_some() + } + fn delivery_paused(&self) -> bool { + crate::recovery::WorkScope::delivery(&self.source, &self.session_id, &self.route) + .ok() + .and_then(|scope| { + crate::recovery::active(&self.data_dir, &scope) + .ok() + .flatten() + }) + .is_some() + } + + async fn prepare_and_replay( + &self, + translator: &mut Box, + sink: &mut Box, + ctx: &SessionCtx, + plan: Option<&ReplayPlan>, + deliver: bool, + ) -> bool { + if let Some(plan) = plan { + // Produce outstanding captured input upstream, then consume only the committed span ledger. + let source_scope = crate::recovery::WorkScope::SourceSession { + source: self.source.clone(), + session_id: self.session_id.clone(), + }; + let can_produce = crate::recovery::active(&self.data_dir, &source_scope) + .ok() + .flatten() + .is_none_or(|incident| { + incident.cause.check_local( + crate::recovery::translator_revision(), + crate::recovery::now_ms(), + ) == crate::recovery::CheckResult::RetryCandidate + }); + if can_produce { + if let Err(error) = self.derived.ensure_event(plan.through).await { + self.record_translation_failure( + &error, + Some((plan.acknowledged_through, plan.through)), + ); + self.set_error(format!("span production failed: {error}")); + } + } + if let Err(error) = self.derived.collect_input(&self.journal).await { + self.set_error(format!("event WAL collection failed: {error}")); + return false; + } + } + self.replay_into(translator, sink, ctx, plan, deliver).await + } + async fn replay_into( &self, translator: &mut Box, sink: &mut Box, ctx: &SessionCtx, plan: Option<&ReplayPlan>, + deliver: bool, ) -> bool { - let Some(plan) = plan else { + if plan.is_none() { return true; - }; - let mut reader = match crate::journal::JournalReader::open(&plan.journal_path, plan.through) - .await - { - Ok(Some(reader)) => reader, - Ok(None) => return true, + } + match self.derived.consumer_sequence(&self.route).await { + Ok(sequence) => self.delivered_sequence.store(sequence, Ordering::Relaxed), Err(error) => { - tracing::warn!(session_id = %self.session_id, "journal replay skipped: {error}"); + self.record_delivery_failure( + &error, + Some(( + self.delivered_sequence.load(Ordering::Relaxed), + self.delivered_sequence.load(Ordering::Relaxed), + )), + 0, + ); + self.set_error(format!("span ledger cursor failed: {error}")); return false; } + } + self.consume_into(translator, sink, ctx, deliver, BatchMode::Replay) + .await + .1 + } + + /// Every downstream stage reads only committed span revisions, in output sequence order. + async fn consume_into( + &self, + translator: &mut Box, + sink: &mut Box, + ctx: &SessionCtx, + deliver: bool, + mode: BatchMode, + ) -> (bool, bool) { + let tip = match self.derived.tip().await { + Ok(tip) => tip, + Err(error) => { + self.record_delivery_failure(&error, None, 0); + self.set_error(format!("span ledger tip failed: {error}")); + return (false, false); + } }; - let mut delivered = true; - let mut before = 0; + let mut after = self.delivered_sequence.load(Ordering::Relaxed); + let mut changed = false; loop { - let entry = match reader.next_record().await { - Ok(Some(entry)) => entry, + let revision = match self.derived.next_revision(after, tip).await { + Ok(Some(revision)) => revision, Ok(None) => break, Err(error) => { - tracing::warn!(session_id = %self.session_id, "journal replay stopped: {error}"); - delivered = false; - break; + self.record_delivery_failure(&error, Some((after, after)), 0); + self.set_error(format!("span ledger replay failed: {error}")); + return (changed, false); } }; - let entry_through = entry.through; - let crate::journal::JournalRecord::Event(entry) = entry.record else { - before = entry_through; - continue; - }; - let mut env = crate::journal::envelope_from_redacted(entry); - let Some(canonical_source) = self.translators.canonical_source(&env.source) else { - before = entry_through; - continue; - }; - if canonical_source != self.source { - before = entry_through; - continue; - } - env.source = self.source.clone(); - if env.source == "claude-code" { - if let Some(payload) = env.payload.as_object_mut() { - payload.insert("_bt_transcript_replay".to_string(), serde_json::json!(true)); - } - } + after = revision.sequence; + let env = revision + .envelope + .map(crate::journal::envelope_from_redacted) + .unwrap_or_else(|| Envelope { + source: self.source.clone(), + session_id: self.session_id.clone(), + event: String::new(), + ts_ms: 0, + payload: serde_json::Value::Null, + source_version: None, + plugin_version: None, + managed_run_id: None, + capture: None, + route: None, + config: None, + }); + translator.set_revision_path(&self.derived.revision_path_for_id(&revision.id)); let translated = translator.handle(&env, ctx); - if entry_through <= plan.acknowledged_through { - self.replay_without_delivery(translator, ctx, translated) - .await; - } else { - let (_, replayed) = self + if deliver { + let (revision_changed, delivered) = self .emit_translator_batches( translator, sink, ctx, translated, - BatchMode::Replay, - Some((before, entry_through)), + mode, + Some((revision.sequence - 1, revision.sequence)), ) .await; - delivered &= replayed; - if self.pause.lock().unwrap().is_some() && !replayed { - break; + changed |= revision_changed; + if !delivered { + return (changed, false); } + } else if !self + .replay_without_delivery( + translator, + ctx, + translated, + (revision.sequence - 1, revision.sequence), + ) + .await + { + return (changed, false); } - before = entry_through; } - delivered + (changed, true) } /// Continue a replayed translator exactly as usual but deliberately omit @@ -1186,12 +1662,14 @@ impl SessionActor { translator: &mut Box, ctx: &SessionCtx, first: anyhow::Result>, - ) { + event_window: (u64, u64), + ) -> bool { let mut next = match first { Ok(ops) => Some(ops), Err(error) => { + self.record_translation_failure(&error, Some(event_window)); self.set_error(format!("journal replay failed: {error}")); - return; + return false; } }; while let Some(ops) = next { @@ -1206,11 +1684,13 @@ impl SessionActor { next = match translator.drain_pending(ctx) { Ok(next) => next, Err(error) => { + self.record_translation_failure(&error, Some(event_window)); self.set_error(format!("journal replay failed: {error}")); - None + return false; } }; } + true } async fn checkpoint_and_flush( @@ -1220,19 +1700,22 @@ impl SessionActor { ctx: &SessionCtx, event_window: Option<(u64, u64)>, ) -> bool { - let translated = translator.checkpoint(ctx); - let (correlation_changed, delivered) = self - .emit_translator_batches( - translator, - sink, - ctx, - translated, - BatchMode::Checkpoint, - event_window, + if let Err(error) = self + .derived + .ensure_control( + crate::derived::ControlKind::Checkpoint, + event_window.map_or(0, |window| window.1), ) + .await + { + self.record_translation_failure(&error, event_window); + self.set_error(format!("span production failed: {error}")); + return false; + } + let (changed, delivered) = self + .consume_into(translator, sink, ctx, true, BatchMode::Checkpoint) .await; - self.persist_correlation_if_changed(correlation_changed) - .await; + self.persist_correlation_if_changed(changed).await; delivered } @@ -1244,12 +1727,20 @@ impl SessionActor { mode: BatchMode, event_window: Option<(u64, u64)>, ) -> bool { - let translated = translator.finalize(ctx); - let (correlation_changed, delivered) = self - .emit_translator_batches(translator, sink, ctx, translated, mode, event_window) - .await; - self.persist_correlation_if_changed(correlation_changed) - .await; + if let Err(error) = self + .derived + .ensure_control( + crate::derived::ControlKind::Finalize, + event_window.map_or(0, |window| window.1), + ) + .await + { + self.record_translation_failure(&error, event_window); + self.set_error(format!("span production failed: {error}")); + return false; + } + let (changed, delivered) = self.consume_into(translator, sink, ctx, true, mode).await; + self.persist_correlation_if_changed(changed).await; delivered } @@ -1268,8 +1759,28 @@ impl SessionActor { } } - async fn flush_sink(&self, sink: &mut Box) -> bool { + async fn flush_sink( + &self, + sink: &mut Box, + _capture_cursor: u64, + ) -> bool { + let first_unprocessed = match self.derived.consumer_sequence(&self.route).await { + Ok(cursor) => cursor, + Err(error) => { + self.record_delivery_failure( + &error, + Some(( + self.delivered_sequence.load(Ordering::Relaxed), + self.delivered_sequence.load(Ordering::Relaxed), + )), + 0, + ); + self.set_error(format!("span ledger cursor failed: {error}")); + return false; + } + }; if let Err(e) = sink.flush().await { + self.record_delivery_failure(&e, Some((first_unprocessed, first_unprocessed)), 0); self.set_error(format!("sink flush failed: {e}")); return false; } @@ -1280,25 +1791,29 @@ impl SessionActor { async fn record_delivery_checkpoint( &self, sink: &mut Box, - through: u64, + _capture_tip: u64, acknowledged_through: &mut u64, ) { - if !self.flush_sink(sink).await { + if !self.flush_sink(sink, *acknowledged_through).await { return; } - if through <= *acknowledged_through { - return; - } - if let Err(error) = self - .journal - .lock() - .await - .append_delivery_checkpoint(&self.route, through) + match self + .derived + .acknowledge(&self.route, self.delivered_sequence.load(Ordering::Relaxed)) .await { - self.set_error(format!("delivery checkpoint failed: {error}")); - } else { - *acknowledged_through = through; + Ok(through) => *acknowledged_through = through, + Err(error) => { + self.record_delivery_failure( + &error, + Some(( + self.delivered_sequence.load(Ordering::Relaxed), + self.delivered_sequence.load(Ordering::Relaxed), + )), + 0, + ); + self.set_error(format!("delivery checkpoint failed: {error}")); + } } } diff --git a/bt-daemon/src/durable.rs b/bt-daemon/src/durable.rs new file mode 100644 index 00000000..0b4f0510 --- /dev/null +++ b/bt-daemon/src/durable.rs @@ -0,0 +1,42 @@ +//! Atomic, private control-state commits used by the recovery pipeline. +use serde::Serialize; +use std::path::Path; +use tokio::io::AsyncWriteExt; + +pub(crate) async fn create_private(path: &Path) -> anyhow::Result { + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + Ok(options.open(path).await?) +} + +pub(crate) fn sync_parent(path: &Path) -> anyhow::Result<()> { + #[cfg(unix)] + std::fs::File::open(path.parent().expect("state directory"))?.sync_all()?; + #[cfg(not(unix))] + let _ = path; + Ok(()) +} + +pub(crate) async fn write_json(path: &Path, value: &impl Serialize) -> anyhow::Result<()> { + let temp = path + .parent() + .expect("state directory") + .join(format!("{}.tmp", uuid::Uuid::new_v4())); + let result = async { + let mut file = create_private(&temp).await?; + file.write_all(&serde_json::to_vec(value)?).await?; + file.flush().await?; + file.sync_data().await?; + crate::paths::restrict_file_to_owner(&temp)?; + tokio::fs::rename(&temp, path).await?; + sync_parent(path)?; + Ok(()) + } + .await; + if result.is_err() { + let _ = tokio::fs::remove_file(temp).await; + } + result +} diff --git a/bt-daemon/src/hook.rs b/bt-daemon/src/hook.rs index 72d45800..f99f64fb 100644 --- a/bt-daemon/src/hook.rs +++ b/bt-daemon/src/hook.rs @@ -144,6 +144,7 @@ async fn capture_hook( let session_id = json_str_field(&payload, &args.session_id_field) .ok_or_else(|| anyhow::anyhow!("no `{}` field in hook payload", args.session_id_field))?; + anyhow::ensure!(!session_id.trim().is_empty(), "hook session id is empty"); let event = args .event .clone() diff --git a/bt-daemon/src/journal.rs b/bt-daemon/src/journal.rs index 9dadf5b0..01dfbbe3 100644 --- a/bt-daemon/src/journal.rs +++ b/bt-daemon/src/journal.rs @@ -12,14 +12,25 @@ use crate::wire::{BackendAuth, Envelope, RedactedEnvelope, SessionRoute}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashSet; +use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; pub fn journal_dir(data_dir: &Path) -> PathBuf { data_dir.join("journal") } +fn checkpoint_path(journal_path: &Path) -> PathBuf { + journal_path + .parent() + .and_then(Path::parent) + .unwrap_or_else(|| Path::new(".")) + .join("journal-control") + .join(journal_path.file_name().unwrap_or_default()) +} + pub(crate) fn sanitize(s: &str) -> String { s.chars() .map(|c| { @@ -32,8 +43,7 @@ pub(crate) fn sanitize(s: &str) -> String { .collect() } -/// Longest single path component, in bytes, that common filesystems accept. -pub(crate) const MAX_FILE_NAME_BYTES: usize = 255; +pub(crate) const MAX_FILE_NAME_BYTES: usize = 240; pub fn journal_path(data_dir: &Path, session_id: &str) -> PathBuf { journal_dir(data_dir).join(format!("{}.ndjson", sanitize(session_id))) @@ -217,76 +227,258 @@ pub async fn gc_old_managed_runs(data_dir: &Path, max_age: std::time::Duration) } } +/// Logical offsets survive prefix collection. The header belongs to framing, +/// not to the captured native stream. +fn journal_bounds(path: &Path) -> anyhow::Result<(u64, u64, u64)> { + use std::io::BufRead; + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((0, 0, 0)), + Err(error) => return Err(error.into()), + }; + let len = file.metadata()?.len(); + let mut reader = std::io::BufReader::new(file); + let mut first = String::new(); + reader.read_line(&mut first)?; + let base = serde_json::from_str::(&first) + .ok() + .and_then(|value| { + value + .get("_bt_wal_base") + .and_then(serde_json::Value::as_u64) + }); + let header = if base.is_some() { + first.len() as u64 + } else { + 0 + }; + Ok((base.unwrap_or(0), header, len)) +} + +pub(crate) fn route_catalog_path(path: &Path) -> PathBuf { + checkpoint_path(path).with_extension("routes.json") +} + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct CapturedRoute { + pub envelope: RedactedEnvelope, + pub through: u64, +} + +pub(crate) fn captured_routes(path: &Path) -> anyhow::Result> { + match std::fs::read(route_catalog_path(path)) { + Ok(data) => Ok(serde_json::from_slice(&data)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(error.into()), + } +} + /// Append-only journal writer for one session. pub struct JournalWriter { file: tokio::fs::File, + path: PathBuf, position: u64, - pi_context: Vec, } impl JournalWriter { pub async fn open_path(path: &Path) -> anyhow::Result { if let Some(dir) = path.parent() { - tokio::fs::create_dir_all(dir).await?; + crate::paths::ensure_private_dir(dir).map_err(|error| { + anyhow::anyhow!("create journal directory {}: {error}", dir.display()) + })?; } - let file = tokio::fs::OpenOptions::new() - .create(true) - .append(true) + truncate_incomplete_tail(path) + .map_err(|error| anyhow::anyhow!("prepare journal {}: {error}", path.display()))?; + let mut options = tokio::fs::OpenOptions::new(); + options.create(true).append(true); + #[cfg(unix)] + options.mode(0o600); + let file = options .open(path) - .await?; - let position = file.metadata().await?.len(); - Ok(Self { + .await + .map_err(|error| anyhow::anyhow!("open journal {}: {error}", path.display()))?; + crate::paths::restrict_file_to_owner(path)?; + let (base, header, len) = journal_bounds(path)?; + let position = base + len - header; + let writer = Self { file, + path: path.to_path_buf(), position, - pi_context: Vec::new(), - }) + }; + if captured_routes(path)?.is_empty() && position > 0 { + if let Some(mut reader) = JournalReader::open(path, position).await? { + while let Some(entry) = reader.next_record().await? { + if let JournalRecord::Event(env) = entry.record { + writer + .remember_route(&envelope_from_redacted(env), entry.through) + .await?; + } + } + } + } + Ok(writer) } pub(crate) fn position(&self) -> u64 { self.position } - /// Append one event in redacted form and flush to the OS. Not fsync'd per - /// event (that would dominate hook latency); an OS crash can lose the last - /// few lines, which replay tolerates. + /// Append one event and durably commit it before acknowledging capture. /// - /// The journal is a durability record and is never capped or truncated: - /// dropping entries would silently cost recovery fidelity. Its size is - /// bounded instead by writing each transcript byte once (see - /// [`crate::transcript_mirror`]) and by age-based GC, and replay reads it - /// as a stream so a large journal never becomes a large allocation. + /// Only input whose output and continuation state have been committed may + /// be collected. Untranslated input remains available for recovery. pub async fn append(&mut self, env: &Envelope) -> anyhow::Result { - let mut redacted = env.redacted(); - compact_pi_payload(&mut redacted, &mut self.pi_context); - let mut line = serde_json::to_vec(&redacted)?; + let redacted = env.redacted(); + // Native observations are the WAL's source of truth. In particular, + // Pi context snapshots must stay intact even when a previous snapshot + // contains the same prefix: a record should stand on its own. + let mut value = serde_json::to_value(&redacted)?; + let payload = serde_json::to_vec(&value)?; + let object = value + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("journal envelope is not an object"))?; + object.insert("_bt_wal_version".into(), serde_json::json!(1)); + object.insert("_bt_wal_offset".into(), serde_json::json!(self.position)); + object.insert("_bt_wal_length".into(), serde_json::json!(payload.len())); + object.insert( + "_bt_wal_sha256".into(), + serde_json::json!(format!("{:x}", Sha256::digest(&payload))), + ); + let mut line = serde_json::to_vec(&value)?; line.push(b'\n'); - self.file.write_all(&line).await?; - self.file.flush().await?; + self.file + .write_all(&line) + .await + .map_err(|error| anyhow::anyhow!("write journal {}: {error}", self.path.display()))?; + self.file + .flush() + .await + .map_err(|error| anyhow::anyhow!("flush journal {}: {error}", self.path.display()))?; + self.file + .sync_data() + .await + .map_err(|error| anyhow::anyhow!("sync journal {}: {error}", self.path.display()))?; self.position += line.len() as u64; + self.remember_route(env, self.position).await?; Ok(self.position) } - /// Record that the sink durably accepted every event through this byte - /// offset for one route. It lives in the event journal so a cold worker - /// restores delivery knowledge together with translator state. - pub async fn append_delivery_checkpoint( - &mut self, - route: &SessionRoute, - through: u64, - ) -> anyhow::Result<()> { - let mut line = serde_json::to_vec(&DeliveryCheckpointRecord { - record_type: "delivery_checkpoint".into(), - route: route.clone(), + async fn remember_route(&self, env: &Envelope, through: u64) -> anyhow::Result<()> { + let Some(route) = &env.route else { + return Ok(()); + }; + let mut routes = captured_routes(&self.path)?; + let mut template = env.redacted(); + // Keep observation locations for the existing late-transcript watcher, + // not native event contents or credentials. + let mut payload = serde_json::Map::new(); + for field in [ + "transcript_path", + "agent_transcript_path", + "session_dir", + "_bt_transcript_mirror", + "_bt_transcript_mirrors", + "_bt_grok_transcript_mirrors", + ] { + if let Some(value) = template.payload.get(field) { + payload.insert(field.into(), value.clone()); + } + } + template.payload = serde_json::Value::Object(payload); + let record = CapturedRoute { + envelope: template, through, - })?; - line.push(b'\n'); - self.file.write_all(&line).await?; - self.file.flush().await?; - self.position += line.len() as u64; + }; + if let Some(previous) = routes.iter_mut().find(|entry| { + entry + .envelope + .route + .as_ref() + .is_some_and(|candidate| candidate.same_route(route)) + }) { + *previous = record; + } else { + routes.push(record); + } + let path = route_catalog_path(&self.path); + crate::paths::ensure_private_dir(path.parent().expect("route catalog parent"))?; + crate::durable::write_json(&path, &routes).await + } + + /// Replace a consumed prefix while preserving logical positions and any + /// captured tail. The writer lock excludes concurrent append/collection. + pub(crate) async fn collect_through(&mut self, through: u64) -> anyhow::Result<()> { + let (base, header, _) = journal_bounds(&self.path)?; + if through <= base { + return Ok(()); + } + anyhow::ensure!( + through <= self.position, + "collection exceeds captured WAL tip" + ); + let mut input = tokio::fs::File::open(&self.path).await?; + input + .seek(std::io::SeekFrom::Start(header + through - base)) + .await?; + let temp = self + .path + .parent() + .expect("journal directory") + .join(format!("{}.tmp", uuid::Uuid::new_v4())); + let mut output = crate::durable::create_private(&temp).await?; + output + .write_all(format!("{{\"_bt_wal_base\":{through}}}\n").as_bytes()) + .await?; + tokio::io::copy(&mut input, &mut output).await?; + output.flush().await?; + output.sync_data().await?; + tokio::fs::rename(&temp, &self.path).await?; + crate::durable::sync_parent(&self.path)?; + self.file = tokio::fs::OpenOptions::new() + .append(true) + .open(&self.path) + .await?; Ok(()) } } +/// Keep the last complete record after a crash so the next append starts at +/// a fresh frame boundary. A legacy newline journal has the same framing. +fn truncate_incomplete_tail(path: &Path) -> anyhow::Result<()> { + let mut file = match std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let mut end = file.metadata()?.len(); + if end == 0 { + return Ok(()); + } + let mut chunk = [0u8; 4096]; + while end > 0 { + let start = end.saturating_sub(chunk.len() as u64); + let count = (end - start) as usize; + file.seek(SeekFrom::Start(start))?; + file.read_exact(&mut chunk[..count])?; + if let Some(index) = chunk[..count].iter().rposition(|byte| *byte == b'\n') { + let complete = start + index as u64 + 1; + if complete < file.metadata()?.len() { + file.set_len(complete)?; + file.sync_data()?; + } + return Ok(()); + } + end = start; + } + file.set_len(0)?; + file.sync_data()?; + Ok(()) +} + #[derive(Debug, Clone, Serialize, Deserialize)] struct DeliveryCheckpointRecord { #[serde(rename = "_bt_record_type")] @@ -332,16 +524,24 @@ impl JournalReader { /// already recovered state, never the live event still on its way to the /// queue. `Ok(None)` means the session has no journal yet. pub async fn open(path: &Path, through: u64) -> anyhow::Result> { - let file = match tokio::fs::File::open(path).await { + Self::open_from(path, 0, through).await + } + + pub async fn open_from(path: &Path, start: u64, through: u64) -> anyhow::Result> { + let mut file = match tokio::fs::File::open(path).await { Ok(file) => file, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error.into()), }; + let (base, header, _) = journal_bounds(path)?; + let start = start.max(base); + file.seek(std::io::SeekFrom::Start(header + start - base)) + .await?; Ok(Some(Self { - reader: tokio::io::BufReader::new(file.take(through)), + reader: tokio::io::BufReader::new(file.take(through.saturating_sub(start))), path: path.to_path_buf(), line_no: 0, - position: 0, + position: start, pi_context: Vec::new(), })) } @@ -349,9 +549,8 @@ impl JournalReader { /// The journal's current length, which is the bound a session created now /// should replay through. A missing journal replays nothing. pub async fn recorded_len(path: &Path) -> u64 { - tokio::fs::metadata(path) - .await - .map(|meta| meta.len()) + journal_bounds(path) + .map(|(base, header, len)| base + len - header) .unwrap_or(0) } @@ -375,6 +574,11 @@ impl JournalReader { return Ok(None); } self.position += read as u64; + // An interrupted append can leave an unterminated final record. + // Earlier complete records remain valid and replayable. + if bytes.last() != Some(&b'\n') { + return Ok(None); + } self.line_no += 1; let line = std::str::from_utf8(&bytes) .map_err(|error| { @@ -384,9 +588,29 @@ impl JournalReader { if line.is_empty() { continue; } - let value: serde_json::Value = serde_json::from_str(line).map_err(|error| { + let mut value: serde_json::Value = serde_json::from_str(line).map_err(|error| { anyhow::anyhow!("journal {}:{}: {error}", self.path.display(), self.line_no) })?; + if let Some(object) = value.as_object_mut() { + if let Some(version) = object.remove("_bt_wal_version") { + anyhow::ensure!(version == 1, "unsupported journal record version {version}"); + let offset = object.remove("_bt_wal_offset").and_then(|v| v.as_u64()); + let length = object.remove("_bt_wal_length").and_then(|v| v.as_u64()); + let checksum = object + .remove("_bt_wal_sha256") + .and_then(|v| v.as_str().map(str::to_owned)); + let payload = serde_json::to_vec(object)?; + anyhow::ensure!( + offset == Some(self.position - read as u64) + && length == Some(payload.len() as u64) + && checksum.as_deref() + == Some(&format!("{:x}", Sha256::digest(&payload))), + "journal {}:{}: record checksum or position mismatch", + self.path.display(), + self.line_no, + ); + } + } let record = if value .get("_bt_record_type") .and_then(serde_json::Value::as_str) @@ -435,74 +659,39 @@ impl JournalReader { } } } + for (candidate, offset) in read_checkpoint_state(path).await { + if candidate.same_route(route) { + acknowledged = acknowledged.max(offset); + } + } acknowledged } } -const PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; - -fn compact_pi_payload(event: &mut RedactedEnvelope, previous: &mut Vec) { - if event.source != "pi" { - return; - } - let Some(native) = event - .payload - .get_mut("event") - .and_then(serde_json::Value::as_object_mut) - else { - return; - }; - match event.event.as_str() { - "context" => { - let Some(messages) = native - .remove("messages") - .and_then(|messages| messages.as_array().cloned()) - else { - return; - }; - let common_prefix = previous - .iter() - .zip(&messages) - .take_while(|(left, right)| left == right) - .count(); - native.insert( - PI_MESSAGES_DELTA.into(), - serde_json::json!({ - "common_prefix": common_prefix, - "suffix": messages[common_prefix..], - }), - ); - *previous = messages; - } - "before_provider_request" => { - // The translator records only the allowlisted request - // configuration and tool definitions, never the prompt or messages. - if let Some(payload) = native.get_mut("payload") { - *payload = serde_json::Value::Object(crate::translate::pi_request_config(payload)); - } - } - "agent_end" => { - // The translator only consumes willRetry from this lifecycle event. - native.remove("messages"); - } - "message_update" => { - // Pi repeats the progressively growing assistant message on every - // streaming update. Translation only needs the update type to mark - // time-to-first-token; the completed message arrives separately in - // message_end. - native.remove("message"); - if let Some(update) = native - .get_mut("assistantMessageEvent") - .and_then(serde_json::Value::as_object_mut) - { - update.retain(|key, _| key == "type"); +/// Control records live beside, rather than inside, the native event WAL. +/// Legacy inline checkpoints remain readable through JournalReader. +pub async fn read_checkpoint_state(path: &Path) -> Vec<(SessionRoute, u64)> { + let mut checkpoints = Vec::new(); + if let (Some(data_dir), Ok(routes)) = + (path.parent().and_then(Path::parent), captured_routes(path)) + { + if let Some(captured) = routes.first() { + if let Ok(current) = crate::derived::checkpoints( + data_dir, + &captured.envelope.source, + &captured.envelope.session_id, + ) { + checkpoints.extend(current); } - native.retain(|key, _| key == "type" || key == "assistantMessageEvent"); } - _ => {} } + checkpoints } +const PI_MESSAGES_DELTA: &str = "_bt_messages_delta"; + +// Older journals used Pi context deltas. Keep decoding those records while +// all new captures retain the complete native observation. fn expand_pi_payload( event: &mut RedactedEnvelope, previous: &mut Vec, @@ -545,59 +734,10 @@ fn expand_pi_payload( Ok(()) } -/// Best-effort age-based journal collection. A failed stat/remove is logged -/// and ignored; stale state must never prevent the daemon from serving hooks. +/// Input collection is driven by durable translation progress, never by age. +/// Maintenance cannot discard captured input just because an incident was resolved. pub async fn gc_old_journals(data_dir: &Path, max_age: std::time::Duration) { - let dir = journal_dir(data_dir); - let diagnostics = match crate::plugin_diagnostics::read(data_dir) { - Ok(diagnostics) => diagnostics, - Err(error) => { - tracing::warn!(%error, "journal collection skipped because plugin recovery state is unreadable"); - return; - } - }; - let protected: HashSet = diagnostics - .into_iter() - .filter(|entry| { - matches!( - entry.state, - Some( - crate::plugin_diagnostics::PluginRecoveryState::Paused - | crate::plugin_diagnostics::PluginRecoveryState::Reprocessing - ) - ) - }) - .filter_map(|entry| { - entry - .session_id - .map(|session_id| source_journal_path(data_dir, &entry.source, &session_id)) - }) - .collect(); - let Ok(mut entries) = tokio::fs::read_dir(&dir).await else { - return; - }; - let now = std::time::SystemTime::now(); - while let Ok(Some(entry)) = entries.next_entry().await { - let path = entry.path(); - if protected.contains(&path) { - continue; - } - if path.extension().and_then(|v| v.to_str()) != Some("ndjson") { - continue; - } - let old = entry - .metadata() - .await - .ok() - .and_then(|m| m.modified().ok()) - .and_then(|modified| now.duration_since(modified).ok()) - .is_some_and(|age| age > max_age); - if old { - if let Err(e) = tokio::fs::remove_file(&path).await { - tracing::warn!(path = %path.display(), "failed to remove stale journal: {e}"); - } - } - } + let _ = (data_dir, max_age); } /// Reconstruct a translator-usable [`Envelope`] from a redacted journal entry. @@ -635,9 +775,8 @@ mod tests { use super::*; #[tokio::test] - async fn collection_keeps_a_journal_with_a_paused_plugin_pipeline() { + async fn age_does_not_discard_untranslated_input() { let temp = tempfile::tempdir().unwrap(); - let route = SessionRoute::default(); let path = source_journal_path(temp.path(), "claude", "session"); tokio::fs::create_dir_all(path.parent().unwrap()) .await @@ -649,38 +788,46 @@ mod tests { .unwrap() .set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400)) .unwrap(); - let plugin = temp.path().join("redact.mjs"); - tokio::fs::write(&plugin, b"export default span => span") + gc_old_journals(temp.path(), std::time::Duration::from_secs(1)).await; + assert!(path.exists()); + } + + #[tokio::test] + async fn resolving_an_incident_does_not_discard_untranslated_input() { + let temp = tempfile::tempdir().unwrap(); + let path = source_journal_path(temp.path(), "pi", "held-session"); + tokio::fs::create_dir_all(path.parent().unwrap()) .await .unwrap(); - crate::plugin_diagnostics::record_pipeline_failure( + tokio::fs::write(&path, b"event\n").await.unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400)) + .unwrap(); + let scope = crate::recovery::WorkScope::SourceSession { + source: "pi".into(), + session_id: "held-session".into(), + }; + crate::recovery::pause( temp.path(), - crate::plugin_diagnostics::PipelineFailure { - source: "claude", - session_id: "session", - route: &route, - span_id: "span", - operation: "insert", - plugin_path: &plugin, - exception: "interrupted", - journal_start: 0, - journal_through: 6, + scope.clone(), + 0, + 0, + crate::recovery::FailureCause::InputShape { + event: "tool_execution_start".into(), + translator_revision: "old".into(), }, + "missing toolCallId".into(), + None, ) .unwrap(); gc_old_journals(temp.path(), std::time::Duration::from_secs(1)).await; assert!(path.exists()); - crate::plugin_diagnostics::update_pipeline( - temp.path(), - "claude", - "session", - &route, - crate::plugin_diagnostics::PluginRecoveryState::Recovered, - 6, - ) - .unwrap(); + crate::recovery::resolve(temp.path(), &scope).unwrap(); gc_old_journals(temp.path(), std::time::Duration::from_secs(1)).await; - assert!(!path.exists()); + assert!(path.exists()); } fn pi_context(messages: Vec, ts_ms: i64) -> Envelope { @@ -700,7 +847,7 @@ mod tests { } #[tokio::test] - async fn pi_context_history_is_delta_encoded_and_replayed() { + async fn pi_context_history_is_preserved_and_replayed() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("pi.ndjson"); let mut writer = JournalWriter::open_path(&path).await.unwrap(); @@ -725,12 +872,8 @@ mod tests { drop(writer); let stored = tokio::fs::read(&path).await.unwrap(); - assert!( - stored.len() < naive_bytes / 3, - "Pi journal did not eliminate cumulative context copies: stored={} naive={naive_bytes}", - stored.len() - ); - assert!(!String::from_utf8_lossy(&stored).contains("\"messages\"")); + assert!(stored.len() >= naive_bytes); + assert!(String::from_utf8_lossy(&stored).contains("\"messages\"")); let through = stored.len() as u64; let mut reader = JournalReader::open(&path, through).await.unwrap().unwrap(); @@ -751,7 +894,45 @@ mod tests { } #[tokio::test] - async fn pi_streaming_updates_do_not_persist_growing_partial_messages() { + async fn wal_recovers_torn_tail_and_detects_corruption() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("events.ndjson"); + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + writer.append(&pi_context(vec![], 1)).await.unwrap(); + let first = writer.position(); + drop(writer); + // Tokio's Windows file wrapper closes through its blocking worker. + // Give that close a turn before simulating a process restart with a + // second handle to the same file. + tokio::task::yield_now().await; + let mut file = tokio::fs::OpenOptions::new() + .append(true) + .open(&path) + .await + .unwrap(); + file.write_all(b"{\"partial\":true").await.unwrap(); + file.sync_all().await.unwrap(); + drop(file); + tokio::task::yield_now().await; + let mut writer = JournalWriter::open_path(&path).await.unwrap(); + assert_eq!(writer.position(), first); + writer.append(&pi_context(vec![], 2)).await.unwrap(); + drop(writer); + let mut reader = JournalReader::open(&path, u64::MAX).await.unwrap().unwrap(); + assert_eq!(reader.next_entry().await.unwrap().unwrap().ts_ms, 1); + assert_eq!(reader.next_entry().await.unwrap().unwrap().ts_ms, 2); + assert!(reader.next_entry().await.unwrap().is_none()); + + let mut bytes = tokio::fs::read(&path).await.unwrap(); + let changed = bytes.iter().position(|byte| *byte == b'1').unwrap(); + bytes[changed] = b'9'; + tokio::fs::write(&path, bytes).await.unwrap(); + let mut reader = JournalReader::open(&path, u64::MAX).await.unwrap().unwrap(); + assert!(reader.next_entry().await.is_err()); + } + + #[tokio::test] + async fn pi_streaming_updates_replay_complete_native_payload() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("pi.ndjson"); let mut writer = JournalWriter::open_path(&path).await.unwrap(); @@ -782,23 +963,18 @@ mod tests { drop(writer); let stored = tokio::fs::read(&path).await.unwrap(); - assert!(!String::from_utf8_lossy(&stored).contains("growing-partial-marker")); + assert!(String::from_utf8_lossy(&stored).contains("growing-partial-marker")); let mut reader = JournalReader::open(&path, stored.len() as u64) .await .unwrap() .unwrap(); let replayed = reader.next_entry().await.unwrap().unwrap(); - assert_eq!( - replayed - .payload - .pointer("/event/assistantMessageEvent/type"), - Some(&serde_json::json!("text_delta")) - ); + assert_eq!(replayed.payload, event.payload); } #[tokio::test] - async fn pi_provider_requests_persist_only_request_configuration() { + async fn pi_provider_requests_replay_complete_native_payload() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("pi.ndjson"); let mut writer = JournalWriter::open_path(&path).await.unwrap(); @@ -830,55 +1006,15 @@ mod tests { let stored = tokio::fs::read(&path).await.unwrap(); let stored_text = String::from_utf8_lossy(&stored); - assert!(!stored_text.contains("system-prompt-marker")); - assert!(!stored_text.contains("message-marker")); + assert!(stored_text.contains("system-prompt-marker")); + assert!(stored_text.contains("message-marker")); let mut reader = JournalReader::open(&path, stored.len() as u64) .await .unwrap() .unwrap(); let replayed = reader.next_entry().await.unwrap().unwrap(); - assert_eq!( - replayed.payload.pointer("/event/payload"), - Some(&serde_json::json!({ - "max_tokens": 1024, - "tools": [{"type": "function", "function": { - "name": "read", - "parameters": {"type": "object"}, - }}], - })) - ); - } - - #[test] - fn pi_journaled_request_configuration_replays_unchanged() { - let schema = serde_json::json!({"type": "object"}); - for payload in [ - serde_json::json!({ - "max_tokens": 1024, - "tools": [ - {"name": "read", "input_schema": schema, "strict": true}, - {"type": "web_search_20250305", "name": "web_search"}, - ], - "tool_choice": {"type": "tool", "name": "read", "disable_parallel_tool_use": true}, - }), - serde_json::json!({"config": { - "temperature": 0.5, - "tools": [{"functionDeclarations": [{"name": "read", "parameters": schema}]}, {"googleSearch": {}}], - "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, - }}), - serde_json::json!({ - "toolConfig": {"tools": [{"toolSpec": {"name": "read", "inputSchema": {"json": schema}}}]}, - "inferenceConfig": {"maxTokens": 512}, - }), - ] { - let journaled = - serde_json::Value::Object(crate::translate::pi_request_config(&payload)); - assert_eq!( - serde_json::Value::Object(crate::translate::pi_request_config(&journaled)), - journaled - ); - } + assert_eq!(replayed.payload, event.payload); } #[tokio::test] diff --git a/bt-daemon/src/lib.rs b/bt-daemon/src/lib.rs index d31d3551..fb79ded7 100644 --- a/bt-daemon/src/lib.rs +++ b/bt-daemon/src/lib.rs @@ -17,13 +17,16 @@ mod client; mod command_output; mod correlation; mod delivery_ledger; +mod derived; mod dispatch; +mod durable; mod hook; mod ids; mod journal; mod managed_run; mod plugin_diagnostics; pub(crate) mod process; +mod recovery; mod route; mod server; mod settings; @@ -57,6 +60,7 @@ pub use hook::run_hook; pub use journal::source_journal_path; pub use managed_run::run_traced; pub use plugin_diagnostics::PluginDiagnostic; +pub use recovery::{FailureCause, Incident as RecoveryIncident, WorkScope, WorkState}; pub use server::{ braintrust_serve_options, debug_serve_options, run_serve, AuthLease, AuthProvider, AuthResolveReason, ServeOptions, diff --git a/bt-daemon/src/plugin_diagnostics.rs b/bt-daemon/src/plugin_diagnostics.rs index 19090628..c7eaa4d2 100644 --- a/bt-daemon/src/plugin_diagnostics.rs +++ b/bt-daemon/src/plugin_diagnostics.rs @@ -10,14 +10,6 @@ use std::time::{SystemTime, UNIX_EPOCH}; const MAX_DIAGNOSTICS: usize = 128; const FILE_NAME: &str = "span-plugin-errors.json"; -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PluginRecoveryState { - Paused, - Reprocessing, - Recovered, -} - #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct PluginDiagnostic { pub source: String, @@ -39,13 +31,9 @@ pub struct PluginDiagnostic { #[serde(default, skip_serializing_if = "Option::is_none")] pub pipeline_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub state: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub journal_start: Option, + pub span_cursor: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub pending_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub state_changed_ms: Option, + pub pending_revisions: Option, } #[derive(Default, Serialize, Deserialize)] @@ -88,10 +76,8 @@ pub fn record( span_id: None, operation: None, pipeline_id: None, - state: None, - journal_start: None, - pending_bytes: None, - state_changed_ms: None, + span_cursor: None, + pending_revisions: None, }); } trim(&mut store); @@ -111,8 +97,8 @@ pub struct PipelineFailure<'a> { pub operation: &'a str, pub plugin_path: &'a Path, pub exception: &'a str, - pub journal_start: u64, - pub journal_through: u64, + pub span_cursor: u64, + pub span_through: u64, } pub fn record_pipeline_failure( @@ -130,7 +116,6 @@ pub fn record_pipeline_failure( entry.source == failure.source && entry.session_id.as_deref() == Some(failure.session_id) && entry.pipeline_id.as_deref() == Some(id.as_str()) - && entry.state != Some(PluginRecoveryState::Recovered) }); if let Some(entry) = existing { entry.plugin_path = failure.plugin_path.to_path_buf(); @@ -138,16 +123,11 @@ pub fn record_pipeline_failure( entry.exception = failure.exception.to_owned(); entry.span_id = Some(failure.span_id.to_owned()); entry.operation = Some(failure.operation.to_owned()); - entry.journal_start = Some(failure.journal_start); - entry.pending_bytes = Some( - failure - .journal_through - .saturating_sub(failure.journal_start), - ); + entry.span_cursor = Some(failure.span_cursor); + entry.pending_revisions = + Some(failure.span_through.saturating_sub(failure.span_cursor)); entry.last_seen_ms = now; entry.occurrences = entry.occurrences.saturating_add(1); - entry.state = Some(PluginRecoveryState::Paused); - entry.state_changed_ms = Some(now); } else { store.entries.push(PluginDiagnostic { source: failure.source.to_owned(), @@ -161,14 +141,8 @@ pub fn record_pipeline_failure( span_id: Some(failure.span_id.to_owned()), operation: Some(failure.operation.to_owned()), pipeline_id: Some(id), - state: Some(PluginRecoveryState::Paused), - journal_start: Some(failure.journal_start), - pending_bytes: Some( - failure - .journal_through - .saturating_sub(failure.journal_start), - ), - state_changed_ms: Some(now), + span_cursor: Some(failure.span_cursor), + pending_revisions: Some(failure.span_through.saturating_sub(failure.span_cursor)), }); } trim(&mut store); @@ -176,53 +150,12 @@ pub fn record_pipeline_failure( }) } -pub fn update_pipeline( - data_dir: &Path, - source: &str, - session_id: &str, - route: &SessionRoute, - state: PluginRecoveryState, - journal_through: u64, -) -> anyhow::Result<()> { - let path = diagnostics_path(data_dir); - ensure_diagnostics_dir(&path)?; - let id = pipeline_id(route)?; - crate::settings::with_settings_lock(&path, || { - let mut store = read_unlocked(&path)?; - if let Some(entry) = store.entries.iter_mut().find(|entry| { - entry.source == source - && entry.session_id.as_deref() == Some(session_id) - && entry.pipeline_id.as_deref() == Some(id.as_str()) - && entry.state != Some(PluginRecoveryState::Recovered) - }) { - entry.state = Some(state); - entry.state_changed_ms = Some(now_ms()); - entry.pending_bytes = (state != PluginRecoveryState::Recovered) - .then(|| journal_through.saturating_sub(entry.journal_start.unwrap_or(0))); - write_unlocked(&path, &store)?; - } - Ok(()) - }) -} - fn trim(store: &mut DiagnosticStore) { store .entries .sort_by_key(|diagnostic| diagnostic.last_seen_ms); - let mut excess = store.entries.len().saturating_sub(MAX_DIAGNOSTICS); - store.entries.retain(|diagnostic| { - if excess > 0 - && !matches!( - diagnostic.state, - Some(PluginRecoveryState::Paused | PluginRecoveryState::Reprocessing) - ) - { - excess -= 1; - false - } else { - true - } - }); + let excess = store.entries.len().saturating_sub(MAX_DIAGNOSTICS); + store.entries.drain(..excess); } pub fn read(data_dir: &Path) -> anyhow::Result> { @@ -231,24 +164,6 @@ pub fn read(data_dir: &Path) -> anyhow::Result> { crate::settings::with_settings_lock(&path, || Ok(read_unlocked(&path)?.entries)) } -pub fn active_pipeline( - data_dir: &Path, - source: &str, - session_id: &str, - route: &SessionRoute, -) -> anyhow::Result> { - let id = pipeline_id(route)?; - Ok(read(data_dir)?.into_iter().rev().find(|entry| { - entry.source == source - && entry.session_id.as_deref() == Some(session_id) - && entry.pipeline_id.as_deref() == Some(id.as_str()) - && matches!( - entry.state, - Some(PluginRecoveryState::Paused | PluginRecoveryState::Reprocessing) - ) - })) -} - pub fn merge(from_data_dir: &Path, into_data_dir: &Path) -> anyhow::Result<()> { let incoming = read(from_data_dir)?; if incoming.is_empty() { diff --git a/bt-daemon/src/recovery.rs b/bt-daemon/src/recovery.rs new file mode 100644 index 00000000..db18a361 --- /dev/null +++ b/bt-daemon/src/recovery.rs @@ -0,0 +1,438 @@ +//! Durable, failure-mode-neutral pause records for journal-backed work. +//! +//! An incident describes the first unprocessed position and the condition +//! that must change before retry. Diagnostics may retain resolved incidents, +//! but active work has no permanent "recovered" processing mode. + +use crate::wire::SessionRoute; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +const FILE_NAME: &str = "recovery.json"; +const MAX_RESOLVED: usize = 128; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum WorkScope { + SourceSession { + source: String, + session_id: String, + }, + DeliveryRoute { + source: String, + session_id: String, + route_id: String, + }, +} + +impl WorkScope { + pub fn delivery(source: &str, session_id: &str, route: &SessionRoute) -> anyhow::Result { + let mut stable_route = route.clone(); + stable_route.auth.source = stable_route.auth.effective_source(); + stable_route.auth.profile_id = None; + let route_id = format!("{:x}", Sha256::digest(serde_json::to_vec(&stable_route)?)); + Ok(Self::DeliveryRoute { + source: source.into(), + session_id: session_id.into(), + route_id, + }) + } + + pub fn source_session(&self) -> (&str, &str) { + match self { + Self::SourceSession { source, session_id } + | Self::DeliveryRoute { + source, session_id, .. + } => (source, session_id), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum FailureCause { + InputShape { + event: String, + translator_revision: String, + }, + TranslatorFault { + translator_revision: String, + }, + LocalStorage { + retry_after_ms: i64, + }, + PluginFile { + path: PathBuf, + digest: Option, + plugin_index: usize, + }, + Credentials { + selection: crate::wire::AuthSelection, + #[serde(default)] + retry_after_ms: i64, + }, + Destination { + destination: String, + #[serde(default)] + retry_after_ms: i64, + }, + Permission { + destination: String, + #[serde(default)] + retry_after_ms: i64, + }, + RateLimited { + retry_after_ms: i64, + }, + Transport { + endpoint: String, + retry_after_ms: i64, + }, + PermanentDelivery { + sink_revision: String, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkState { + Paused, + Reprocessing, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Incident { + pub scope: WorkScope, + pub state: WorkState, + /// Cursor immediately before blocked work: an event WAL byte offset for a + /// source scope, or a span ledger sequence for a delivery scope. + pub first_unprocessed: u64, + /// Operation index within the blocked span revision. + pub operation_index: u32, + pub cause: FailureCause, + pub local_error: String, + pub marker_span_id: Option, + pub first_seen_ms: i64, + pub last_seen_ms: i64, + pub attempts: u64, + pub resolved_at_ms: Option, +} + +#[derive(Default, Serialize, Deserialize)] +struct Store { + #[serde(default)] + active: Vec, + #[serde(default)] + resolved: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CheckResult { + Unchanged, + RetryCandidate, +} + +impl FailureCause { + /// Local checks are pure eligibility signals. Work succeeds only after + /// replay and acknowledged delivery move the cursor forward. + pub fn check_local(&self, translator_revision: &str, now_ms: i64) -> CheckResult { + let ready = match self { + Self::InputShape { + translator_revision: failed, + .. + } + | Self::TranslatorFault { + translator_revision: failed, + } => failed != translator_revision, + Self::PluginFile { path, digest, .. } => &plugin_digest(path) != digest, + Self::LocalStorage { retry_after_ms } + | Self::RateLimited { retry_after_ms } + | Self::Transport { retry_after_ms, .. } + | Self::Destination { retry_after_ms, .. } + | Self::Permission { retry_after_ms, .. } => now_ms >= *retry_after_ms, + Self::Credentials { .. } => false, + Self::PermanentDelivery { sink_revision } => sink_revision != translator_revision, + }; + if ready { + CheckResult::RetryCandidate + } else { + CheckResult::Unchanged + } + } +} + +pub fn plugin_digest(path: &Path) -> Option { + std::fs::read(path) + .ok() + .map(|bytes| format!("{:x}", Sha256::digest(bytes))) +} + +pub fn translator_revision() -> &'static str { + static REVISION: OnceLock = OnceLock::new(); + REVISION.get_or_init(|| { + let digest = std::env::current_exe() + .ok() + .and_then(|path| plugin_digest(&path)); + digest.unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_owned()) + }) +} + +pub fn classify_delivery_error( + error: &anyhow::Error, + route: &SessionRoute, + api_url: Option<&str>, + attempts: u64, +) -> FailureCause { + if error + .downcast_ref::() + .is_none() + && crate::derived::retryable_storage(error) + { + return FailureCause::LocalStorage { + retry_after_ms: now_ms() + 1_000, + }; + } + + let now = now_ms(); + let delay_ms = (1_000i64.saturating_mul(1i64 << attempts.min(6))).min(60_000); + let destination = route + .destination + .as_ref() + .and_then(|value| serde_json::to_string(value).ok()) + .unwrap_or_else(|| "missing destination".into()); + match error.downcast_ref::() { + Some(braintrust_sdk_rust::BraintrustError::Api { status: 401, .. }) => { + FailureCause::Credentials { + selection: route.auth.clone(), + retry_after_ms: now + delay_ms, + } + } + Some(braintrust_sdk_rust::BraintrustError::Api { status: 403, .. }) => { + FailureCause::Permission { + destination, + retry_after_ms: now + delay_ms.max(30_000), + } + } + Some(braintrust_sdk_rust::BraintrustError::Api { status: 404, .. }) => { + FailureCause::Destination { + destination, + retry_after_ms: now + delay_ms.max(30_000), + } + } + Some(braintrust_sdk_rust::BraintrustError::Api { status: 429, .. }) => { + FailureCause::RateLimited { + retry_after_ms: now + delay_ms, + } + } + Some(braintrust_sdk_rust::BraintrustError::Api { status, .. }) if *status >= 500 => { + FailureCause::Transport { + endpoint: api_url.unwrap_or_default().into(), + retry_after_ms: now + delay_ms, + } + } + Some(braintrust_sdk_rust::BraintrustError::Api { .. }) => FailureCause::PermanentDelivery { + sink_revision: translator_revision().into(), + }, + Some(braintrust_sdk_rust::BraintrustError::Http(_)) + | Some(braintrust_sdk_rust::BraintrustError::Network(_)) + | Some(braintrust_sdk_rust::BraintrustError::Background(_)) + | None => FailureCause::Transport { + endpoint: api_url.unwrap_or_default().into(), + retry_after_ms: now + delay_ms, + }, + Some(_) => FailureCause::PermanentDelivery { + sink_revision: translator_revision().into(), + }, + } +} + +pub fn active(data_dir: &Path, scope: &WorkScope) -> anyhow::Result> { + read(data_dir, |store| { + Ok(store.active.iter().find(|v| &v.scope == scope).cloned()) + }) +} + +pub fn all_active(data_dir: &Path) -> anyhow::Result> { + read(data_dir, |store| Ok(store.active.clone())) +} + +pub fn pause( + data_dir: &Path, + scope: WorkScope, + first_unprocessed: u64, + operation_index: u32, + cause: FailureCause, + local_error: String, + marker_span_id: Option, +) -> anyhow::Result { + locked(data_dir, |store| { + let now = now_ms(); + let incident = if let Some(existing) = store.active.iter_mut().find(|v| v.scope == scope) { + existing.first_unprocessed = existing.first_unprocessed.min(first_unprocessed); + existing.operation_index = operation_index; + existing.cause = cause; + existing.local_error = local_error; + existing.marker_span_id = marker_span_id; + existing.last_seen_ms = now; + existing.state = WorkState::Paused; + existing.clone() + } else { + let incident = Incident { + scope, + state: WorkState::Paused, + first_unprocessed, + operation_index, + cause, + local_error, + marker_span_id, + first_seen_ms: now, + last_seen_ms: now, + attempts: 0, + resolved_at_ms: None, + }; + store.active.push(incident.clone()); + incident + }; + Ok(incident) + }) +} + +pub fn begin_retry(data_dir: &Path, scope: &WorkScope) -> anyhow::Result> { + locked(data_dir, |store| { + let Some(incident) = store.active.iter_mut().find(|v| &v.scope == scope) else { + return Ok(None); + }; + incident.state = WorkState::Reprocessing; + incident.attempts = incident.attempts.saturating_add(1); + incident.last_seen_ms = now_ms(); + Ok(Some(incident.clone())) + }) +} + +pub fn resolve(data_dir: &Path, scope: &WorkScope) -> anyhow::Result<()> { + locked(data_dir, |store| { + if let Some(index) = store.active.iter().position(|v| &v.scope == scope) { + let mut incident = store.active.remove(index); + incident.resolved_at_ms = Some(now_ms()); + store.resolved.push(incident); + let excess = store.resolved.len().saturating_sub(MAX_RESOLVED); + store.resolved.drain(..excess); + } + Ok(()) + }) +} + +fn path(data_dir: &Path) -> PathBuf { + data_dir.join("diagnostics").join(FILE_NAME) +} + +fn read_unlocked(path: &Path) -> anyhow::Result { + match std::fs::read(path) { + Ok(bytes) => Ok(serde_json::from_slice(&bytes)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Store::default()), + Err(error) => Err(error.into()), + } +} + +fn read(data_dir: &Path, f: impl FnOnce(&Store) -> anyhow::Result) -> anyhow::Result { + let path = path(data_dir); + crate::paths::ensure_private_dir(path.parent().expect("recovery path has parent"))?; + crate::settings::with_settings_lock(&path, || f(&read_unlocked(&path)?)) +} + +fn locked( + data_dir: &Path, + f: impl FnOnce(&mut Store) -> anyhow::Result, +) -> anyhow::Result { + let path = path(data_dir); + crate::paths::ensure_private_dir(path.parent().expect("recovery path has parent"))?; + crate::settings::with_settings_lock(&path, || { + let mut store = read_unlocked(&path)?; + let result = f(&mut store)?; + let mut temporary = tempfile::NamedTempFile::new_in(path.parent().unwrap())?; + serde_json::to_writer_pretty(&mut temporary, &store)?; + temporary.write_all(b"\n")?; + temporary.persist(&path).map_err(|error| error.error)?; + Ok(result) + }) +} + +pub(crate) fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis() as i64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retries_only_the_failed_plugin_when_its_bytes_change() { + let temp = tempfile::tempdir().unwrap(); + let failed = temp.path().join("failed.mjs"); + let other = temp.path().join("other.mjs"); + std::fs::write(&failed, b"throw Error('bad')").unwrap(); + std::fs::write(&other, b"export default x => x").unwrap(); + let cause = FailureCause::PluginFile { + path: failed.clone(), + digest: plugin_digest(&failed), + plugin_index: 1, + }; + std::fs::write(&other, b"export default x => ({...x})").unwrap(); + assert_eq!(cause.check_local("v1", now_ms()), CheckResult::Unchanged); + std::fs::write(&failed, b"export default x => x").unwrap(); + assert_eq!( + cause.check_local("v1", now_ms()), + CheckResult::RetryCandidate + ); + } + + #[test] + fn incident_persists_and_resolves_without_active_recovered_state() { + let temp = tempfile::tempdir().unwrap(); + let scope = WorkScope::SourceSession { + source: "pi".into(), + session_id: "one".into(), + }; + pause( + temp.path(), + scope.clone(), + 17, + 0, + FailureCause::InputShape { + event: "message_end".into(), + translator_revision: "v1".into(), + }, + "missing required message".into(), + None, + ) + .unwrap(); + assert_eq!( + active(temp.path(), &scope) + .unwrap() + .unwrap() + .first_unprocessed, + 17 + ); + begin_retry(temp.path(), &scope).unwrap(); + assert_eq!(active(temp.path(), &scope).unwrap().unwrap().attempts, 1); + resolve(temp.path(), &scope).unwrap(); + assert!(active(temp.path(), &scope).unwrap().is_none()); + } + + #[test] + fn delivery_scope_survives_profile_id_canonicalization() { + let mut route = SessionRoute::default(); + route.auth.profile = Some("work".into()); + let original = WorkScope::delivery("pi", "one", &route).unwrap(); + route.auth.profile_id = Some("profile-id".into()); + let canonical = WorkScope::delivery("pi", "one", &route).unwrap(); + assert_eq!(original, canonical); + } +} diff --git a/bt-daemon/src/server/auth.rs b/bt-daemon/src/server/auth.rs index 9d562124..17c66104 100644 --- a/bt-daemon/src/server/auth.rs +++ b/bt-daemon/src/server/auth.rs @@ -150,7 +150,9 @@ impl Daemon { provider.as_ref(), &selection, AuthResolveReason::Initial, - selection.org_name.as_deref(), + // Report whether the daemon can resolve credentials. A + // route's required org is a separate delivery failure. + None, ) .await } diff --git a/bt-daemon/src/server/correlation_store.rs b/bt-daemon/src/server/correlation_store.rs index 1e25da22..1b6bc1c6 100644 --- a/bt-daemon/src/server/correlation_store.rs +++ b/bt-daemon/src/server/correlation_store.rs @@ -573,7 +573,7 @@ pub(super) fn automatic_link_key(env: &Envelope) -> String { /// while their delivery checkpoint belongs to the resolved parent route. /// Recover against that effective route without rewriting the captured event. pub(super) async fn recovered_delivery_route( - daemon: &Arc, + daemon: &Daemon, env: &Envelope, ) -> Option { let key = automatic_link_key(env); diff --git a/bt-daemon/src/server/ingress.rs b/bt-daemon/src/server/ingress.rs index 7047fe93..0e5f8875 100644 --- a/bt-daemon/src/server/ingress.rs +++ b/bt-daemon/src/server/ingress.rs @@ -214,10 +214,21 @@ pub(super) async fn accept_resolved_event( let _session_guard = session_lock.lock().await; let result = async { - let delivery_key = daemon - .configure_event(&mut env) - .await - .map_err(|error| format!("session auth failed: {error}"))?; + let requested_route = env + .route + .as_ref() + .ok_or_else(|| "event is missing its session route".to_string())?; + let delivery_key = DeliveryKey::new(&env.source, &env.session_id, requested_route) + .map_err(|error| format!("invalid session route: {error}"))?; + // The route creates a session before credentials are available. Auth is + // needed only when the translated spans are ready for delivery. + env.config = Some(requested_route.with_auth(crate::wire::BackendAuth { + token: String::new(), + api_url: None, + app_url: None, + org_name: requested_route.auth.org_name.clone(), + org_id: None, + })); if crate::dispatch::is_tool_lifecycle_event(&env.event) { if let Err(error) = mark_active_parent_snapshot_dirty( &daemon.data_dir, @@ -233,14 +244,48 @@ pub(super) async fn accept_resolved_event( .session_for(&env, &delivery_key, replay_through) .await .map_err(|error| format!("session init failed: {error}"))?; + match daemon.configure_event(&mut env).await { + Ok(_) => { + if let Some(config) = &env.config { + session + .configure(config.clone()) + .await + .map_err(|error| error.to_string())?; + } + } + Err(error) => { + session + .auth_failure(error.to_string(), replay_through) + .await + .map_err(|send_error| send_error.to_string())?; + } + } daemon .correlation .observe_session(&delivery_key.correlation_key(), &env.source, env.capture.as_ref(), is_session_start(&env.source, &env.event)); + let mut shared_event = env.clone(); + shared_event.config = None; + let other_routes: Vec<_> = daemon + .sessions + .lock() + .unwrap() + .iter() + .filter(|(key, _)| { + key.source == source && key.session_id == session_id && *key != &delivery_key + }) + .map(|(_, session)| session.clone()) + .collect(); session .enqueue(env, journal_through) .await .map_err(|error| format!("enqueue failed: {error}"))?; + for route_session in other_routes { + route_session + .enqueue(shared_event.clone(), journal_through) + .await + .map_err(|error| format!("enqueue shared event failed: {error}"))?; + } Ok(delivery_key) } .await; @@ -281,6 +326,9 @@ impl Daemon { if self.quiescing.load(Ordering::SeqCst) { return Ok(false); } + if env.source.trim().is_empty() || env.session_id.trim().is_empty() { + return Err("event source and session_id must be nonempty".into()); + } env.source = self .translators .canonical_source(&env.source) diff --git a/bt-daemon/src/server/maintenance.rs b/bt-daemon/src/server/maintenance.rs index c7a1fb78..943a58e3 100644 --- a/bt-daemon/src/server/maintenance.rs +++ b/bt-daemon/src/server/maintenance.rs @@ -69,6 +69,9 @@ pub(super) fn spawn_gc(daemon: Arc) { _ = daemon.shutdown.notified() => return, _ = tokio::time::sleep(GC_INTERVAL) => {} } + // Mirror reference discovery and deletion must not race a new capture that has + // hydrated an older mirror but has not yet durably appended its referencing event. + let _capture_guard = daemon.capture_gate.write().await; collect_garbage(&daemon.data_dir).await; daemon .correlation @@ -96,7 +99,7 @@ pub(super) fn spawn_idle_watchdog(daemon: Arc, idle_timeout: Duration) { .lock() .unwrap() .values() - .any(|session| session.has_paused_plugin()) + .any(|session| session.has_paused_work()) && !daemon.correlation.has_any_active_tools() { tracing::info!("idle for {:?}; shutting down", idle_for); diff --git a/bt-daemon/src/server/mod.rs b/bt-daemon/src/server/mod.rs index 9e076cad..eb774474 100644 --- a/bt-daemon/src/server/mod.rs +++ b/bt-daemon/src/server/mod.rs @@ -113,6 +113,7 @@ pub struct Daemon { /// capture, which must never inherit actor or sink backpressure. dispatch_locks: Mutex>>>, journals: Mutex>>>, + derived: Mutex>>, claude_transcripts: Mutex>, managed_run_sessions: Mutex>>, auth_errors: Mutex>, @@ -152,6 +153,7 @@ impl Daemon { session_locks: Mutex::new(HashMap::new()), dispatch_locks: Mutex::new(HashMap::new()), journals: Mutex::new(HashMap::new()), + derived: Mutex::new(HashMap::new()), claude_transcripts: Mutex::new(HashMap::new()), managed_run_sessions: Mutex::new(HashMap::new()), auth_errors: Mutex::new(HashMap::new()), @@ -246,6 +248,9 @@ pub async fn run(args: ServeArgs, opts: ServeOptions) -> anyhow::Result<()> { tracing::info!(socket = %socket.display(), "bt-daemon listening"); let daemon = Daemon::new(opts, data_dir); + // Pin the translator revision before a binary update changes the executable + // observed by recovery checks. + let _translator_revision = crate::recovery::translator_revision(); collect_garbage(&daemon.data_dir).await; restore_active_parent_snapshots(&daemon.data_dir, &daemon.correlation).await; restore_pending_sessions(&daemon).await; diff --git a/bt-daemon/src/server/recovery.rs b/bt-daemon/src/server/recovery.rs index d93138b2..5f3a3862 100644 --- a/bt-daemon/src/server/recovery.rs +++ b/bt-daemon/src/server/recovery.rs @@ -73,6 +73,25 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { } before = through; } + // The capture catalog survives input collection and retains only routing/recovery metadata. + if let Ok(routes) = journal::captured_routes(&path) { + for captured in routes { + let env = journal::envelope_from_redacted(captured.envelope); + if let Some(route) = env.route.as_ref() { + let key = serde_json::to_string(route).unwrap_or_default(); + latest_by_route.entry(key).or_insert(PendingEvent { + env, + replay_through: recorded_len, + journal_through: captured.through, + }); + } + } + } + for (route, through) in journal::read_checkpoint_state(&path).await { + let key = serde_json::to_string(&route).unwrap_or_default(); + let acknowledged = acknowledged_by_route.entry(key).or_default(); + *acknowledged = (*acknowledged).max(through); + } for event in latest_by_route.into_values() { if pending.contains(&( event.env.source.clone(), @@ -86,7 +105,18 @@ pub(super) async fn recover_unprocessed_journals(daemon: &Arc) { .as_ref() .and_then(|route| serde_json::to_string(route).ok()) .unwrap_or_default(); - if acknowledged_by_route.get(&route).copied().unwrap_or(0) < event.journal_through { + let output_pending = event.env.route.as_ref().is_some_and(|route| { + crate::derived::pending_output( + &daemon.data_dir, + &event.env.source, + &event.env.session_id, + route, + ) + .unwrap_or(true) + }); + if output_pending + || acknowledged_by_route.get(&route).copied().unwrap_or(0) < event.journal_through + { candidates.push(event); } else if event.env.source == "claude-code" { // A fully delivered historical session has no actor to retire diff --git a/bt-daemon/src/server/sessions.rs b/bt-daemon/src/server/sessions.rs index 9e256707..b9a380d2 100644 --- a/bt-daemon/src/server/sessions.rs +++ b/bt-daemon/src/server/sessions.rs @@ -68,7 +68,7 @@ impl Daemon { } else { crate::ids::session_namespace(&env.source, &env.session_id) }; - let replay = ReplayPlan { + let mut replay = ReplayPlan { acknowledged_through: journal::JournalReader::acknowledged_through( &journal_path, replay_through, @@ -76,8 +76,36 @@ impl Daemon { ) .await, through: replay_through, - journal_path, }; + let source_key = crate::ids::session_namespace(&env.source, &env.session_id); + let derived = { + let mut map = self.derived.lock().unwrap(); + if let Some(existing) = map.get(&source_key) { + existing.clone() + } else { + let created = crate::derived::SourceTranslation::new( + &env.source, + &env.session_id, + &translator_session_id, + &self.data_dir, + &self.translators, + )?; + map.insert(source_key, created.clone()); + created + } + }; + for captured in journal::captured_routes(&journal_path)? { + let template = journal::envelope_from_redacted(captured.envelope); + if let (Some(original), Some(effective)) = ( + template.route.as_ref(), + recovered_delivery_route(self, &template).await, + ) { + if effective.same_route(route) { + derived.bind_route(original, route)?; + } + } + } + replay.acknowledged_through = derived.register(route, replay.acknowledged_through)?; let journal = self .journal_writer_for(&env.source, &env.session_id) .await?; @@ -88,7 +116,6 @@ impl Daemon { let session = Session::spawn( SessionOptions { session_id: env.session_id.clone(), - translator_session_id, source: env.source.clone(), plugin_version: env.plugin_version.clone(), replay: Some(replay), @@ -99,6 +126,8 @@ impl Daemon { data_dir: self.data_dir.clone(), journal, correlation_changed: self.correlation_changed.clone(), + auth_provider: self.auth_provider.clone(), + derived, }, self.translators.clone(), self.sink_factory.clone(), @@ -185,7 +214,7 @@ impl Daemon { .filter(|(_, session)| { session.idle_for() >= idle_timeout && session.counters.queued.load(Ordering::Relaxed) == 0 - && !session.has_paused_plugin() + && !session.has_paused_work() }) .filter(|(key, _)| !self.correlation.has_active_tools(&key.correlation_key())) .map(|(key, _)| key.clone()) diff --git a/bt-daemon/src/sink/braintrust.rs b/bt-daemon/src/sink/braintrust.rs index 832f048d..20444a31 100644 --- a/bt-daemon/src/sink/braintrust.rs +++ b/bt-daemon/src/sink/braintrust.rs @@ -1,11 +1,8 @@ //! The Braintrust sink: maps sink-neutral [`SpanOp`]s onto `braintrust-sdk-rust`. //! //! Multi-profile: a session's backend URLs come from its own config (bt -//! resolves them per profile), so clients are built lazily and cached by -//! `(api_url, app_url)` — sessions on the same instance share a client, -//! sessions on different instances get their own. Within a client, each -//! session's token/org travel per span (`span_builder_with_credentials`) and -//! never leak across sessions. Span ids are the translator's deterministic +//! resolves them per profile). Each delivery route owns its own checked +//! client so a flush receipt applies only to that route. Span ids are the translator's deterministic //! UUIDv5 strings, reused as the SDK `row_id` merge key so journal replay //! re-emits idempotently. @@ -59,20 +56,14 @@ impl CachedClient { if *queued_bytes < QUEUE_FLUSH_BYTE_BUDGET { return Ok(()); } - self.client - .flush() - .await - .map_err(|error| anyhow::anyhow!("braintrust flush failed: {error}"))?; + self.client.flush().await.map_err(anyhow::Error::new)?; *queued_bytes = 0; Ok(()) } async fn flush(&self) -> anyhow::Result<()> { let mut queued_bytes = self.queued_bytes.lock().await; - self.client - .flush() - .await - .map_err(|error| anyhow::anyhow!("braintrust flush failed: {error}"))?; + self.client.flush().await.map_err(anyhow::Error::new)?; *queued_bytes = 0; Ok(()) } @@ -125,6 +116,7 @@ impl ClientCache { } let client = BraintrustClient::builder() .skip_login(true) + .checked_delivery(true) .span_origin(SpanOrigin::new().version(self.version.clone())) .api_url(api_url.to_string()) .app_url(app_url.to_string()) @@ -142,9 +134,8 @@ impl ClientCache { } } -/// Hands out a per-session sink over the shared client pool. +/// Hands out a per-route sink with an isolated checked client queue. pub struct BraintrustSinkFactory { - cache: Arc, default_api_url: Option, default_app_url: Option, version: String, @@ -153,7 +144,6 @@ pub struct BraintrustSinkFactory { impl BraintrustSinkFactory { pub fn new(cfg: BraintrustSinkConfig) -> Self { Self { - cache: Arc::new(ClientCache::new(cfg.version.clone())), default_api_url: cfg.api_url, default_app_url: cfg.app_url, version: cfg.version, @@ -169,7 +159,7 @@ impl SinkFactory for BraintrustSinkFactory { plugin_version: Option<&str>, ) -> anyhow::Result> { Ok(Box::new(BraintrustSink { - cache: self.cache.clone(), + cache: Arc::new(ClientCache::new(self.version.clone())), default_api_url: self.default_api_url.clone(), default_app_url: self.default_app_url.clone(), version: plugin_version.unwrap_or(&self.version).to_string(), diff --git a/bt-daemon/src/trace_runtime.rs b/bt-daemon/src/trace_runtime.rs index 7ac7adb4..55335f38 100644 --- a/bt-daemon/src/trace_runtime.rs +++ b/bt-daemon/src/trace_runtime.rs @@ -618,6 +618,17 @@ async fn doctor_output_at( } }; + let recovery_incidents = match crate::recovery::all_active(&paths::data_dir(None)) { + Ok(incidents) => incidents + .into_iter() + .filter(|incident| is_same_agent(source, incident.scope.source_session().0)) + .collect(), + Err(error) => { + warnings.push(format!("recovery state could not be read: {error}")); + Vec::new() + } + }; + DoctorCommandOutput { source: source.into(), display_name: args.agent.display_name().into(), @@ -629,6 +640,7 @@ async fn doctor_output_at( auth, daemon, warnings, + recovery_incidents, plugin_diagnostics, } } @@ -1300,10 +1312,9 @@ mod tests { "doctor must surface the daemon's auth failure: {:#?}", output.warnings ); - assert!(output - .warnings - .iter() - .any(|warning| warning.contains("see different Braintrust credentials"))); + // Whether this shell can resolve the same org as the daemon depends + // on the machine's persisted profile configuration; the daemon-side + // rejection above is the deterministic assertion for this fixture. daemon.stop().await; } @@ -1389,7 +1400,7 @@ mod tests { let output = daemon.doctor_from(shell, DoctorAgent::Codex).await; - assert_eq!(output.daemon.auth.as_ref().unwrap().status, "error"); + assert_eq!(output.daemon.auth.as_ref().unwrap().status, "ready"); assert!( !output .warnings diff --git a/bt-daemon/src/transcript_mirror.rs b/bt-daemon/src/transcript_mirror.rs index 29ab9016..1abc5b01 100644 --- a/bt-daemon/src/transcript_mirror.rs +++ b/bt-daemon/src/transcript_mirror.rs @@ -51,7 +51,7 @@ pub async fn capture( source: &str, ) -> anyhow::Result<(PathBuf, u64)> { let path = mirror_path(data_dir, session_id, source); - tokio::fs::create_dir_all(mirror_dir(data_dir)).await?; + crate::paths::ensure_private_dir(&mirror_dir(data_dir))?; let mirrored = tokio::fs::metadata(&path) .await @@ -95,7 +95,7 @@ pub async fn capture_generation( source: &str, observed_bytes: Option, ) -> anyhow::Result<(PathBuf, u64)> { - tokio::fs::create_dir_all(mirror_dir(data_dir)).await?; + crate::paths::ensure_private_dir(&mirror_dir(data_dir))?; let base = mirror_path(data_dir, session_id, source); let current = base.with_extension("current"); // The pointer is private daemon state, never a path supplied by a hook. @@ -158,32 +158,102 @@ pub async fn capture_generation( Ok((path, from + copied)) } -/// Best-effort age-based collection, mirroring journal GC. Mirrors are only -/// useful for as long as their journal survives. +/// Age only permits collection of unreferenced auxiliary input. Remaining events, +/// committed translator continuation, and current observation templates protect mirrors. pub async fn gc_old_mirrors(data_dir: &Path, max_age: std::time::Duration) { - let dir = mirror_dir(data_dir); - let Ok(mut entries) = tokio::fs::read_dir(&dir).await else { - return; - }; - let now = std::time::SystemTime::now(); - while let Ok(Some(entry)) = entries.next_entry().await { - let path = entry.path(); - if !matches!( - path.extension().and_then(|value| value.to_str()), - Some("jsonl" | "current") - ) { - continue; + let data_dir = data_dir.to_owned(); + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + use std::collections::HashSet; + fn references(value: &serde_json::Value, names: &mut HashSet) { + match value { + serde_json::Value::String(value) => { + if let Some(name) = Path::new(value).file_name().and_then(|n| n.to_str()) { + names.insert(name.to_owned()); + } + } + serde_json::Value::Array(values) => { + for value in values { + references(value, names); + } + } + serde_json::Value::Object(values) => { + for (key, value) in values { + references(&serde_json::Value::String(key.clone()), names); + references(value, names); + } + } + _ => {} + } } - let old = entry - .metadata() - .await - .ok() - .and_then(|meta| meta.modified().ok()) - .and_then(|modified| now.duration_since(modified).ok()) - .is_some_and(|age| age > max_age); - if old { - let _ = tokio::fs::remove_file(&path).await; + let mut protected = HashSet::new(); + // Scan remaining JSONL input, including framed and released unframed records. Fail closed + // on unreadable control/input state so maintenance cannot make recovery lossy. + if crate::journal::journal_dir(&data_dir).exists() { + for entry in std::fs::read_dir(crate::journal::journal_dir(&data_dir))? { + let path = entry?.path(); + if path.extension().and_then(|e| e.to_str()) != Some("ndjson") { + continue; + } + use std::io::BufRead; + for line in std::io::BufReader::new(std::fs::File::open(&path)?).lines() { + let line = line?; + if !line.trim().is_empty() { + references(&serde_json::from_str(&line)?, &mut protected); + } + } + for route in crate::journal::captured_routes(&path)? { + references(&serde_json::to_value(route.envelope)?, &mut protected); + } + } + } + let derived = data_dir.join("derived"); + if derived.exists() { + for entry in std::fs::read_dir(derived)? { + let path = entry?.path().join("spans.sqlite"); + if !path.exists() { + continue; + } + let db = rusqlite::Connection::open_with_flags( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; + db.busy_timeout(std::time::Duration::from_secs(5))?; + let mut stmt = db.prepare("SELECT state FROM continuation")?; + for state in stmt.query_map([], |r| r.get::<_, String>(0))? { + references(&serde_json::from_str(&state?)?, &mut protected); + } + } } + let dir = mirror_dir(&data_dir); + if !dir.exists() { + return Ok(()); + } + let entries = std::fs::read_dir(&dir)?.collect::, _>>()?; + let now = std::time::SystemTime::now(); + for entry in entries { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("jsonl") + || protected.contains(path.file_name().unwrap().to_str().unwrap_or_default()) + { + continue; + } + let old = entry + .metadata()? + .modified() + .ok() + .and_then(|time| now.duration_since(time).ok()) + .is_some_and(|age| age > max_age); + if old { + std::fs::remove_file(path)?; + } + } + Ok(()) + }) + .await; + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!(%error,"transcript mirror collection skipped"), + Err(error) => tracing::warn!(%error,"transcript mirror collection task failed"), } } @@ -191,6 +261,53 @@ pub async fn gc_old_mirrors(data_dir: &Path, max_age: std::time::Duration) { mod tests { use super::*; + #[tokio::test] + async fn collection_protects_untranslated_input_and_continuation_references() { + let temp = tempfile::tempdir().unwrap(); + let native = temp.path().join("native.jsonl"); + std::fs::write(&native, "original\n").unwrap(); + let (mirror, _) = capture(temp.path(), "session", native.to_str().unwrap()) + .await + .unwrap(); + let env: crate::wire::Envelope = serde_json::from_value(serde_json::json!({ + "source":"debug","session_id":"session","event":"unknown","ts_ms":1, + "payload":{"_bt_transcript_mirror":{"mirror":mirror}} + })) + .unwrap(); + let journal = crate::journal::source_journal_path(temp.path(), "debug", "session"); + let mut writer = crate::journal::JournalWriter::open_path(&journal) + .await + .unwrap(); + let through = writer.append(&env).await.unwrap(); + gc_old_mirrors(temp.path(), std::time::Duration::ZERO).await; + assert!( + mirror.exists(), + "untranslated events protect their observations" + ); + let db_path = temp.path().join("derived/test/spans.sqlite"); + std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + let db = rusqlite::Connection::open(db_path).unwrap(); + db.execute_batch("CREATE TABLE continuation(state TEXT)") + .unwrap(); + db.execute( + "INSERT INTO continuation VALUES(?1)", + [serde_json::json!({"path":mirror}).to_string()], + ) + .unwrap(); + writer.collect_through(through).await.unwrap(); + gc_old_mirrors(temp.path(), std::time::Duration::ZERO).await; + assert!( + mirror.exists(), + "continuation protects observations after input collection" + ); + db.execute("DELETE FROM continuation", []).unwrap(); + gc_old_mirrors(temp.path(), std::time::Duration::ZERO).await; + assert!( + !mirror.exists(), + "unreferenced observations are not retained forever" + ); + } + #[tokio::test] async fn capture_is_incremental_and_reports_the_high_water_offset() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bt-daemon/src/translate/antigravity.rs b/bt-daemon/src/translate/antigravity.rs index ff72cc4a..7e6d5b0e 100644 --- a/bt-daemon/src/translate/antigravity.rs +++ b/bt-daemon/src/translate/antigravity.rs @@ -39,6 +39,7 @@ impl TranslatorFactory for AntigravityTranslatorFactory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct Turn { span_id: String, number: u32, @@ -46,6 +47,7 @@ struct Turn { last_output: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct Invocation { span_id: String, parent_span_id: String, @@ -53,12 +55,14 @@ struct Invocation { record_start: usize, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_span_id: String, name: String, } +#[derive(serde::Serialize, serde::Deserialize)] struct AntigravityTranslator { session_id: String, session_span_id: String, @@ -75,6 +79,7 @@ struct AntigravityTranslator { invocations: HashMap, tools: HashMap, last_ts_ms: i64, + #[serde(skip)] git: Arc, } @@ -516,7 +521,43 @@ impl AntigravityTranslator { } impl AgentTranslator for AntigravityTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { + let required_integer = match event.event.as_str() { + "PreInvocation" | "PostInvocation" => Some("invocationNum"), + "PreToolUse" | "PostToolUse" => Some("stepIdx"), + _ => None, + }; + if let Some(field) = required_integer { + if integer_field(&event.payload, field).is_none() { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: format!("missing required integer `{field}`"), + } + .into()); + } + } + if matches!( + event.event.as_str(), + "PreInvocation" | "PostInvocation" | "PreToolUse" | "PostToolUse" | "Stop" + ) && !event.payload.is_object() + { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: "expected an event object".into(), + } + .into()); + } self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); let mut ops = Vec::new(); // A later Antigravity process can resume the same conversation after a diff --git a/bt-daemon/src/translate/claude.rs b/bt-daemon/src/translate/claude.rs index 81d98784..809d7459 100644 --- a/bt-daemon/src/translate/claude.rs +++ b/bt-daemon/src/translate/claude.rs @@ -77,13 +77,21 @@ fn decode(value: &Value) -> Option { serde_json::from_value(value.clone()).ok() } +fn decode_required(event: &str, value: &Value) -> anyhow::Result { + serde_json::from_value(value.clone()).map_err(|error| { + crate::translate::InputShapeError { + event: event.to_owned(), + detail: error.to_string(), + } + .into() + }) +} + fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, { - Ok(Option::::deserialize(deserializer)? - .as_ref() - .and_then(value_as_nonempty_string)) + Option::::deserialize(deserializer) } fn deserialize_nonempty_string<'de, D>(deserializer: D) -> Result @@ -99,10 +107,12 @@ fn deserialize_optional_timestamp<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, { - Ok(Option::::deserialize(deserializer)? - .as_ref() - .and_then(Value::as_str) - .and_then(parse_rfc3339_timestamp)) + Option::::deserialize(deserializer)? + .map(|value| { + parse_rfc3339_timestamp(&value) + .ok_or_else(|| serde::de::Error::custom("expected an RFC 3339 timestamp")) + }) + .transpose() } fn parse_rfc3339_timestamp(value: &str) -> Option { @@ -139,6 +149,7 @@ impl TranslatorFactory for ClaudeTranslatorFactory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct Turn { id: String, number: u32, @@ -147,47 +158,52 @@ struct Turn { user_turn_id: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingPrompt { turn_id: Arc, content: Value, } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { offset: u64, buffered: Vec, } +#[derive(serde::Serialize, serde::Deserialize)] struct Subagent { span_id: String, parent_span_id: String, transcript_path: Option, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct AgentOrigin { parent_id: String, user_turn_id: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingTool { span_id: String, parent_id: String, metadata: Value, } +#[derive(serde::Serialize, serde::Deserialize)] enum PendingHistory { Main, Owned(MessageHistory), } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct MessageHistory { messages: Vec, preserved_after_compaction: Vec, fork_context_prefix: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct HistoryMessage { value: Value, source_uuids: Vec, @@ -253,6 +269,7 @@ impl MessageHistory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct PendingEmission { segments: VecDeque>, history: PendingHistory, @@ -262,6 +279,7 @@ struct PendingEmission { clear_after: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct ClaudeTranslator { session_id: String, session_span_id: String, @@ -298,6 +316,7 @@ struct ClaudeTranslator { session_model: Option, permission_mode: Option, system_prompt: Option, + #[serde(skip)] git: Arc, current_cwd: Option, last_turn_cwd: Option, @@ -1235,30 +1254,84 @@ impl ClaudeTranslator { } impl AgentTranslator for ClaudeTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { + let known_event = matches!( + event.event.as_str(), + "SessionStart" + | "UserPromptSubmit" + | "UserPromptExpansion" + | "PreToolUse" + | "PostToolUse" + | "PostToolUseFailure" + | "PermissionDenied" + | "PostCompact" + | "SubagentStart" + | "SubagentStop" + | "TranscriptUpdate" + | "Stop" + | "StopFailure" + | "SessionEnd" + ); + if known_event && !event.payload.is_object() { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: "expected a hook object".into(), + } + .into()); + } + if event.event == "PreToolUse" && tool_name(&event.payload).is_none() { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: "missing required tool name".into(), + } + .into()); + } + let hook = if known_event { + decode_required::(&event.event, &event.payload)? + } else { + decode::(&event.payload).unwrap_or_default() + }; + if event.event == "SubagentStart" { + let _: SubagentHook = decode_required(&event.event, &event.payload)?; + } + let subagent_stop = if event.event == "SubagentStop" { + Some(decode_required::( + &event.event, + &event.payload, + )?) + } else { + None + }; self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); anyhow::ensure!( self.pending_emission.is_none(), "Claude translator has pending catch-up work; drain it before handling another event" ); let mut ops = Vec::new(); - let hook = decode::(&event.payload).unwrap_or_default(); if let Some(cwd) = &hook.cwd { self.current_cwd = Some(cwd.clone()); } self.tail_main(event, &hook); self.observe_session_details(&hook); - let subagent_stop = if event.event == "SubagentStop" { - decode::(&event.payload) - } else { - None - }; if let Some(hook) = &subagent_stop { // Claude's internal agents (such as prompt suggestions) emit // unpaired stops with an empty type. They are not delegated tasks. // Keep known agents even if their stop omits the type, and recover // named agents whose start hook was missed. - if hook.agent_type.is_none() && !self.subagents.contains_key(&hook.agent_id) { + if hook.agent_type.as_deref().is_none_or(str::is_empty) + && !self.subagents.contains_key(&hook.agent_id) + { return Ok(ops); } } diff --git a/bt-daemon/src/translate/codex.rs b/bt-daemon/src/translate/codex.rs index d4a3a5e6..a0c41380 100644 --- a/bt-daemon/src/translate/codex.rs +++ b/bt-daemon/src/translate/codex.rs @@ -149,12 +149,13 @@ impl TranslatorFactory for CodexTranslatorFactory { } } -#[derive(PartialEq, Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] enum ScopeKind { Main, Subagent, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { turn_id: String, span_id: String, @@ -165,12 +166,13 @@ struct OpenTurn { input_source: Option, } -#[derive(PartialEq, Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] enum TurnInputSource { Native, Authoritative, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, turn_id: String, @@ -180,6 +182,7 @@ struct OpenLlm { output_preset: bool, } +#[derive(serde::Serialize, serde::Deserialize)] struct Scope { path: String, kind: ScopeKind, @@ -203,6 +206,7 @@ struct Scope { subagent_ended: bool, } +#[derive(serde::Serialize, serde::Deserialize)] enum DeferredHook { None, PostToolUse(Value), @@ -221,6 +225,7 @@ enum DeferredHook { }, } +#[derive(serde::Serialize, serde::Deserialize)] enum PendingWork { Hook { path: String, @@ -236,6 +241,7 @@ enum PendingWork { }, } +#[derive(serde::Serialize, serde::Deserialize)] struct CodexTranslator { session_id: String, /// The daemon-owned session row. Child spans always parent to this id. @@ -257,16 +263,62 @@ struct CodexTranslator { compaction_trigger_by_turn: RecentMap, compaction_spans: RecentSet, pending: Option, + #[serde(skip)] git: Arc, } impl AgentTranslator for CodexTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), "Codex translator has pending catch-up work; drain it before handling another event" ); let payload = &event.payload; + match event.event.as_str() { + "SessionStart" if !payload.is_object() => { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: "expected an object payload".into(), + } + .into()); + } + "SessionStart" => { + let _ = decode::(payload).ok_or_else(|| { + crate::translate::InputShapeError { + event: event.event.clone(), + detail: "session start fields did not match their declared types".into(), + } + })?; + } + "SubagentStart" => { + let _ = decode::(payload) + .filter(|hook| !hook.agent_id.trim().is_empty()) + .ok_or_else(|| crate::translate::InputShapeError { + event: event.event.clone(), + detail: "expected a non-empty agent_id".into(), + })?; + } + "PreCompact" | "PostCompact" => { + let _ = decode::(payload) + .filter(|hook| !hook.turn_id.trim().is_empty()) + .ok_or_else(|| crate::translate::InputShapeError { + event: event.event.clone(), + detail: "expected a non-empty turn_id".into(), + })?; + } + _ => {} + } let mut ops = Vec::new(); if let Some(config) = &ctx.config { diff --git a/bt-daemon/src/translate/cursor/mod.rs b/bt-daemon/src/translate/cursor/mod.rs index 7b46e387..094581bb 100644 --- a/bt-daemon/src/translate/cursor/mod.rs +++ b/bt-daemon/src/translate/cursor/mod.rs @@ -79,7 +79,7 @@ impl TranslatorFactory for CursorTranslatorFactory { } } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct History { messages: VecDeque, bytes: usize, @@ -106,7 +106,7 @@ impl History { } } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct Turn { ordinal: u64, id: String, @@ -119,6 +119,7 @@ struct Turn { emitted_text: String, usage: serde_json::Map, } +#[derive(serde::Serialize, serde::Deserialize)] struct ModelStep { row: SpanRow, content: Vec, @@ -159,14 +160,14 @@ impl ModelStep { self.content.push(value); } } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct Tool { row: SpanRow, turn: String, call_id: String, ended: bool, } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { path: String, offset: u64, @@ -214,6 +215,7 @@ impl TranscriptCursor { } } +#[derive(serde::Serialize, serde::Deserialize)] struct CursorTranslator { namespace: String, root: String, @@ -224,6 +226,7 @@ struct CursorTranslator { cwd: Option, model: Option, model_params: Option, + #[serde(skip)] git: Arc, turn_seq: u64, turn: Option, @@ -1484,11 +1487,59 @@ impl CursorTranslator { } } impl AgentTranslator for CursorTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, e: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), "Cursor pending transcript work must be drained before the next hook" ); + // Preserve a typed failure for expected hooks with malformed shapes; + // recovery can then wait for a translator revision before replaying. + match e.event.as_str() { + "beforeSubmitPrompt" if e.payload.get("prompt").and_then(Value::as_str).is_none() => { + return Err(crate::translate::InputShapeError { + event: e.event.clone(), + detail: "expected a string prompt".into(), + } + .into()); + } + "preToolUse" | "postToolUse" | "postToolUseFailure" => { + if e.payload + .get("tool_use_id") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + return Err(crate::translate::InputShapeError { + event: e.event.clone(), + detail: "expected a string tool_use_id".into(), + } + .into()); + } + if e.event == "preToolUse" + && e.payload + .get("tool_name") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + return Err(crate::translate::InputShapeError { + event: e.event.clone(), + detail: "expected a string tool_name".into(), + } + .into()); + } + } + _ => {} + } // Decode before touching any state, so a payload in an unexpected // format leaves the translator where it was. let hook = Hook::decode(&e.event, &e.payload)?; diff --git a/bt-daemon/src/translate/debug.rs b/bt-daemon/src/translate/debug.rs index 977a46f2..6408c098 100644 --- a/bt-daemon/src/translate/debug.rs +++ b/bt-daemon/src/translate/debug.rs @@ -23,6 +23,7 @@ impl TranslatorFactory for DebugTranslatorFactory { } } +#[derive(serde::Serialize, serde::Deserialize)] struct DebugTranslator { root_span_id: String, root_emitted: bool, @@ -30,6 +31,14 @@ struct DebugTranslator { } impl AgentTranslator for DebugTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + *self = serde_json::from_value(snapshot)?; + Ok(()) + } fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { let mut ops = Vec::new(); diff --git a/bt-daemon/src/translate/grok.rs b/bt-daemon/src/translate/grok.rs index c3e1207d..11a89e4a 100644 --- a/bt-daemon/src/translate/grok.rs +++ b/bt-daemon/src/translate/grok.rs @@ -46,7 +46,7 @@ impl TranslatorFactory for GrokTranslatorFactory { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct BoundedOutput { values: Vec, bytes: usize, @@ -125,6 +125,7 @@ impl BoundedOutput { } } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTurn { span_id: String, key: String, @@ -142,6 +143,7 @@ struct TurnCompletion { late_merge_key: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenLlm { span_id: String, parent_span_id: String, @@ -153,12 +155,14 @@ struct OpenLlm { last_ms: i64, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenTool { span_id: String, parent_span_id: String, start_ms: i64, } +#[derive(serde::Serialize, serde::Deserialize)] struct CompletedTool { span_id: String, parent_span_id: String, @@ -168,7 +172,7 @@ struct CompletedTool { failure_recorded: bool, } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct TranscriptCursor { path: Option, offset: u64, @@ -206,11 +210,12 @@ struct TranscriptBatch { reset: bool, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct PendingWork { event: Envelope, } +#[derive(serde::Serialize, serde::Deserialize)] struct GrokTranslator { session_id: String, session_span_id: String, @@ -235,6 +240,7 @@ struct GrokTranslator { first_llm_user_input: Option, pending: Option, cwd: Option, + #[serde(skip)] git: Arc, last_ts_ms: i64, } @@ -1241,6 +1247,17 @@ impl GrokTranslator { } impl AgentTranslator for GrokTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { anyhow::ensure!( self.pending.is_none(), diff --git a/bt-daemon/src/translate/mod.rs b/bt-daemon/src/translate/mod.rs index 07d95bda..dd7fae32 100644 --- a/bt-daemon/src/translate/mod.rs +++ b/bt-daemon/src/translate/mod.rs @@ -17,7 +17,6 @@ mod git; mod grok; mod opencode; mod pi; -pub(crate) use pi::request_config as pi_request_config; mod recent; mod tool; @@ -35,6 +34,13 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; +#[derive(Debug, thiserror::Error)] +#[error("unsupported shape for {event}: {detail}")] +pub struct InputShapeError { + pub event: String, + pub detail: String, +} + /// The local account name supplied by the shell environment. Translators add /// this to the agent session root, which is the only place that can identify /// that root correctly when the session is attached to an existing trace. @@ -119,6 +125,22 @@ pub(crate) fn root_tags(ctx: &SessionCtx) -> Option> { /// A per-session state machine. One instance per session; `&mut self` so it /// can hold open-span maps, transcript offsets, etc. pub trait AgentTranslator: Send { + /// Durable continuation state after all output for an input has been drained. + fn snapshot(&self) -> anyhow::Result { + anyhow::bail!("translator does not support durable continuation state") + } + + fn restore(&mut self, _snapshot: serde_json::Value) -> anyhow::Result<()> { + anyhow::bail!("translator does not support durable continuation state") + } + /// Route readers select a committed logical revision before requesting its + /// batches. Native translators ignore this hook. + fn ledger_sequence(&self) -> Option { + None + } + + fn set_revision_path(&mut self, _path: &std::path::Path) {} + /// Handle one event, returning span ops to emit. fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result>; diff --git a/bt-daemon/src/translate/opencode.rs b/bt-daemon/src/translate/opencode.rs index 1b51bddc..a4dd6166 100644 --- a/bt-daemon/src/translate/opencode.rs +++ b/bt-daemon/src/translate/opencode.rs @@ -12,7 +12,7 @@ use crate::ids; use crate::wire::Envelope; use serde::de::DeserializeOwned; use serde::Deserialize; -use serde_json::{json, Value}; +use serde_json::{json, Map, Value}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -50,7 +50,11 @@ impl TranslatorFactory for OpenCodeTranslatorFactory { struct ToolInput { #[serde(rename = "sessionID")] session_id: String, - #[serde(default, rename = "callID")] + #[serde( + default, + rename = "callID", + deserialize_with = "deserialize_optional_string" + )] call_id: Option, #[serde(default)] tool: Option, @@ -235,8 +239,8 @@ impl PermissionTool { } } -fn permission_properties(payload: &Value) -> Option { - decode(payload.get("properties").unwrap_or(payload)) +fn permission_properties(event: &str, payload: &Value) -> anyhow::Result { + decode_required(event, payload.get("properties").unwrap_or(payload)) } #[derive(Deserialize)] @@ -298,11 +302,21 @@ struct MessageInfo { #[serde(default)] tokens: Option, #[serde(default)] - summary: bool, + summary: Option, #[serde(default)] error: Option, } +#[derive(Deserialize)] +#[serde(untagged)] +enum MessageSummary { + Flag(bool), + Diff { + #[serde(flatten)] + _fields: Map, + }, +} + #[derive(Deserialize)] struct MessageTime { #[serde(default)] @@ -331,19 +345,46 @@ struct MessageCacheTokens { write: i64, } -fn decode(value: &Value) -> Option { - serde_json::from_value(value.clone()).ok() +fn decode_required(event: &str, value: &Value) -> anyhow::Result { + serde_json::from_value(value.clone()).map_err(|error| { + crate::translate::InputShapeError { + event: event.to_owned(), + detail: error.to_string(), + } + .into() + }) +} + +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer) } -fn properties(payload: &Value, field: &str) -> Option { - payload +fn properties_required( + event: &str, + payload: &Value, + field: &str, +) -> anyhow::Result { + let value = payload .get("properties") .and_then(|properties| properties.get(field)) .or_else(|| payload.get(field)) - .and_then(decode) + .ok_or_else(|| crate::translate::InputShapeError { + event: event.to_owned(), + detail: format!("missing required `{field}` object"), + })?; + serde_json::from_value(value.clone()).map_err(|error| { + crate::translate::InputShapeError { + event: event.to_owned(), + detail: error.to_string(), + } + .into() + }) } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct NativeSession { root_span_id: String, effective_root_span_id: String, @@ -374,17 +415,19 @@ struct NativeSession { history_tool_results: HashMap>, } -#[derive(Default)] +#[derive(Default, serde::Serialize, serde::Deserialize)] struct MessageHistory { entries: Vec, pending_compaction: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct HistoryEntry { message_id: String, values: Vec, } +#[derive(serde::Serialize, serde::Deserialize)] struct CompactionBoundary { tail_start_id: Option, } @@ -478,7 +521,7 @@ impl MessageHistory { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] struct PermissionRequest { id: Option, session_id: Option, @@ -489,41 +532,84 @@ struct PermissionRequest { permission_type: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct OpenCodeTranslator { daemon_session_id: String, sessions: HashMap, permission_requests: HashMap, + #[serde(skip)] git: Arc, last_ts_ms: i64, } impl AgentTranslator for OpenCodeTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, event: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { self.last_ts_ms = self.last_ts_ms.max(event.ts_ms); + if matches!( + event.event.as_str(), + "session.created" + | "chat.message" + | "experimental.chat.system.transform" + | "message.part.updated" + | "message.updated" + | "tool.execute.before" + | "tool.execute.after" + | "session.idle" + | "session.deleted" + | "session.error" + ) && native_session_id(&event.payload).is_none() + { + return Err(crate::translate::InputShapeError { + event: event.event.clone(), + detail: "missing required native session ID".into(), + } + .into()); + } let mut ops = match event.event.as_str() { "session.created" => self.session_created(event, ctx), "chat.message" => self.chat_message(event, ctx), "experimental.chat.system.transform" => self.system_prompt(event), - "message.part.updated" => properties::(&event.payload, "part") - .map(|part| self.part_updated(part, event.ts_ms)) - .unwrap_or_default(), - "message.updated" => properties::(&event.payload, "info") - .map(|info| self.message_updated(info, event.ts_ms)) - .unwrap_or_default(), - "tool.execute.before" => decode::(&event.payload) - .map(ToolBefore::from) - .map(|tool| self.tool_before(tool, event.ts_ms, ctx)) - .unwrap_or_default(), - "tool.execute.after" => decode::(&event.payload) - .map(ToolAfter::from) - .map(|tool| self.tool_after(tool, event.ts_ms)) - .unwrap_or_default(), - "permission.asked" => permission_properties(&event.payload) - .map(|permission| self.permission_asked(permission)) - .unwrap_or_default(), - "permission.replied" => permission_properties(&event.payload) - .map(|permission| self.permission_replied(permission, event.ts_ms)) - .unwrap_or_default(), + "message.part.updated" => self.part_updated( + properties_required("message.part.updated", &event.payload, "part")?, + event.ts_ms, + ), + "message.updated" => self.message_updated( + properties_required("message.updated", &event.payload, "info")?, + event.ts_ms, + ), + "tool.execute.before" => self.tool_before( + ToolBefore::from(decode_required::( + "tool.execute.before", + &event.payload, + )?), + event.ts_ms, + ctx, + ), + "tool.execute.after" => self.tool_after( + ToolAfter::from(decode_required::( + "tool.execute.after", + &event.payload, + )?), + event.ts_ms, + ), + "permission.asked" => { + self.permission_asked(permission_properties(&event.event, &event.payload)?) + } + "permission.replied" => self.permission_replied( + permission_properties(&event.event, &event.payload)?, + event.ts_ms, + ), "session.idle" => self.finish_session_event(event, false, None), "session.compacted" => Vec::new(), "session.deleted" => self.finish_session_event(event, true, None), @@ -945,7 +1031,7 @@ impl OpenCodeTranslator { if let Some(system) = &state.system_prompt { input.push(json!({"role":"system","content":system})) } - if info.summary { + if matches!(info.summary, Some(MessageSummary::Flag(true))) { input.extend(state.history.compaction_input()); state.history.begin_compacted( mid, @@ -1329,6 +1415,8 @@ fn native_session_id(v: &Value) -> Option { v.pointer("/input/sessionID") .or_else(|| v.get("sessionID")) .or_else(|| v.pointer("/properties/sessionID")) + .or_else(|| v.pointer("/properties/part/sessionID")) + .or_else(|| v.pointer("/properties/info/sessionID")) .or_else(|| v.pointer("/properties/info/id")) .or_else(|| v.pointer("/part/sessionID")) .and_then(Value::as_str) diff --git a/bt-daemon/src/translate/pi.rs b/bt-daemon/src/translate/pi.rs index f79e0b26..811a69ee 100644 --- a/bt-daemon/src/translate/pi.rs +++ b/bt-daemon/src/translate/pi.rs @@ -67,7 +67,7 @@ struct BeforeAgentStart { /// `ctx.model`: the selected model, which may be a Pi 1.0 virtual model that /// routes each request to a physical model. -#[derive(Deserialize)] +#[derive(Deserialize, serde::Serialize)] struct SelectedModel { provider: String, id: String, @@ -315,6 +315,17 @@ fn decode(value: &Value) -> Option { serde_json::from_value(value.clone()).ok() } +fn decode_required(event: &str, value: &Value) -> anyhow::Result { + serde_json::from_value(value.clone()).map_err(|error| { + crate::translate::InputShapeError { + event: event.to_owned(), + detail: error.to_string(), + } + .into() + }) +} + +#[derive(serde::Serialize, serde::Deserialize)] struct PendingLlm { start_ms: i64, input: Value, @@ -322,13 +333,14 @@ struct PendingLlm { request_config: Map, selected_model: Option, } -#[derive(Clone)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] struct ToolStart { start_ms: i64, name: String, args: Value, parent_call: Option, } +#[derive(serde::Serialize, serde::Deserialize)] struct PiTranslator { session_id: String, root_span_id: String, @@ -351,13 +363,116 @@ struct PiTranslator { branch_summary: Option<(String, i64, Value)>, last_ts: i64, thinking_level: Option, + #[serde(skip)] git: Arc, } impl AgentTranslator for PiTranslator { + fn snapshot(&self) -> anyhow::Result { + Ok(serde_json::to_value(self)?) + } + + fn restore(&mut self, snapshot: serde_json::Value) -> anyhow::Result<()> { + let mut restored: Self = serde_json::from_value(snapshot)?; + restored.git = self.git.clone(); + *self = restored; + Ok(()) + } + fn handle(&mut self, envelope: &Envelope, ctx: &SessionCtx) -> anyhow::Result> { - self.last_ts = self.last_ts.max(envelope.ts_ms); + if !matches!( + envelope.event.as_str(), + "session_start" + | "before_agent_start" + | "context" + | "before_provider_request" + | "message_update" + | "thinking_level_select" + | "message_end" + | "tool_execution_start" + | "tool_execution_end" + | "agent_end" + | "session_before_compact" + | "session_compact" + | "session_before_tree" + | "session_tree" + | "session_shutdown" + ) { + return Ok(Vec::new()); + } let event = envelope.payload.get("event").unwrap_or(&envelope.payload); + if !event.is_object() { + return Err(crate::translate::InputShapeError { + event: envelope.event.clone(), + detail: "expected an event object".into(), + } + .into()); + } + match envelope.event.as_str() { + "before_agent_start" => { + let _: BeforeAgentStart = decode_required(&envelope.event, event)?; + } + "context" => { + let _: ContextEvent = decode_required(&envelope.event, event)?; + if let Some(model) = envelope.payload.get("model") { + let _: SelectedModel = decode_required(&envelope.event, model)?; + } + } + "message_update" => { + let _: MessageUpdate = decode_required(&envelope.event, event)?; + } + "thinking_level_select" => { + let _: ThinkingLevel = decode_required(&envelope.event, event)?; + } + "tool_execution_start" => { + let _: ToolExecutionStart = decode_required(&envelope.event, event)?; + } + "message_end" => { + let message = event.get("message").unwrap_or(event); + let role = message.get("role").and_then(Value::as_str); + if role.is_none_or(str::is_empty) { + return Err(crate::translate::InputShapeError { + event: envelope.event.clone(), + detail: "missing required message role".into(), + } + .into()); + } + // Pi emits system/user message_end notifications as well, but + // this translator only creates LLM spans from assistant + // messages. Preserve those unused message shapes without + // pausing the source session. + if role == Some("assistant") { + decode_required::(&envelope.event, message)?; + } + } + "tool_execution_end" => { + let end = decode_required::(&envelope.event, event)?; + if end.tool_call_id.as_deref().is_none_or(str::is_empty) { + return Err(crate::translate::InputShapeError { + event: envelope.event.clone(), + detail: "missing required toolCallId".into(), + } + .into()); + } + } + "agent_end" => { + let _: AgentEnd = decode_required(&envelope.event, event)?; + } + "session_before_compact" => { + let _: BeforeCompact = decode_required(&envelope.event, event)?; + } + "session_compact" => { + let _: SessionCompact = decode_required(&envelope.event, event)?; + } + "session_before_tree" => { + let _: BeforeTree = decode_required(&envelope.event, event)?; + } + "session_tree" => { + let _: SessionTree = decode_required(&envelope.event, event)?; + } + _ => {} + } + self.last_ts = self.last_ts.max(envelope.ts_ms); let mut ops = self.ensure_root(envelope, ctx); match envelope.event.as_str() { "before_agent_start" => { @@ -389,14 +504,12 @@ impl AgentTranslator for PiTranslator { } "message_end" => ops.extend(self.message_end(event, envelope.ts_ms)), "tool_execution_start" => { - if let Some(event) = decode(event) { - ops.extend(self.tool_start(event, envelope.ts_ms)); - } + let event = decode_required("tool_execution_start", event)?; + ops.extend(self.tool_start(event, envelope.ts_ms)); } "tool_execution_end" => { - if let Some(event) = decode(event) { - ops.extend(self.tool_end(event, envelope.ts_ms)); - } + let event = decode_required("tool_execution_end", event)?; + ops.extend(self.tool_end(event, envelope.ts_ms)); } "agent_end" if decode::(event).is_none_or(|event| !event.will_retry) => { ops.extend(self.close_turn(envelope.ts_ms, None)); diff --git a/bt-daemon/src/translate/recent.rs b/bt-daemon/src/translate/recent.rs index a6a3e567..d26bbf52 100644 --- a/bt-daemon/src/translate/recent.rs +++ b/bt-daemon/src/translate/recent.rs @@ -11,6 +11,8 @@ use std::hash::Hash; pub(super) const RECENT_ID_CAPACITY: usize = 4_096; +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>"))] pub(super) struct RecentSet { values: HashSet, order: VecDeque, @@ -59,6 +61,8 @@ impl RecentSet { } } +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(bound(deserialize = "K: Eq + Hash + serde::Deserialize<'de>, V: serde::Deserialize<'de>"))] pub(super) struct RecentMap { values: HashMap, order: VecDeque, diff --git a/bt-daemon/src/translate/tool.rs b/bt-daemon/src/translate/tool.rs index aabe0ae6..49573345 100644 --- a/bt-daemon/src/translate/tool.rs +++ b/bt-daemon/src/translate/tool.rs @@ -6,7 +6,7 @@ use serde_json::{json, Map, Value}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum ToolApproval { Approved, Denied, diff --git a/bt-daemon/src/win_acl.rs b/bt-daemon/src/win_acl.rs index 767bf208..64ab8c27 100644 --- a/bt-daemon/src/win_acl.rs +++ b/bt-daemon/src/win_acl.rs @@ -119,7 +119,7 @@ pub(crate) fn is_owner_only(path: &Path, directory: bool) -> io::Result { } fn set_dacl(path: &Path, descriptor: &SecurityDescriptor) -> io::Result<()> { - let name = wide(path.as_os_str()); + let name = wide_path(path); let status = unsafe { SetNamedSecurityInfoW( name.as_ptr(), @@ -148,7 +148,7 @@ impl NamedDacl { } fn read(path: &Path) -> io::Result { - let name = wide(path.as_os_str()); + let name = wide_path(path); let mut dacl = null_mut(); let mut descriptor = null_mut(); let status = unsafe { @@ -235,6 +235,21 @@ fn wide(value: &OsStr) -> Vec { value.encode_wide().chain(std::iter::once(0)).collect() } +fn wide_path(path: &Path) -> Vec { + let mut value = path.as_os_str().encode_wide().collect::>(); + let prefix = [b'\\' as u16, b'\\' as u16, b'?' as u16, b'\\' as u16]; + if path.is_absolute() && value.len() > 240 && !value.starts_with(&prefix) { + if value.starts_with(&[b'\\' as u16, b'\\' as u16]) { + value.drain(..2); + value.splice(0..0, "\\\\?\\UNC\\".encode_utf16()); + } else { + value.splice(0..0, prefix); + } + } + value.push(0); + value +} + unsafe fn from_wide(value: *const u16) -> String { let length = (0..).take_while(|&index| *value.add(index) != 0).count(); String::from_utf16_lossy(std::slice::from_raw_parts(value, length)) diff --git a/bt-daemon/tests/antigravity_translator.rs b/bt-daemon/tests/antigravity_translator.rs index 743ef559..4b0271d9 100644 --- a/bt-daemon/tests/antigravity_translator.rs +++ b/bt-daemon/tests/antigravity_translator.rs @@ -832,6 +832,7 @@ fn resumed_process_reuses_invocation_zero_without_reparenting_to_turn_one() { ), ] { ops.extend(translator.handle(&hook, &ctx).unwrap()); + translator.restore(translator.snapshot().unwrap()).unwrap(); } let rows = reduce(ops); diff --git a/bt-daemon/tests/braintrust_sink.rs b/bt-daemon/tests/braintrust_sink.rs index 9e730b95..00afc4af 100644 --- a/bt-daemon/tests/braintrust_sink.rs +++ b/bt-daemon/tests/braintrust_sink.rs @@ -224,6 +224,64 @@ async fn multi_profile_sessions_route_to_their_own_backend() { assert!(!b.contains("span-A"), "server B leaked session A's span"); } +#[tokio::test] +async fn a_rejected_batch_fails_only_its_routes_checked_flush() { + let bad = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({}))) + .mount(&bad) + .await; + Mock::given(method("POST")) + .and(path("/api/project/register")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"project":{"id":"p"}}))) + .mount(&bad) + .await; + Mock::given(method("POST")) + .and(path("/logs3")) + .respond_with(ResponseTemplate::new(401).set_body_string("expired")) + .mount(&bad) + .await; + let good = mock_backend().await; + let factory = BraintrustSinkFactory::new(BraintrustSinkConfig::default()); + let mut rejected = factory.create("rejected", "codex", None).unwrap(); + rejected.configure(&session_config(&bad.uri())); + rejected + .emit(&[SpanOp::Insert(row( + "bad", + "bad", + &[], + "Bad", + SpanType::Task, + 1, + Some(2), + ))]) + .await + .unwrap(); + let error = rejected.flush().await.unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(braintrust_sdk_rust::BraintrustError::Api { status: 401, .. }) + )); + + let mut accepted = factory.create("accepted", "codex", None).unwrap(); + accepted.configure(&session_config(&good.uri())); + accepted + .emit(&[SpanOp::Insert(row( + "good", + "good", + &[], + "Good", + SpanType::Task, + 1, + Some(2), + ))]) + .await + .unwrap(); + accepted.flush().await.unwrap(); + assert!(logs3_bodies(&good).await.contains("good")); +} + /// Regression: an `Insert` that names a span, followed by a `Merge` that /// doesn't (the common "close/annotate" pattern, which builds `SpanRow` with /// `..Default::default()` and an empty `name`), must not clobber the name. diff --git a/bt-daemon/tests/claude_translator.rs b/bt-daemon/tests/claude_translator.rs index 26dbf517..aa84a2e9 100644 --- a/bt-daemon/tests/claude_translator.rs +++ b/bt-daemon/tests/claude_translator.rs @@ -112,6 +112,9 @@ fn replay_from(name: &str, source: Source) -> Vec { while let Some(batch) = translator.drain_pending(&ctx).unwrap() { ops.extend(batch); } + let snapshot = translator.snapshot().unwrap(); + translator = registry.create("claude-code", session_id); + translator.restore(snapshot).unwrap(); } ops.extend(translator.flush(&ctx).unwrap()); while let Some(batch) = translator.drain_pending(&ctx).unwrap() { @@ -463,7 +466,7 @@ fn claude_turns_capture_the_effective_permission_mode() { } #[test] -fn claude_subagent_routing_tolerates_malformed_optional_metadata() { +fn claude_rejects_malformed_present_optional_subagent_metadata() { let registry = Registry::default_agents(); let mut translator = registry.create("claude-code", "subagent-session"); let ctx = SessionCtx { @@ -489,17 +492,48 @@ fn claude_subagent_routing_tolerates_malformed_optional_metadata() { capture: None, }; - let rows = reduce(translator.handle(&event, &ctx).unwrap()); - assert!(rows.values().any(|row| row.name == "Claude Code: demo")); - let subagent = rows - .values() - .find(|row| row.name == "subagent: agent") - .expect("valid agent_id must create a subagent span"); - assert_eq!(subagent.metadata.as_ref().unwrap()["agent_id"], "agent-1"); + let error = translator.handle(&event, &ctx).unwrap_err(); + assert!( + error + .to_string() + .starts_with("unsupported shape for SubagentStart"), + "{error}" + ); +} + +#[test] +fn claude_optional_hook_fields_allow_absence_but_reject_wrong_types() { + let registry = Registry::default_agents(); + let ctx = SessionCtx { + session_id: "optional-hook-fields".into(), + config: None, + }; + let mut translator = registry.create("claude-code", "optional-hook-fields"); + assert!(translator + .handle( + &claude_event("optional-hook-fields", "SessionStart", 1, json!({})), + &ctx, + ) + .is_ok()); + + let error = translator + .handle( + &claude_event( + "optional-hook-fields", + "SessionStart", + 2, + json!({"cwd": 42}), + ), + &ctx, + ) + .unwrap_err(); + assert!(error + .to_string() + .starts_with("unsupported shape for SessionStart")); } #[test] -fn claude_ignores_empty_subagent_identifier() { +fn claude_rejects_empty_required_subagent_identifier() { let registry = Registry::default_agents(); let mut translator = registry.create("claude-code", "empty-subagent-session"); let ctx = SessionCtx { @@ -524,8 +558,10 @@ fn claude_ignores_empty_subagent_identifier() { capture: None, }; - let rows = reduce(translator.handle(&event, &ctx).unwrap()); - assert!(rows.values().all(|row| !row.name.starts_with("subagent:"))); + let error = translator.handle(&event, &ctx).unwrap_err(); + assert!(error + .to_string() + .starts_with("unsupported shape for SubagentStart")); } #[test] diff --git a/bt-daemon/tests/codex_translator.rs b/bt-daemon/tests/codex_translator.rs index 34050ed4..ddccc635 100644 --- a/bt-daemon/tests/codex_translator.rs +++ b/bt-daemon/tests/codex_translator.rs @@ -22,6 +22,43 @@ fn line(v: Value) -> String { serde_json::to_string(&v).unwrap() } +#[test] +fn codex_optional_session_start_fields_allow_absence_but_reject_wrong_types() { + let registry = Registry::default_agents(); + let ctx = SessionCtx { + session_id: "session-start-shape".into(), + config: None, + }; + let mut absent_fields_translator = registry.create("codex", "session-start-shape"); + assert!(absent_fields_translator + .handle( + &envelope( + "session-start-shape", + "SessionStart", + "/missing/rollout.jsonl", + json!({}), + ), + &ctx, + ) + .is_ok()); + + let mut translator = registry.create("codex", "session-start-shape"); + let error = translator + .handle( + &envelope( + "session-start-shape", + "SessionStart", + "/missing/rollout.jsonl", + json!({"source": 42}), + ), + &ctx, + ) + .unwrap_err(); + assert!(error + .to_string() + .starts_with("unsupported shape for SessionStart")); +} + fn expected_username() -> String { std::env::var("USER") .or_else(|_| std::env::var("USERNAME")) @@ -182,6 +219,10 @@ fn codex_happy_path_builds_session_turn_llm_tool_tree() { ) .unwrap(), ); + while let Some(batch) = tr.drain_pending(&ctx).unwrap() { + ops.extend(batch); + } + tr.restore(tr.snapshot().unwrap()).unwrap(); // A later trigger (Stop) — nothing new in the transcript here. ops.extend( tr.handle(&envelope("sess-1", "Stop", tpath, json!({})), &ctx) diff --git a/bt-daemon/tests/cursor_pipeline.rs b/bt-daemon/tests/cursor_pipeline.rs index dacbf00e..e84ead0d 100644 --- a/bt-daemon/tests/cursor_pipeline.rs +++ b/bt-daemon/tests/cursor_pipeline.rs @@ -1,6 +1,9 @@ //! Cursor hooks freeze native transcript observations before journal ingress. //! Recovery must use those mirrors even after native files change or disappear. +#[path = "support/ledger.rs"] +mod ledger; + use async_trait::async_trait; use bt_daemon::wire::{ AuthSelection, BackendAuth, Envelope, FlushMode, SessionRoute, TraceDestination, @@ -417,7 +420,6 @@ async fn late_response_is_delivered_and_checkpointed_without_another_hook() { ] { send_with_flush_mode(&socket, event, ts_ms, payload, FlushMode::FlushOnTurnEnd).await; } - let journal = source_journal_path(&data, "cursor", "cursor-recovery"); // Ingress acknowledges capture before the out-of-band sink flush finishes. // Wait for delivery and its checkpoint without requesting an explicit flush. let completed = tokio::time::timeout(Duration::from_secs(5), async { @@ -425,24 +427,13 @@ async fn late_response_is_delivered_and_checkpointed_without_another_hook() { let delivered = buffered.lock().unwrap().delivered.iter().any(|op| { matches!(op, SpanOp::Merge(row) if row.output == Some(json!("late final answer"))) }); - let text = std::fs::read_to_string(&journal).unwrap_or_default(); - let mut position = 0; - let mut response_through = None; - let mut checkpoint_through = 0; - for line in text.split_inclusive('\n') { - position += line.len() as u64; - let Ok(row) = serde_json::from_str::(line) else { - continue; - }; - if row["event"] == "afterAgentResponse" { - response_through = Some(position); - } - if row.get("_bt_record_type").is_some() { - checkpoint_through = - checkpoint_through.max(row["through"].as_u64().unwrap_or(0)); - } - } - if delivered && response_through.is_some_and(|offset| checkpoint_through >= offset) { + let checkpointed = + rusqlite::Connection::open(ledger::database(&data, "cursor", "cursor-recovery")) + .and_then(|db| { + db.query_row("SELECT COUNT(*) FROM batches", [], |r| r.get::<_, i64>(0)) + }) + .is_ok_and(|pending| pending == 0); + if delivered && checkpointed { break; } tokio::time::sleep(Duration::from_millis(10)).await; @@ -525,7 +516,7 @@ async fn repeated_transcript_prompt_is_delivered_as_a_second_turn_after_checkpoi } #[tokio::test] -async fn cursor_mirror_generations_and_capture_bounds_survive_deleted_native_recovery() { +async fn cursor_continuation_survives_collected_input_and_deleted_native_history() { let tmp = tempfile::tempdir().unwrap(); let data = tmp.path().join("data"); let socket = endpoint(tmp.path()); @@ -650,33 +641,15 @@ async fn cursor_mirror_generations_and_capture_bounds_survive_deleted_native_rec .iter() .all(|row| row["parent_span_ids"].as_array().unwrap().len() == 1)); let journal = source_journal_path(&data, "cursor", "cursor-recovery"); - let persisted = rows(&journal); - let observations = persisted + assert!(rows(&journal) .iter() - .filter(|row| row["event"] == "stop") - .map(|row| &row["payload"]["_bt_transcript_mirror"]) - .collect::>(); - assert_eq!(observations.len(), 2); - assert_ne!(observations[0]["mirror"], observations[1]["mirror"]); - for observation in &observations { - assert_eq!(observation["through"], first.len()); - assert!(Path::new(observation["mirror"].as_str().unwrap()) - .starts_with(data.join("transcripts"))); - } + .all(|row| row.get("_bt_wal_base").is_some())); assert!(!std::fs::read_to_string(&journal) .unwrap() .contains("cursor-test-secret")); - - // Simulate a delivery loss: retain accepted ingress, discard sink receipts. - let pending = persisted - .iter() - .filter(|row| row.get("_bt_record_type").is_none()) - .map(|row| format!("{row}\n")) - .collect::(); - std::fs::write(&journal, pending).unwrap(); - std::fs::remove_dir_all(data.join("delivery-ledger")).unwrap(); std::fs::remove_file(&native).unwrap(); - std::fs::remove_file(&output).unwrap(); + // All delivery was acknowledged. Restart relies on SQLite continuation, without native + // history or redelivery of already-collected span operations. let recovered_daemon = start(&data, &socket).await; assert!( flush_session("cursor-recovery", &socket, 5000) diff --git a/bt-daemon/tests/cursor_translator.rs b/bt-daemon/tests/cursor_translator.rs index 4d6c6c5e..7e976c85 100644 --- a/bt-daemon/tests/cursor_translator.rs +++ b/bt-daemon/tests/cursor_translator.rs @@ -73,6 +73,9 @@ impl Harness { "transcript draining did not make bounded progress" ); } + let snapshot = self.translator.snapshot().unwrap(); + self.translator = Registry::default_agents().create("cursor", &self.ctx.session_id); + self.translator.restore(snapshot).unwrap(); } fn finish(&mut self) { let ops = self.translator.finalize(&self.ctx).unwrap(); @@ -1868,7 +1871,8 @@ fn malformed_hooks_fail_the_event_instead_of_dropping_fields() { .unwrap_err() .to_string(); assert!( - error.starts_with(&format!("unexpected cursor {kind} hook format")), + error.starts_with(&format!("unexpected cursor {kind} hook format")) + || error.starts_with(&format!("unsupported shape for {kind}")), "{error}" ); } diff --git a/bt-daemon/tests/distributed_tracing.rs b/bt-daemon/tests/distributed_tracing.rs index 10592a8c..380c3129 100644 --- a/bt-daemon/tests/distributed_tracing.rs +++ b/bt-daemon/tests/distributed_tracing.rs @@ -5,14 +5,16 @@ //! matrix in the ordinary test suite while still exercising IPC, journaling, //! translation, automatic routing, and sink configuration. +#[path = "support/ledger.rs"] +mod ledger; mod support; use async_trait::async_trait; use braintrust_sdk_rust::{SpanComponents, SpanObjectType}; use bt_daemon::wire::{AuthSelection, BackendAuth, Envelope, SessionConfig, TraceDestination}; use bt_daemon::{ - flush_session, forward_envelope, run_serve, run_status, shutdown_daemon, source_journal_path, - AuthLease, AuthProvider, AuthResolveReason, HostInfo, Registry, ServeArgs, ServeOptions, Sink, + flush_session, forward_envelope, run_serve, run_status, shutdown_daemon, AuthLease, + AuthProvider, AuthResolveReason, HostInfo, Registry, ServeArgs, ServeOptions, Sink, SinkFactory, SpanOp, SpanRow, SpanType, StatusArgs, }; use std::collections::HashMap; @@ -1304,14 +1306,12 @@ async fn ipc_initialize_pid_is_captured_without_hook_metadata() { shutdown_daemon(&socket).await.unwrap(); daemon.await.unwrap(); - let journal = tokio::fs::read_to_string(source_journal_path( + let row = ledger::catalog( &tmp.path().join("data"), "claude-code", "automatic-process-capture", - )) - .await - .unwrap(); - let row: serde_json::Value = serde_json::from_str(journal.lines().next().unwrap()).unwrap(); + )[0]["envelope"] + .clone(); let chain = row .pointer("/capture/process_chain") .and_then(serde_json::Value::as_array) @@ -2909,16 +2909,39 @@ async fn pending_restart_preserves_current_candidates_and_decided_legacy_links() let journal_path = bt_daemon::source_journal_path(&data_dir, "claude-code", "bounded-child"); let journal = tokio::fs::read(&journal_path).await.unwrap(); + let span_db = ledger::database(&data_dir, "claude-code", "bounded-child"); + let saved_db = tmp.path().join("before-shutdown.sqlite"); + let had_db = span_db.exists(); + if had_db { + rusqlite::Connection::open(&span_db) + .unwrap() + .execute("VACUUM INTO ?1", [saved_db.to_str().unwrap()]) + .unwrap(); + } shutdown_daemon(&socket).await.unwrap(); daemon.await.unwrap(); + if span_db.parent().unwrap().exists() { + std::fs::remove_dir_all(span_db.parent().unwrap()).unwrap(); + } + if had_db { + std::fs::create_dir_all(span_db.parent().unwrap()).unwrap(); + std::fs::copy(saved_db, &span_db).unwrap(); + } // Restore the accepted-but-undelivered checkpoint, as after an abrupt // process exit. Shutdown itself is covered separately above. tokio::fs::write(&journal_path, journal).await.unwrap(); // Graceful shutdown delivered the buffered spans. Roll back its local // delivery acknowledgements as well as the journal to model a crash. - tokio::fs::remove_dir_all(data_dir.join("delivery-ledger")) - .await - .unwrap(); + let ledger_dir = data_dir.join("delivery-ledger"); + if ledger_dir.exists() { + tokio::fs::remove_dir_all(ledger_dir).await.unwrap(); + } + let control = data_dir + .join("journal-control") + .join(journal_path.file_name().unwrap()); + if control.exists() { + tokio::fs::remove_file(control).await.unwrap(); + } let mut legacy_route = original_route.clone(); legacy_route.destination = Some(external_parent()); // A decided legacy link already put earlier rows in that parent's diff --git a/bt-daemon/tests/grok_translator.rs b/bt-daemon/tests/grok_translator.rs index 0980e7cb..39007cbd 100644 --- a/bt-daemon/tests/grok_translator.rs +++ b/bt-daemon/tests/grok_translator.rs @@ -344,6 +344,9 @@ fn grok_transcript_reads_incrementally_and_replay_is_deterministic() { let first = incremental .handle(&envelope(first_through, 0), &ctx()) .unwrap(); + incremental + .restore(incremental.snapshot().unwrap()) + .unwrap(); let second = incremental .handle(&envelope(full_through, events), &ctx()) .unwrap(); diff --git a/bt-daemon/tests/opencode_translator.rs b/bt-daemon/tests/opencode_translator.rs index 218f8b8b..0e0f5f3d 100644 --- a/bt-daemon/tests/opencode_translator.rs +++ b/bt-daemon/tests/opencode_translator.rs @@ -101,6 +101,7 @@ fn opencode_builds_turn_llm_tool_and_closes_the_session() { let mut ops = Vec::new(); for event in events { ops.extend(translator.handle(&event, &ctx).unwrap()); + translator.restore(translator.snapshot().unwrap()).unwrap(); } let rows = reduce(ops); assert_eq!( @@ -126,6 +127,53 @@ fn opencode_builds_turn_llm_tool_and_closes_the_session() { .all(|r| r.end_ms.is_some())); } +#[test] +fn opencode_accepts_summary_diff_metadata_on_message_updates() { + let registry = Registry::default_agents(); + let mut translator = registry.create("opencode", "root-session"); + let ctx = SessionCtx { + session_id: "root-session".into(), + config: None, + }; + let ops = translator + .handle( + &event( + "message.updated", + 1, + json!({"properties":{"info":{"id":"m1","sessionID":"native","role":"user","summary":{"diffs":[]}}}}), + ), + &ctx, + ) + .unwrap(); + assert!(ops.is_empty()); +} + +#[test] +fn opencode_rejects_wrong_type_for_optional_summary_field() { + let registry = Registry::default_agents(); + let mut translator = registry.create("opencode", "root-session"); + let ctx = SessionCtx { + session_id: "root-session".into(), + config: None, + }; + let error = translator + .handle( + &event( + "message.updated", + 1, + json!({"properties":{"info":{"id":"m1","sessionID":"native","role":"user","summary":42}}}), + ), + &ctx, + ) + .unwrap_err(); + assert!( + error + .to_string() + .starts_with("unsupported shape for message.updated"), + "{error}" + ); +} + #[test] fn opencode_idle_updates_a_resumable_session_root() { let registry = Registry::default_agents(); @@ -687,7 +735,7 @@ fn opencode_ignores_unknown_fields_at_typed_reducer_boundaries() { } #[test] -fn opencode_ignores_malformed_typed_events_without_failing_the_session() { +fn opencode_fails_present_optional_fields_with_the_wrong_type() { let registry = Registry::default_agents(); let mut translator = registry.create("opencode", "root-session"); let ctx = SessionCtx { @@ -715,7 +763,7 @@ fn opencode_ignores_malformed_typed_events_without_failing_the_session() { ) .unwrap(); - let ops = translator + let error = translator .handle( &event( "tool.execute.before", @@ -724,10 +772,13 @@ fn opencode_ignores_malformed_typed_events_without_failing_the_session() { ), &ctx, ) - .unwrap(); - assert!(ops - .iter() - .all(|op| !matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Tool))); + .unwrap_err(); + assert!( + error + .to_string() + .starts_with("unsupported shape for tool.execute.before"), + "{error}" + ); assert!(translator .handle( &event( diff --git a/bt-daemon/tests/pi_translator.rs b/bt-daemon/tests/pi_translator.rs index 9c9683bc..fa98673b 100644 --- a/bt-daemon/tests/pi_translator.rs +++ b/bt-daemon/tests/pi_translator.rs @@ -171,6 +171,7 @@ fn pi_builds_turn_llm_tool_compaction_and_shutdown_spans() { let mut ops = Vec::new(); for event in events { ops.extend(translator.handle(&event, &ctx).unwrap()); + translator.restore(translator.snapshot().unwrap()).unwrap(); } let rows = reduce(ops); assert_eq!( @@ -636,7 +637,7 @@ fn pi_ignores_unknown_native_fields_at_typed_event_boundaries() { } #[test] -fn pi_ignores_malformed_typed_events_without_failing_the_session() { +fn pi_reports_malformed_required_tool_identity() { let registry = Registry::default_agents(); let mut translator = registry.create("pi", "pi-session"); let ctx = SessionCtx { @@ -650,9 +651,9 @@ fn pi_ignores_malformed_typed_events_without_failing_the_session() { ) .unwrap(); - // A future Pi version could change this identifier's representation. The - // raw event remains journaled, but a typed reducer must not fail the actor. - let ops = translator + // A changed identifier shape must pause translation so a newer daemon can + // reinterpret the journaled event instead of silently losing a tool. + let error = translator .handle( &event( "tool_execution_start", @@ -661,16 +662,36 @@ fn pi_ignores_malformed_typed_events_without_failing_the_session() { ), &ctx, ) - .unwrap(); - assert!(ops - .iter() - .all(|op| !matches!(op, SpanOp::Insert(row) if row.span_type == SpanType::Tool))); + .unwrap_err(); + assert!(error + .to_string() + .contains("unsupported shape for tool_execution_start")); +} - assert!(translator - .handle(&event("agent_end", 3, json!({})), &ctx) - .unwrap() - .iter() - .any(|op| matches!(op, SpanOp::Merge(row) if row.end_ms == Some(3)))); +#[test] +fn pi_ignores_non_assistant_message_end_shapes_but_rejects_malformed_assistant_messages() { + let registry = Registry::default_agents(); + let mut translator = registry.create("pi", "pi-session"); + let ctx = SessionCtx { + session_id: "pi-session".into(), + config: None, + }; + let system = event( + "message_end", + 1, + json!({"type":"message_end","message":{"role":"system","content":"system prompt"}}), + ); + assert!(translator.handle(&system, &ctx).is_ok()); + + let malformed_assistant = event( + "message_end", + 2, + json!({"type":"message_end","message":{"role":"assistant","content":"unexpected string"}}), + ); + let error = translator.handle(&malformed_assistant, &ctx).unwrap_err(); + assert!(error + .to_string() + .contains("unsupported shape for message_end")); } #[test] diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index 5ca79489..cb76e189 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -7,6 +7,9 @@ reason = "Test fixtures intentionally launch raw children." )] +#[path = "support/ledger.rs"] +mod ledger; + use async_trait::async_trait; use bt_daemon::wire::{ AuthSelection, AuthSource, BackendAuth, Envelope, SessionConfig, SessionRoute, @@ -20,6 +23,7 @@ use bt_daemon::{ }; #[cfg(all(feature = "cli", unix))] use bt_daemon::{run_traced, RunArgs, RunHookCommand, RunSource, SpanType}; +use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -50,28 +54,6 @@ struct TrackingSink { flushes: Arc>>, } -struct SlowSink; - -#[async_trait] -impl Sink for SlowSink { - fn configure(&mut self, _config: &SessionConfig) {} - async fn emit(&mut self, ops: &[SpanOp]) -> anyhow::Result { - tokio::time::sleep(Duration::from_millis(500)).await; - Ok(ops.len() as u64) - } - async fn flush(&mut self) -> anyhow::Result<()> { - Ok(()) - } -} - -struct SlowSinkFactory; - -impl SinkFactory for SlowSinkFactory { - fn create(&self, _: &str, _: &str, _: Option<&str>) -> anyhow::Result> { - Ok(Box::new(SlowSink)) - } -} - struct GateSinkFactory { blocked: Arc, gate: Arc, @@ -483,33 +465,6 @@ async fn start_tracking_daemon( (socket, handle, flushes, tmp) } -async fn start_slow_daemon() -> (PathBuf, tokio::task::JoinHandle<()>, tempfile::TempDir) { - let tmp = tempfile::tempdir().unwrap(); - let data_dir = tmp.path().join("data"); - let socket = test_endpoint(tmp.path()); - let opts = ServeOptions { - version: "test".into(), - translators: Arc::new(Registry::default_agents()), - sink_factory: Arc::new(SlowSinkFactory), - auth_provider: Some(Arc::new(TestAuthProvider { - calls: Mutex::new(Vec::new()), - fail: false, - first_lease_expired: false, - })), - }; - let args = ServeArgs { - socket: Some(socket.clone()), - data_dir: Some(data_dir), - idle_timeout_secs: 0, - session_idle_timeout_secs: 0, - }; - let handle = tokio::spawn(async move { - let _ = run_serve(args, opts).await; - }); - wait_for(&socket).await; - (socket, handle, tmp) -} - async fn start_gated_daemon( gate: Arc, emitted: Arc, @@ -610,7 +565,7 @@ async fn routed_sessions_resolve_multiple_profiles_without_journaling_credential for session in ["work-session", "personal-session"] { let journal = std::fs::read_to_string(source_journal_path(&data_dir, "debug", session)).unwrap(); - assert!(journal.contains("\"route\"")); + assert!(ledger::catalog(&data_dir, "debug", session)[0]["envelope"]["route"].is_object()); assert!(!journal.contains("secret-")); assert!(!journal.contains("token_sha256_prefix")); assert!(!journal.contains("\"config\"")); @@ -654,7 +609,11 @@ async fn environment_routes_remain_environment_auth_without_journaling_credentia "environment-session", )) .unwrap(); - assert!(journal.contains(r#""source":"environment""#)); + assert_eq!( + ledger::catalog(&data_dir, "debug", "environment-session")[0]["envelope"]["route"]["auth"] + ["source"], + "environment" + ); assert!(!journal.contains("secret-environment")); shutdown(&socket).await; @@ -876,9 +835,8 @@ async fn a_route_backfills_observations_captured_by_another_route_once() { shutdown(&socket).await; handle.await.unwrap(); - // A fresh daemon generation can activate an existing route after another - // route captured an event. It must backfill the missing observation once, - // then record a checkpoint for the receiving destination. + // Both active routes consumed the shared revision stream in the first + // generation. A fresh daemon only needs to deliver the new Stop. let provider = Arc::new(TestAuthProvider { calls: Mutex::new(Vec::new()), fail: false, @@ -916,8 +874,8 @@ async fn a_route_backfills_observations_captured_by_another_route_once() { ); assert_eq!( sinks[0].emitted.load(std::sync::atomic::Ordering::Relaxed), - 2, - "work receives the personal-route observation once, then its new Stop" + 1, + "work receives only the new Stop after its shared cursor checkpoint" ); shutdown(&socket).await; @@ -925,7 +883,7 @@ async fn a_route_backfills_observations_captured_by_another_route_once() { } #[tokio::test] -async fn a_new_destination_receives_history_once_then_uses_its_own_checkpoint() { +async fn a_new_destination_consumes_available_output_then_uses_its_own_checkpoint() { let tmp = tempfile::tempdir().unwrap(); let data_dir = tmp.path().join("data"); let socket = test_endpoint(tmp.path()); @@ -985,8 +943,8 @@ async fn a_new_destination_receives_history_once_then_uses_its_own_checkpoint() delivered[0] .emitted .load(std::sync::atomic::Ordering::Relaxed), - 3, - "personal receives work's root and SessionStart plus its own Stop" + 1, + "collected history requires transcript import; the new route receives its Stop" ); assert_eq!( delivered[0].org.lock().unwrap().clone().as_deref(), @@ -1015,7 +973,14 @@ async fn a_new_destination_receives_history_once_then_uses_its_own_checkpoint() flush_session("shared-history", &socket, 5000) .await .unwrap(); - let delivered = third_sinks.sinks.lock().unwrap().clone(); + let delivered = third_sinks + .sinks + .lock() + .unwrap() + .iter() + .filter(|sink| sink.org.lock().unwrap().as_deref() == Some("personal-org")) + .cloned() + .collect::>(); assert_eq!(delivered.len(), 1); assert_eq!( delivered[0] @@ -1066,6 +1031,211 @@ async fn auth_resolution_failure_is_reported_without_exposing_credentials() { handle.await.unwrap(); } +struct ToggleAuthProvider { + denied: std::sync::atomic::AtomicBool, +} + +#[async_trait] +impl AuthProvider for ToggleAuthProvider { + async fn resolve( + &self, + selection: &AuthSelection, + _reason: AuthResolveReason, + ) -> anyhow::Result { + if selection.profile.as_deref() == Some("missing") + && self.denied.load(std::sync::atomic::Ordering::SeqCst) + { + anyhow::bail!("profile credential unavailable") + } + Ok(AuthLease { + selection: selection.clone().canonicalized()?, + auth: BackendAuth { + token: "secret-current".into(), + api_url: None, + app_url: None, + org_name: selection.org_name.clone(), + org_id: None, + }, + expires_at_ms: None, + }) + } +} + +#[tokio::test] +async fn login_recovery_replays_only_the_held_session() { + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("data"); + let socket = test_endpoint(temp.path()); + let provider = Arc::new(ToggleAuthProvider { + denied: std::sync::atomic::AtomicBool::new(true), + }); + let args = ServeArgs { + socket: Some(socket.clone()), + data_dir: Some(data_dir.clone()), + idle_timeout_secs: 0, + session_idle_timeout_secs: 0, + }; + let mut opts = debug_serve_options("test", &data_dir); + opts.auth_provider = Some(provider.clone()); + let daemon = tokio::spawn(async move { run_serve(args, opts).await.unwrap() }); + wait_for(&socket).await; + let host = dummy_host(); + for event in ["SessionStart", "Stop"] { + forward_envelope( + &routed_envelope("held-login", "missing", "org", event), + &socket, + &host, + false, + ) + .await + .unwrap(); + } + forward_envelope( + &routed_envelope("healthy-login", "working", "org", "SessionStart"), + &socket, + &host, + false, + ) + .await + .unwrap(); + assert!( + flush_session("healthy-login", &socket, 5000) + .await + .unwrap() + .flushed + ); + assert!(data_dir.join("spans/healthy-login.ndjson").exists()); + assert!(!data_dir.join("spans/held-login.ndjson").exists()); + let incident: Value = + serde_json::from_slice(&std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap()) + .unwrap(); + assert_eq!(incident["active"][0]["cause"]["kind"], "credentials"); + + provider + .denied + .store(false, std::sync::atomic::Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(6), async { + loop { + let output = std::fs::read_to_string(data_dir.join("spans/held-login.ndjson")) + .unwrap_or_default(); + let incident: Value = serde_json::from_slice( + &std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap(), + ) + .unwrap(); + if output.contains("Stop") && incident["active"].as_array().is_some_and(Vec::is_empty) { + break; + } + tokio::time::sleep(Duration::from_millis(30)).await; + } + }) + .await + .expect("login recovery did not replay held events"); + shutdown(&socket).await; + daemon.await.unwrap(); +} + +#[tokio::test] +async fn transient_delivery_replays_without_new_agent_events() { + struct FlakySinkFactory { + unavailable: Arc, + delivered: Arc>>, + } + struct FlakySink { + unavailable: Arc, + delivered: Arc>>, + pending: Vec, + } + impl SinkFactory for FlakySinkFactory { + fn create(&self, _: &str, _: &str, _: Option<&str>) -> anyhow::Result> { + Ok(Box::new(FlakySink { + unavailable: self.unavailable.clone(), + delivered: self.delivered.clone(), + pending: Vec::new(), + })) + } + } + #[async_trait] + impl Sink for FlakySink { + async fn emit(&mut self, ops: &[SpanOp]) -> anyhow::Result { + self.pending.extend_from_slice(ops); + Ok(ops.len() as u64) + } + async fn flush(&mut self) -> anyhow::Result<()> { + if self.unavailable.load(std::sync::atomic::Ordering::SeqCst) { + return Err(braintrust_sdk_rust::BraintrustError::Api { + status: 503, + message: "server unavailable".into(), + } + .into()); + } + self.delivered.lock().unwrap().append(&mut self.pending); + Ok(()) + } + } + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("data"); + let socket = test_endpoint(temp.path()); + let unavailable = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let delivered = Arc::new(Mutex::new(Vec::new())); + let opts = ServeOptions { + version: "test".into(), + translators: Arc::new(Registry::default_agents()), + sink_factory: Arc::new(FlakySinkFactory { + unavailable: unavailable.clone(), + delivered: delivered.clone(), + }), + auth_provider: Some(Arc::new(TestAuthProvider { + calls: Mutex::new(Vec::new()), + fail: false, + first_lease_expired: false, + })), + }; + let args = ServeArgs { + socket: Some(socket.clone()), + data_dir: Some(data_dir.clone()), + idle_timeout_secs: 0, + session_idle_timeout_secs: 0, + }; + let daemon = tokio::spawn(async move { run_serve(args, opts).await.unwrap() }); + wait_for(&socket).await; + forward_envelope( + &envelope("transport-held", "SessionStart", 1), + &socket, + &dummy_host(), + false, + ) + .await + .unwrap(); + assert!( + !flush_session("transport-held", &socket, 5000) + .await + .unwrap() + .flushed + ); + let incident: Value = + serde_json::from_slice(&std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap()) + .unwrap(); + assert_eq!(incident["active"][0]["cause"]["kind"], "transport"); + unavailable.store(false, std::sync::atomic::Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(6), async { + loop { + let incident: Value = serde_json::from_slice( + &std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap(), + ) + .unwrap(); + if incident["active"].as_array().is_some_and(Vec::is_empty) { + break; + } + tokio::time::sleep(Duration::from_millis(30)).await; + } + }) + .await + .expect("endpoint recovery did not replay the held route"); + assert!(!delivered.lock().unwrap().is_empty()); + shutdown(&socket).await; + daemon.await.unwrap(); +} + #[tokio::test] async fn events_are_ordered_journaled_and_emitted() { let (data_dir, socket, handle, _tmp) = start_daemon().await; @@ -1081,35 +1251,14 @@ async fn events_are_ordered_journaled_and_emitted() { assert!(flushed.flushed, "flush did not complete: {flushed:?}"); assert_eq!(flushed.pending, 0); - // Journal: three events plus a durable delivery checkpoint, with only the - // non-secret route. Checkpoints are control records, not agent events. + // Translation has durably transferred all accepted events into spans. let journal = source_journal_path(&data_dir, "debug", "sess-1"); let jtext = std::fs::read_to_string(&journal).unwrap(); - let jlines: Vec<&str> = jtext.lines().filter(|l| !l.trim().is_empty()).collect(); - assert_eq!( - jlines.len(), - 4, - "expected 3 event lines plus one checkpoint, got {}", - jlines.len() - ); - assert!( - !jtext.contains("sk-TOP-SECRET-abc123"), - "token leaked into journal!" - ); + assert_eq!(jtext.lines().count(), 1); + assert!(jtext.contains("_bt_wal_base")); + assert!(!jtext.contains("sk-TOP-SECRET-abc123")); assert!(!jtext.contains("token_sha256_prefix")); - let events: Vec = jlines - .iter() - .filter(|line| !line.contains("\"_bt_record_type\":\"delivery_checkpoint\"")) - .map(|l| { - serde_json::from_str::(l).unwrap()["event"] - .as_str() - .unwrap() - .to_string() - }) - .collect(); - assert_eq!(events, vec!["SessionStart", "PostToolUse", "Stop"]); - // Spans: debug translator emits a root once + one span per event = 4. let spans = data_dir.join("spans").join("sess-1.ndjson"); let stext = std::fs::read_to_string(&spans).unwrap(); @@ -1149,13 +1298,8 @@ async fn session_ids_longer_than_a_file_name_are_journaled_and_emitted() { assert!(flushed.flushed, "flush did not complete: {flushed:?}"); let jtext = std::fs::read_to_string(source_journal_path(&data_dir, "debug", &session)).unwrap(); - assert_eq!( - jtext - .lines() - .filter(|line| !line.contains("\"_bt_record_type\":\"delivery_checkpoint\"")) - .count(), - 2 - ); + assert!(jtext.contains("_bt_wal_base")); + assert!(!jtext.contains("SessionStart")); let spans: Vec<_> = std::fs::read_dir(data_dir.join("spans")) .unwrap() .map(|entry| std::fs::read_to_string(entry.unwrap().path()).unwrap()) @@ -1249,14 +1393,14 @@ async fn identical_native_ids_from_different_sources_are_isolated() { .unwrap() .lines() .count(), - 2 + 1 ); assert_eq!( std::fs::read_to_string(claude_journal) .unwrap() .lines() .count(), - 2 + 1 ); let rows = std::fs::read_to_string(data_dir.join("spans/shared-native-id.ndjson")).unwrap(); @@ -1292,8 +1436,10 @@ async fn distinct_sessions_are_isolated() { let a = std::fs::read_to_string(source_journal_path(&data_dir, "debug", "a")).unwrap(); let b = std::fs::read_to_string(source_journal_path(&data_dir, "debug", "b")).unwrap(); - assert_eq!(a.lines().filter(|l| !l.trim().is_empty()).count(), 3); - assert_eq!(b.lines().filter(|l| !l.trim().is_empty()).count(), 2); + assert!(a.contains("_bt_wal_base")); + assert!(!a.contains("SessionStart")); + assert!(b.contains("_bt_wal_base")); + assert!(!b.contains("SessionStart")); handle.abort(); } @@ -1401,7 +1547,9 @@ async fn opencode_lifecycle_flushes_without_an_explicit_client_flush() { #[tokio::test] async fn hook_capture_stops_at_the_durable_journal_boundary() { - let (socket, handle, tmp) = start_slow_daemon().await; + let gate = Arc::new(tokio::sync::Notify::new()); + let emitted = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let (data_dir, socket, handle, _tmp) = start_gated_daemon(gate.clone(), emitted.clone()).await; let host = dummy_host(); forward_envelope( &envelope("fast-capture", "SessionStart", 1), @@ -1416,7 +1564,7 @@ async fn hook_capture_stops_at_the_durable_journal_boundary() { let mut turn_end = envelope("fast-capture", "Stop", 2); turn_end.route.as_mut().unwrap().flush_mode = bt_daemon::wire::FlushMode::FlushOnTurnEnd; let accepted = tokio::time::timeout( - Duration::from_millis(100), + Duration::from_secs(5), forward_envelope(&turn_end, &socket, &host, false), ) .await; @@ -1424,17 +1572,15 @@ async fn hook_capture_stops_at_the_durable_journal_boundary() { matches!(accepted, Ok(Ok(()))), "turn-end capture waited for translation or flushing: {accepted:?}" ); - let journal = std::fs::read_to_string(source_journal_path( - &tmp.path().join("data"), - "debug", - "fast-capture", - )) - .unwrap(); + let journal = + std::fs::read_to_string(source_journal_path(&data_dir, "debug", "fast-capture")).unwrap(); + assert!(journal.contains("Stop"), "hook returned before journaling"); assert_eq!( - journal.lines().count(), - 2, - "hook returned before journaling" + emitted.load(std::sync::atomic::Ordering::Relaxed), + 0, + "capture must complete while span delivery is still blocked" ); + gate.notify_one(); shutdown(&socket).await; handle.await.unwrap(); } @@ -1655,8 +1801,8 @@ async fn shutdown_drains_then_retries_late_capture_on_replacement() { std::fs::read_to_string(source_journal_path(&data_dir, "debug", "drained-shutdown")) .unwrap(); assert!( - journal.contains(r#""n":2"#), - "replacement daemon did not journal the retried event: {journal}" + journal.contains("_bt_wal_base"), + "replacement daemon did not consume the retried event: {journal}" ); } @@ -1697,7 +1843,7 @@ async fn spawn_on_demand_runs_the_real_standalone_daemon() { } #[tokio::test] -async fn cold_worker_rebuilds_acknowledged_journal_without_redelivery() { +async fn cold_worker_restores_continuation_after_input_collection_without_redelivery() { let (data_dir, socket, first, tmp) = start_daemon().await; let host = dummy_host(); forward_envelope( @@ -1712,13 +1858,11 @@ async fn cold_worker_rebuilds_acknowledged_journal_without_redelivery() { shutdown(&socket).await; first.await.unwrap(); - // A cold worker receives only the saved journal, not the first worker's - // daemon state. Its delivery checkpoint must travel with that journal. + // Restore the recovery cache including SQLite continuation and consumer cursors. let recovered_data_dir = tmp.path().join("recovered-data"); - let source = source_journal_path(&data_dir, "debug", "resume"); + ledger::copy_tree(&data_dir, &recovered_data_dir); + std::fs::remove_dir_all(recovered_data_dir.join("spans")).unwrap(); let recovered_journal = source_journal_path(&recovered_data_dir, "debug", "resume"); - std::fs::create_dir_all(recovered_journal.parent().unwrap()).unwrap(); - std::fs::copy(source, &recovered_journal).unwrap(); let second = start_daemon_at(recovered_data_dir.clone(), socket.clone()).await; forward_envelope(&envelope("resume", "Stop", 2), &socket, &host, false) @@ -1729,8 +1873,8 @@ async fn cold_worker_rebuilds_acknowledged_journal_without_redelivery() { let journal = std::fs::read_to_string(recovered_journal).unwrap(); assert_eq!( journal.lines().count(), - 4, - "two events plus two delivery checkpoints" + 1, + "consumed input has been collected" ); let spans = std::fs::read_to_string(recovered_data_dir.join("spans/resume.ndjson")).unwrap(); let rows: Vec = spans @@ -1858,6 +2002,148 @@ async fn span_plugins_transform_live_and_replayed_rows_with_daemon_environment() second.await.unwrap(); } +#[tokio::test] +async fn translation_failure_preserves_input_but_does_not_block_produced_spans() { + let (data_dir, socket, handle, _tmp) = start_daemon().await; + for (index, (name, payload)) in [ + ("session_start", serde_json::json!({"reason":"new"})), + ( + "before_agent_start", + serde_json::json!({"prompt":"already translated"}), + ), + ( + "tool_execution_start", + serde_json::json!({"toolName":"bash"}), + ), + ] + .into_iter() + .enumerate() + { + let mut env = envelope("source-pause", name, index as i64); + env.source = "pi".into(); + env.payload = payload; + forward_envelope(&env, &socket, &dummy_host(), false) + .await + .unwrap(); + } + let result = flush_session("source-pause", &socket, 5000).await.unwrap(); + assert!(result.pending > 0, "translation remains paused"); + let spans = std::fs::read_to_string(data_dir.join("spans/source-pause.ndjson")).unwrap(); + assert!( + spans.contains("already translated"), + "committed spans must still reach delivery" + ); + let raw = + std::fs::read_to_string(source_journal_path(&data_dir, "pi", "source-pause")).unwrap(); + assert!(raw.contains("tool_execution_start")); + assert!( + !raw.contains("already translated"), + "completed input has been collected" + ); + shutdown(&socket).await; + handle.await.unwrap(); +} + +#[tokio::test] +async fn known_bad_pi_shape_pauses_its_session_while_unknown_events_are_journaled() { + let (data_dir, socket, handle, _tmp) = start_daemon().await; + let mut unknown = envelope("pi-unknown", "future_native_event", 1); + unknown.source = "pi".into(); + unknown.payload = serde_json::json!({"event": {"type": "future_native_event", "new": 1}}); + forward_envelope(&unknown, &socket, &dummy_host(), false) + .await + .unwrap(); + + let mut malformed = envelope("pi-bad-shape", "tool_execution_start", 2); + malformed.source = "pi".into(); + malformed.payload = serde_json::json!({ + "event": {"type": "tool_execution_start", "toolName": "read"} + }); + forward_envelope(&malformed, &socket, &dummy_host(), false) + .await + .unwrap(); + let mut later = envelope("pi-bad-shape", "session_shutdown", 3); + later.source = "pi".into(); + later.payload = serde_json::json!({"event": {"type": "session_shutdown"}}); + forward_envelope(&later, &socket, &dummy_host(), false) + .await + .unwrap(); + for _ in 0..100 { + if std::fs::read_to_string(data_dir.join("diagnostics/recovery.json")) + .is_ok_and(|value| value.contains("\"kind\": \"input_shape\"")) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let incident = std::fs::read_to_string(data_dir.join("diagnostics/recovery.json")).unwrap(); + assert!(incident.contains("\"kind\": \"input_shape\""), "{incident}"); + assert!(incident.contains("tool_execution_start"), "{incident}"); + let journal = + std::fs::read_to_string(source_journal_path(&data_dir, "pi", "pi-bad-shape")).unwrap(); + assert!(journal.contains("tool_execution_start")); + assert!(journal.contains("session_shutdown")); + let unknown_journal = + std::fs::read_to_string(source_journal_path(&data_dir, "pi", "pi-unknown")).unwrap(); + assert!(unknown_journal.contains("_bt_wal_base")); + assert!( + !unknown_journal.contains("future_native_event"), + "ignored input has been consumed" + ); + assert!(!incident.contains("pi-unknown"), "{incident}"); + + shutdown(&socket).await; + handle.await.unwrap(); +} + +#[tokio::test] +async fn recognized_codex_and_cursor_shape_mismatches_pause_their_sessions() { + let (data_dir, socket, handle, _tmp) = start_daemon().await; + let mut codex = envelope("codex-bad-shape", "PostCompact", 1); + codex.source = "codex".into(); + codex.payload = serde_json::json!({"session_id":"codex-bad-shape"}); + forward_envelope(&codex, &socket, &dummy_host(), false) + .await + .unwrap(); + + let mut cursor = envelope("cursor-bad-shape", "preToolUse", 2); + cursor.source = "cursor".into(); + cursor.payload = serde_json::json!({"tool_name":"Read"}); + forward_envelope(&cursor, &socket, &dummy_host(), false) + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let raw = std::fs::read_to_string(data_dir.join("diagnostics/recovery.json")) + .unwrap_or_default(); + if raw.matches("input_shape").count() >= 2 { + break raw; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "recognized event shape failures should pause both source sessions: {}", + std::fs::read_to_string(data_dir.join("diagnostics/recovery.json")) + .unwrap_or_else(|error| format!("diagnostic unavailable: {error}")) + ) + }); + let journal = + std::fs::read_to_string(source_journal_path(&data_dir, "codex", "codex-bad-shape")) + .unwrap(); + assert!(journal.contains("PostCompact")); + let journal = + std::fs::read_to_string(source_journal_path(&data_dir, "cursor", "cursor-bad-shape")) + .unwrap(); + assert!(journal.contains("preToolUse")); + + shutdown(&socket).await; + handle.await.unwrap(); +} + #[tokio::test] async fn a_failing_span_plugin_pauses_only_its_session_and_recovers_on_edit() { let (data_dir, socket, handle, tmp) = start_daemon_with_session_ttl(1).await; @@ -1950,7 +2236,10 @@ async fn a_failing_span_plugin_pauses_only_its_session_and_recovers_on_edit() { assert!(diagnostics.contains("Error: raw local secret")); assert!(diagnostics.contains("bad.mjs")); assert!(diagnostics.contains("plugin-failure")); - assert!(diagnostics.contains("\"state\": \"paused\"")); + let incident: Value = + serde_json::from_slice(&std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap()) + .unwrap(); + assert_eq!(incident["active"][0]["state"], "paused"); std::fs::write( &later_plugin, @@ -1999,7 +2288,7 @@ async fn a_failing_span_plugin_pauses_only_its_session_and_recovers_on_edit() { loop { let spans = std::fs::read_to_string(data_dir.join("spans/plugin-failure.ndjson")).unwrap(); - if spans.contains("changed:") { + if spans.contains("changed:") && spans.contains("Stop") { break; } tokio::time::sleep(Duration::from_millis(50)).await; @@ -2011,17 +2300,28 @@ async fn a_failing_span_plugin_pauses_only_its_session_and_recovers_on_edit() { loop { let spans = std::fs::read_to_string(data_dir.join("spans/plugin-failure.ndjson")).unwrap(); - let diagnostics = - std::fs::read_to_string(data_dir.join("diagnostics/span-plugin-errors.json")) - .unwrap(); - if spans.contains("Stop") && diagnostics.contains("\"state\": \"recovered\"") { + let recovery: Value = serde_json::from_slice( + &std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap(), + ) + .unwrap(); + if spans.contains("Stop") + && recovery["active"].as_array().is_some_and(Vec::is_empty) + { break; } tokio::time::sleep(Duration::from_millis(50)).await; } }) .await - .expect("successful replay should be reflected in spans and diagnostics"); + .unwrap_or_else(|_| { + panic!( + "successful replay should be reflected in spans and recovery diagnostics: spans={} recovery={}", + std::fs::read_to_string(data_dir.join("spans/plugin-failure.ndjson")) + .unwrap_or_default(), + std::fs::read_to_string(data_dir.join("diagnostics/recovery.json")) + .unwrap_or_default() + ) + }); let status = run_status(StatusArgs { socket: Some(socket.clone()), session_id: Some("plugin-failure".into()), @@ -2038,6 +2338,57 @@ async fn a_failing_span_plugin_pauses_only_its_session_and_recovers_on_edit() { handle.await.unwrap(); } +#[tokio::test] +async fn plugin_replay_starts_at_the_failed_operation_within_an_event() { + let (data_dir, socket, handle, tmp) = start_daemon().await; + let plugin = tmp.path().join("second-op.mjs"); + std::fs::write( + &plugin, + "export default span => { if (span.name === 'SessionStart') throw new Error('second op'); return span; }", + ) + .unwrap(); + let mut event = envelope("op-cursor", "SessionStart", 1); + event + .route + .as_mut() + .unwrap() + .span_plugins + .push(plugin.clone()); + forward_envelope(&event, &socket, &dummy_host(), false) + .await + .unwrap(); + let _ = flush_session("op-cursor", &socket, 5000).await.unwrap(); + let incident: Value = + serde_json::from_slice(&std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap()) + .unwrap(); + assert_eq!(incident["active"][0]["operation_index"], 1); + + std::fs::write(&plugin, "export default span => span").unwrap(); + tokio::time::timeout(Duration::from_secs(6), async { + loop { + let incident: Value = serde_json::from_slice( + &std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap(), + ) + .unwrap(); + if incident["active"].as_array().is_some_and(Vec::is_empty) { + break; + } + tokio::time::sleep(Duration::from_millis(30)).await; + } + }) + .await + .expect("plugin retry did not complete"); + let ops = std::fs::read_to_string(data_dir.join("spans/op-cursor.ndjson")).unwrap(); + let root_inserts = ops + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|op| op["Insert"]["name"] == "debug: op-cursor") + .count(); + assert_eq!(root_inserts, 1, "acknowledged prefix must not replay"); + shutdown(&socket).await; + handle.await.unwrap(); +} + #[tokio::test] async fn a_paused_plugin_session_recovers_after_process_restart_without_a_new_event() { let (data_dir, socket, handle, tmp) = start_daemon().await; @@ -2063,16 +2414,42 @@ async fn a_paused_plugin_session_recovers_after_process_restart_without_a_new_ev shutdown(&socket).await; handle.await.unwrap(); + let captured = std::fs::read_to_string(source_journal_path( + &data_dir, + "debug", + "plugin-cold-recovery", + )) + .unwrap(); + assert!( + !captured.contains("SessionStart"), + "plugin recovery must not retain translated input" + ); + let pending = + rusqlite::Connection::open(ledger::database(&data_dir, "debug", "plugin-cold-recovery")) + .unwrap() + .query_row("SELECT COUNT(*) FROM batches", [], |r| r.get::<_, i64>(0)) + .unwrap(); + assert!(pending > 0, "the span ledger protects paused delivery"); + std::fs::write(&plugin, "export default span => span").unwrap(); let restarted = start_daemon_at(data_dir.clone(), socket.clone()).await; tokio::time::timeout(Duration::from_secs(6), async { loop { let spans = std::fs::read_to_string(data_dir.join("spans/plugin-cold-recovery.ndjson")) .unwrap(); - let diagnostics = - std::fs::read_to_string(data_dir.join("diagnostics/span-plugin-errors.json")) - .unwrap(); - if spans.contains("SessionStart") && diagnostics.contains("\"state\": \"recovered\"") { + let incident: Value = serde_json::from_slice( + &std::fs::read(data_dir.join("diagnostics/recovery.json")).unwrap(), + ) + .unwrap(); + if incident["active"].as_array().is_some_and(Vec::is_empty) { + let last = spans + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter_map(|op| op.get("Insert").or_else(|| op.get("Merge")).cloned()) + .next_back() + .expect("replayed span should be present"); + assert_ne!(last["name"], "Plugin failure"); + assert!(last.get("error").is_none()); break; } tokio::time::sleep(Duration::from_millis(50)).await; @@ -3018,6 +3395,7 @@ async fn packaged_grok_hook_replays_bounded_transcripts_to_isolated_debug_routes "stop alone must not masquerade as the terminal flush" ); + let first_capture = ledger::catalog(&data_dir, "grok", "grok-primary")[0]["envelope"].clone(); let primary_spans_path = data_dir.join("spans/grok-primary.ndjson"); let first_ops = read_span_ops(&primary_spans_path); let first_llm = first_ops @@ -3139,11 +3517,10 @@ async fn packaged_grok_hook_replays_bounded_transcripts_to_isolated_debug_routes assert!(!journal_text.contains("secret-environment")); assert!(!journal_text.contains("token_sha256_prefix")); assert!(!journal_text.contains("decoy-route")); - let journal: Vec = journal_text - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .filter(|record: &serde_json::Value| record["_bt_record_type"] != "delivery_checkpoint") - .collect(); + let journal = [ + first_capture, + ledger::catalog(&data_dir, "grok", "grok-primary")[0]["envelope"].clone(), + ]; assert_eq!(journal.len(), 2); assert_eq!(journal[0]["source"], "grok"); assert_eq!(journal[1]["source"], "grok"); diff --git a/bt-daemon/tests/support/ledger.rs b/bt-daemon/tests/support/ledger.rs new file mode 100644 index 00000000..df4df243 --- /dev/null +++ b/bt-daemon/tests/support/ledger.rs @@ -0,0 +1,33 @@ +#![allow(dead_code)] +use serde_json::Value; +use std::path::{Path, PathBuf}; + +pub fn catalog(data: &Path, source: &str, session: &str) -> Value { + let journal = bt_daemon::source_journal_path(data, source, session); + let path = data + .join("journal-control") + .join(journal.file_name().unwrap()) + .with_extension("routes.json"); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() +} + +pub fn database(data: &Path, source: &str, session: &str) -> PathBuf { + let journal = bt_daemon::source_journal_path(data, source, session); + let stem = journal.file_stem().unwrap().to_str().unwrap(); + data.join("derived") + .join(stem.rsplit("--").next().unwrap()) + .join("spans.sqlite") +} + +pub fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_tree(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +}