diff --git a/cli/src/services/sync/progress.rs b/cli/src/services/sync/progress.rs index f6858dd4e..d30d9010c 100644 --- a/cli/src/services/sync/progress.rs +++ b/cli/src/services/sync/progress.rs @@ -35,7 +35,7 @@ impl ProgressReporter for NoopProgressReporter { fn report(&mut self, _event: E) {} } -const STREAM_LABELS: [&str; 4] = ["messages", "parts", "diff_traces", "agent_traces"]; +const STREAM_LABELS: [&str; 3] = ["messages", "parts", "agent_traces"]; const STREAM_LABEL_WIDTH: usize = 15; const STEADY_TICK_INTERVAL: Duration = Duration::from_millis(100); @@ -270,8 +270,21 @@ mod tests { let output = String::from_utf8(output).expect("progress output should be UTF-8"); assert!(output.contains("messages ")); assert!(output.contains("parts ")); - assert!(output.contains("diff_traces ")); assert!(output.contains("agent_traces ")); + assert!(!output.contains("diff_traces")); + let first_render = output + .lines() + .take(STREAM_LABELS.len() + 1) + .map(|line| { + line.split_whitespace() + .nth(1) + .expect("progress row has a stream label") + }) + .collect::>(); + assert_eq!( + first_render, + vec!["messages", "parts", "agent_traces", "messages"] + ); assert!(output.contains("messages 500 rows uploaded")); assert!(output.contains("parts 0 rows uploaded")); assert!(output.contains("✓ parts 0 rows uploaded")); diff --git a/cli/src/services/sync/render_sync.rs b/cli/src/services/sync/render_sync.rs index 65b5b2585..bc2d90347 100644 --- a/cli/src/services/sync/render_sync.rs +++ b/cli/src/services/sync/render_sync.rs @@ -22,7 +22,6 @@ fn render_text(report: &AgentTraceSyncReport) -> String { let uploaded = [ report.streams.messages.uploaded, report.streams.parts.uploaded, - report.streams.diff_traces.uploaded, report.streams.agent_traces.uploaded, ]; let heading = if uploaded.iter().all(|count| *count == 0) { @@ -126,4 +125,41 @@ mod tests { assert!(value.get("diff_traces").is_none()); assert!(value.get("agent_traces").is_none()); } + + fn report_with_uploads( + messages: usize, + parts: usize, + diff_traces: usize, + agent_traces: usize, + ) -> AgentTraceSyncReport { + let mut report = sample_report(); + report.streams.messages.uploaded = messages; + report.streams.parts.uploaded = parts; + report.streams.diff_traces.uploaded = diff_traces; + report.streams.agent_traces.uploaded = agent_traces; + report + } + + #[test] + fn text_heading_is_already_synced_when_three_active_streams_uploaded_nothing() { + assert_eq!( + render_text(&report_with_uploads(0, 0, 0, 0)), + style::heading(ALREADY_SYNCED_HEADING) + ); + assert_eq!( + render_text(&report_with_uploads(0, 0, 1, 0)), + style::heading(ALREADY_SYNCED_HEADING) + ); + } + + #[test] + fn text_heading_is_complete_when_any_active_stream_uploaded_rows() { + for report in [ + report_with_uploads(1, 0, 0, 0), + report_with_uploads(0, 1, 0, 0), + report_with_uploads(0, 0, 0, 1), + ] { + assert_eq!(render_text(&report), style::heading(COMPLETE_HEADING)); + } + } } diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index 6b5b73dad..3658d9e6e 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -1,8 +1,3 @@ -//! `sce sync` orchestration: resolves the current repository's Agent -//! Trace storage, fetches authoritative control-plane cursors, then -//! synchronizes each of the four independent capture streams in the fixed -//! `messages -> parts -> diff_traces -> agent_traces` order. -//! //! Consumes the already-shipped [`crate::services::agent_trace_sync`] engine //! and [`crate::services::agent_trace_sync::control_plane`] client as-is; adds //! no local sync cursor or persisted progress of its own. @@ -36,7 +31,6 @@ use crate::services::sync::progress::{NoopProgressReporter, ProgressReporter}; static SYNC_RUNTIME: OnceLock = OnceLock::new(); -/// Full sync result across all four capture streams, ready for rendering. #[derive(Clone, Debug, PartialEq, Eq)] pub struct AgentTraceSyncReport { pub repository_id: String, @@ -61,7 +55,6 @@ pub struct StreamSyncReport { pub batches: usize, } -/// Progress emitted while the four trace streams are synchronized. #[derive(Clone, Debug, PartialEq, Eq)] pub enum SyncProgressEvent { Started { @@ -159,10 +152,6 @@ impl TraceSyncError { } } -/// Resolves the current repository's Agent Trace storage (the same -/// `ContextWithRepoRoot`/`AgentTraceStorageContext`/`resolve_agent_trace_storage` -/// path used by the sync command) and control-plane configuration, then -/// synchronizes all four capture streams. #[allow(dead_code)] pub fn run_current_sync(repo_root: &Path) -> Result { let mut progress = NoopProgressReporter; @@ -236,9 +225,6 @@ where ) } -/// Testable core: synchronizes all four streams given an already-resolved -/// repository identity, an open Agent Trace database, and a configured -/// control-plane client (production or test-double). #[cfg(test)] pub(crate) fn run_sync_against( repository_id: &str, @@ -338,7 +324,14 @@ where .map_err(TraceSyncError::ControlPlane)?; let progress = Rc::new(RefCell::new(progress)); - let (messages, parts, diff_traces, agent_traces) = try_join_four( + let diff_traces = StreamSyncReport { + uploaded: 0, + initial_cursor: state.cursors.diff_traces, + final_cursor: state.cursors.diff_traces, + batches: 0, + }; + + let (messages, parts, agent_traces) = try_join_three( sync_one_stream( client, repository_id, @@ -361,17 +354,6 @@ where |request| Box::pin(async move { client.ingest_parts(&request).await }), Rc::clone(&progress), ), - sync_one_stream( - client, - repository_id, - source_instance_id, - IngestionStream::DiffTraces, - state.cursors.diff_traces, - "diff_traces", - |cursor, limit| reader.read_diff_traces_after(cursor, limit), - |request| Box::pin(async move { client.ingest_diff_traces(&request).await }), - Rc::clone(&progress), - ), sync_one_stream( client, repository_id, @@ -398,26 +380,18 @@ where }) } -async fn try_join_four( - a: A, - b: B, - c: C, - d: D, -) -> Result<(OA, OB, OC, OD), E> +async fn try_join_three(a: A, b: B, c: C) -> Result<(OA, OB, OC), E> where A: Future>, B: Future>, C: Future>, - D: Future>, { let mut a = Box::pin(a); let mut b = Box::pin(b); let mut c = Box::pin(c); - let mut d = Box::pin(d); let mut a_output = None; let mut b_output = None; let mut c_output = None; - let mut d_output = None; poll_fn(|context| { if a_output.is_none() { @@ -435,24 +409,13 @@ where c_output = Some(result?); } } - if d_output.is_none() { - if let Poll::Ready(result) = d.as_mut().poll(context) { - d_output = Some(result?); - } - } - match ( - a_output.take(), - b_output.take(), - c_output.take(), - d_output.take(), - ) { - (Some(a), Some(b), Some(c), Some(d)) => Poll::Ready(Ok((a, b, c, d))), - (a, b, c, d) => { + match (a_output.take(), b_output.take(), c_output.take()) { + (Some(a), Some(b), Some(c)) => Poll::Ready(Ok((a, b, c))), + (a, b, c) => { a_output = a; b_output = b; c_output = c; - d_output = d; Poll::Pending } } @@ -633,9 +596,10 @@ mod tests { AgentTraceInsert, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, MessageRole, PartType, PAYLOAD_TYPE_PATCH, }; + use crate::services::agent_trace_export::JS_MAX_SAFE_INTEGER; use crate::services::agent_trace_sync::control_plane::CredentialStore; use crate::services::agent_trace_sync::test_http_server::{ - CannedResponse, ConcurrentBatchTestServer, TestHttpServer, + CannedResponse, CapturedRequest, ConcurrentBatchTestServer, TestHttpServer, }; use crate::services::auth::TokenResponse; use crate::services::token_storage::StoredTokens; @@ -753,6 +717,106 @@ mod tests { .expect("seed progress messages"); } + #[derive(Clone, Debug, PartialEq, Eq)] + struct DiffTraceRowSnapshot { + id: i64, + time_ms: i64, + session_id: String, + patch: String, + created_at: String, + model_id: Option, + tool_name: Option, + tool_version: Option, + payload_type: String, + } + + fn diff_trace_rows(db: &RepositoryAgentTraceDb) -> Vec { + db.query_map( + "SELECT id, time_ms, session_id, patch, created_at, model_id, tool_name, tool_version, payload_type FROM diff_traces ORDER BY id ASC", + (), + |row| { + Ok(DiffTraceRowSnapshot { + id: row.get(0)?, + time_ms: row.get(1)?, + session_id: row.get(2)?, + patch: row.get(3)?, + created_at: row.get(4)?, + model_id: row.get(5)?, + tool_name: row.get(6)?, + tool_version: row.get(7)?, + payload_type: row.get(8)?, + }) + }, + ) + .expect("diff_traces snapshot query should succeed") + } + + fn insert_diff_trace_row(db: &RepositoryAgentTraceDb, id: i64, time_ms: i64) { + db.execute( + "INSERT INTO diff_traces (id, time_ms, session_id, patch, model_id, tool_name, tool_version, payload_type) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + ( + id, + time_ms, + "sess-1", + "Index: a\n", + Option::<&str>::None, + Option::<&str>::None, + Option::<&str>::None, + PAYLOAD_TYPE_PATCH, + ), + ) + .expect("direct diff_trace insert should succeed"); + } + + fn diff_traces_compatibility_report(server_cursor: i64) -> StreamSyncReport { + StreamSyncReport { + uploaded: 0, + initial_cursor: server_cursor, + final_cursor: server_cursor, + batches: 0, + } + } + + fn assert_state_request( + request: &CapturedRequest, + repository_id: &str, + source_instance_id: &str, + ) { + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/agent-trace/ingestion/state"); + assert_eq!( + serde_json::from_str::(&request.body) + .expect("state request body should be JSON"), + json!({ + "repositoryId": repository_id, + "sourceInstanceId": source_instance_id, + }) + ); + } + + fn state_request_count(requests: &[CapturedRequest]) -> usize { + requests + .iter() + .filter(|request| request.path == "/agent-trace/ingestion/state") + .count() + } + + fn sorted_batch_streams(requests: &[CapturedRequest]) -> Vec { + let mut streams = requests + .iter() + .filter(|request| request.path == "/agent-trace/ingestion/batch") + .map(|request| { + serde_json::from_str::(&request.body) + .expect("batch request body should be JSON")["stream"] + .as_str() + .expect("batch request should name its stream") + .to_string() + }) + .collect::>(); + streams.sort(); + streams + } + fn state_response( messages: i64, parts: i64, @@ -856,12 +920,6 @@ mod tests { cursor: 0, batches: 0, }, - SyncProgressEvent::StreamCompleted { - stream: "diff_traces", - uploaded: 0, - cursor: 0, - batches: 0, - }, SyncProgressEvent::StreamCompleted { stream: "agent_traces", uploaded: 0, @@ -902,20 +960,22 @@ mod tests { } #[test] - fn full_sync_uploads_all_four_streams_and_second_run_is_naturally_incremental() { + fn full_sync_uploads_three_remote_streams_and_second_run_is_naturally_incremental() { let db_path = unique_test_db_path("full-sync"); let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); let metadata = db .verify_or_initialize_repository_metadata("repo-full-sync") .expect("metadata should initialize"); seed_one_row_per_stream(&db); + let diff_traces_before = diff_trace_rows(&db); + assert_eq!(diff_traces_before.len(), 1); + let unsynced_diff_traces = diff_traces_compatibility_report(0); let server = TestHttpServer::start(); server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); server.queue_response(CannedResponse::json(200, &batch_response(1))); server.queue_response(CannedResponse::json(200, &batch_response(1))); server.queue_response(CannedResponse::json(200, &batch_response(1))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); let client = test_client(&server); let report = run_sync_against( @@ -926,11 +986,21 @@ mod tests { ) .expect("first sync should succeed"); - assert_eq!(server.call_count(), 5); + let first_run_requests = server.captured_requests(); + assert_eq!(first_run_requests.len(), 4); + assert_state_request( + &first_run_requests[0], + &metadata.repository_id, + &metadata.source_instance_id, + ); + assert_eq!(state_request_count(&first_run_requests), 1); + assert_eq!( + sorted_batch_streams(&first_run_requests), + vec!["agent_traces", "messages", "parts"] + ); for stream in [ report.streams.messages, report.streams.parts, - report.streams.diff_traces, report.streams.agent_traces, ] { assert_eq!(stream.uploaded, 1); @@ -938,8 +1008,10 @@ mod tests { assert_eq!(stream.final_cursor, 1); assert_eq!(stream.batches, 1); } + assert_eq!(report.streams.diff_traces, unsynced_diff_traces); + assert_eq!(diff_trace_rows(&db), diff_traces_before); - server.queue_response(CannedResponse::json(200, &state_response(1, 1, 1, 1))); + server.queue_response(CannedResponse::json(200, &state_response(1, 1, 0, 1))); let second_report = run_sync_against( &metadata.repository_id, @@ -949,15 +1021,21 @@ mod tests { ) .expect("second sync should succeed"); + let all_requests = server.captured_requests(); assert_eq!( - server.call_count(), - 6, + all_requests.len(), + 5, "second run should only re-check /state and re-read no already-synced rows" ); + assert_eq!(all_requests[4].path, "/agent-trace/ingestion/state"); + assert_eq!( + sorted_batch_streams(&all_requests), + vec!["agent_traces", "messages", "parts"] + ); + assert!(!sorted_batch_streams(&all_requests).contains(&"diff_traces".to_string())); for stream in [ second_report.streams.messages, second_report.streams.parts, - second_report.streams.diff_traces, second_report.streams.agent_traces, ] { assert_eq!(stream.uploaded, 0); @@ -965,6 +1043,8 @@ mod tests { assert_eq!(stream.final_cursor, 1); assert_eq!(stream.batches, 0); } + assert_eq!(second_report.streams.diff_traces, unsynced_diff_traces); + assert_eq!(diff_trace_rows(&db), diff_traces_before); let db_file_name = db_path .file_name() @@ -992,7 +1072,81 @@ mod tests { } #[test] - fn concurrent_sync_overlaps_all_four_stream_batches_after_one_state_request() { + fn sync_succeeds_when_local_diff_trace_row_is_rejected_by_export_reader() { + let db_path = unique_test_db_path("rejected-diff-trace"); + let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); + let metadata = db + .verify_or_initialize_repository_metadata("repo-rejected-diff-trace") + .expect("metadata should initialize"); + seed_messages(&db, 1); + insert_diff_trace_row(&db, 1, JS_MAX_SAFE_INTEGER + 1); + let diff_traces_before = diff_trace_rows(&db); + + let server = TestHttpServer::start(); + server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); + server.queue_response(CannedResponse::json(200, &batch_response(1))); + let client = test_client(&server); + + let report = run_sync_against( + &metadata.repository_id, + &metadata.source_instance_id, + &db, + &client, + ) + .expect("sync should not read local diff_traces rows"); + + let requests = server.captured_requests(); + assert_eq!(requests.len(), 2); + assert_eq!(sorted_batch_streams(&requests), vec!["messages"]); + assert_eq!(report.streams.messages.uploaded, 1); + assert_eq!( + report.streams.diff_traces, + diff_traces_compatibility_report(0) + ); + assert_eq!(diff_trace_rows(&db), diff_traces_before); + + remove_test_db(&db_path); + } + + #[test] + fn diff_traces_report_echoes_server_cursor_without_uploading_newer_local_rows() { + let db_path = unique_test_db_path("diff-traces-compat"); + let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); + let metadata = db + .verify_or_initialize_repository_metadata("repo-diff-traces-compat") + .expect("metadata should initialize"); + seed_one_row_per_stream(&db); + insert_diff_trace_row(&db, 124, 1_700_000_000_124); + let diff_traces_before = diff_trace_rows(&db); + assert_eq!(diff_traces_before.len(), 2); + + let server = TestHttpServer::start(); + server.queue_response(CannedResponse::json(200, &state_response(1, 1, 123, 1))); + let client = test_client(&server); + + let report = run_sync_against( + &metadata.repository_id, + &metadata.source_instance_id, + &db, + &client, + ) + .expect("sync should succeed"); + + let requests = server.captured_requests(); + assert_eq!(requests.len(), 1); + assert_eq!(state_request_count(&requests), 1); + assert!(sorted_batch_streams(&requests).is_empty()); + assert_eq!( + report.streams.diff_traces, + diff_traces_compatibility_report(123) + ); + assert_eq!(diff_trace_rows(&db), diff_traces_before); + + remove_test_db(&db_path); + } + + #[test] + fn concurrent_sync_overlaps_all_three_stream_batches_after_one_state_request() { let db_path = unique_test_db_path("concurrent-overlap"); let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); let metadata = db @@ -1002,7 +1156,7 @@ mod tests { let server = ConcurrentBatchTestServer::start(Duration::from_millis(100)); server.queue_state_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - for stream in ["messages", "parts", "diff_traces", "agent_traces"] { + for stream in ["messages", "parts", "agent_traces"] { server.queue_batch_response(stream, 0, CannedResponse::json(200, &batch_response(1))); } let client = test_client_at(&server.base_url); @@ -1016,16 +1170,17 @@ mod tests { .expect("concurrent sync should succeed"); assert_eq!(server.state_request_count(), 1); - assert_eq!(server.captured_requests().len(), 5); - assert_eq!(server.max_in_flight(), 4); - for stream in ["messages", "parts", "diff_traces", "agent_traces"] { + assert_eq!(server.captured_requests().len(), 4); + assert_eq!(server.max_in_flight(), 3); + for stream in ["messages", "parts", "agent_traces"] { assert_eq!(server.max_in_flight_for(stream), 1, "stream {stream}"); assert_eq!(server.expected_cursors_for(stream), vec![0]); } + assert_eq!(server.max_in_flight_for("diff_traces"), 0); + assert!(server.expected_cursors_for("diff_traces").is_empty()); for stream in [ report.streams.messages, report.streams.parts, - report.streams.diff_traces, report.streams.agent_traces, ] { assert_eq!(stream.uploaded, 1); @@ -1207,7 +1362,7 @@ mod tests { let server = TestHttpServer::start(); server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - for _ in 0..4 { + for _ in 0..3 { server.queue_response(CannedResponse::json( 404, &json!({"message": "unknown ingestion route"}), diff --git a/context/architecture.md b/context/architecture.md index 5c84a1d80..4ed19d5d0 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -150,7 +150,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - Generated Claude settings register `SessionStart` and `PostModelSwitch` only for the local model-state hook; `sce hooks session-model` is no longer a supported hook command. Claude Code 2.1.250 and 2.1.251 compatibility smoke passed with the unknown `PostModelSwitch` registration, so SCE installs it unconditionally without a raised minimum or capability gate. The `session_models` table/API and generic session-level fallback lookup were removed in T02 of the `remove-session-models-direct-claude-model-id` plan; the separate `sce hooks claude-model-state` command writes Claude lifecycle observations into the non-exported exact-scope register through the no-migration hook-runtime DB path, without restoring that generic abstraction. `diff-trace` uses direct-first/event-transcript-second Claude `model_id` resolution and consults the exact local lifecycle state only as its final fallback, with direct `tool_version` values. - `cli/src/services/resilience.rs` defines bounded retry/timeout/backoff execution policy (`RetryPolicy`, `run_with_retry`) for transient operation hardening with deterministic failure messaging and retry observability. - `context/cli/agent-trace-auto-sync.md` documents the sync-owned, one-shot post-commit launcher boundary: it reuses `sce sync`, has no daemon or local retry machinery, and fails open when child startup cannot be completed; doctor reports this capability without invoking the launcher. -- `cli/src/services/sync/progress.rs` owns the sync-local, consumer-typed progress seam: generic `ProgressReporter` supports event delivery plus explicit successful finalization, closure-based collectors, and a no-op implementation alongside the fixed `indicatif` stderr presentation adapter. `cli/src/services/sync/sync.rs` owns `SyncProgressEvent` and its four-stream payload semantics, while `sync/command.rs` selects the terminal adapter for text and the no-op reporter for JSON. There is no top-level `cli/src/services/progress/` module; sync orchestration depends only on its sync-owned contract, so terminal-library details stay at the sync presentation boundary. +- `cli/src/services/sync/progress.rs` owns the sync-local, consumer-typed progress seam: generic `ProgressReporter` supports event delivery plus explicit successful finalization, closure-based collectors, and a no-op implementation alongside the fixed `indicatif` stderr presentation adapter. `cli/src/services/sync/sync.rs` owns `SyncProgressEvent` and its payload semantics for the three remotely synchronized streams (`messages`, `parts`, `agent_traces`; `diff_traces` is captured locally but not synchronized), while `sync/command.rs` selects the terminal adapter for text and the no-op reporter for JSON. There is no top-level `cli/src/services/progress/` module; sync orchestration depends only on its sync-owned contract, so terminal-library details stay at the sync presentation boundary. - `sce sync [--format text|json]` is implemented: `cli/src/services/sync/sync.rs` resolves repository-scoped Agent Trace storage, authenticates against the control plane with stored WorkOS credentials, uses the config-resolved `control_plane_base_url` with baked default `https://sce.crocoderlab.dev`, calls the ingestion `/state` endpoint once, then starts the `messages`/`parts`/`diff_traces`/`agent_traces` capture-stream state machines concurrently via `AgentTraceExportReader` and a shared per-stream reconciliation engine. Batches and cursor refreshes remain sequential within each stream, while fixed stream order is retained for final and stream-completion reporting; `cli/src/services/sync/render_sync.rs` renders the converged `AgentTraceSyncReport` as concise per-stream text or `camelCase` JSON without a nested subcommand field (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization otherwise still flow through lifecycle providers aggregated by setup, while repository-scoped DB health/repair flows through the doctor surface. The former trace database inspection and nested sync surfaces are unavailable. - `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helpers (`load_patch_from_json` for string input, `load_patch_from_json_bytes` for byte input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. - `cli/src/services/structured_patch.rs` defines the synchronous structured editor-hook derivation seam. It derives Claude `PostToolUse` `Write` structured-update hunks, `Write` `tool_input.content` create fallback, and `Edit` structured-patch payloads into canonical `ParsedPatch` values plus Claude session/tool metadata, returning deterministic skip reasons for unsupported events/tools/payload shapes. The module is pure and side-effect-free. It is wired into `sce hooks diff-trace` for Claude payload classification at intake (T04) and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing dispatch at read time (T05). diff --git a/context/cli/agent-trace-sync-command.md b/context/cli/agent-trace-sync-command.md index 6c0b1831d..3ba024b10 100644 --- a/context/cli/agent-trace-sync-command.md +++ b/context/cli/agent-trace-sync-command.md @@ -23,12 +23,35 @@ flowchart LR ``` - **hooks/plugins** write local capture rows (`messages`, `parts`, `diff_traces`, `agent_traces`) into the current repository's `RepositoryAgentTraceDb` during normal Git/editor activity — this is unchanged by sync. -- **`AgentTraceExportReader`** (PR #198) is the read-only local export boundary sync uses to read rows after a cursor; sync never queries the repository DB directly. -- **`sce sync`** resolves repository storage through `agent_trace_storage` (not the hook-runtime resolver), builds an `AuthenticatedControlPlaneClient` from stored WorkOS credentials and the resolved `control_plane_base_url`, and drives one authoritative `/state` call before starting four concurrent stream state machines. Each stream keeps its own batches and reconciliation refreshes sequential and cursor-safe; the bounded per-stream reconciliation loop remains independent. With no environment or config override, that base is `https://sce.crocoderlab.dev`; it is distinct from the `https://sce.crocoder.dev` SCE web and config-schema URL owner. +- **`AgentTraceExportReader`** (PR #198) is the read-only local export boundary sync uses to read rows after a cursor; sync never queries the repository DB directly. The reader exposes four `read_*_after` methods; `sce sync` consumes three and never calls `read_diff_traces_after`. +- **`sce sync`** resolves repository storage through `agent_trace_storage` (not the hook-runtime resolver), builds an `AuthenticatedControlPlaneClient` from stored WorkOS credentials and the resolved `control_plane_base_url`, and drives one authoritative `/state` call before starting three concurrent remote stream state machines (`messages`, `parts`, `agent_traces`). Each stream keeps its own batches and reconciliation refreshes sequential and cursor-safe; the bounded per-stream reconciliation loop remains independent. With no environment or config override, that base is `https://sce.crocoderlab.dev`; it is distinct from the `https://sce.crocoder.dev` SCE web and config-schema URL owner. - **Credential runtime boundary:** `AuthenticatedControlPlaneClient` keeps the synchronous `CredentialStore` behind an `Arc` and runs every token-storage `load`/`save` through `tokio::task::spawn_blocking`. The underlying encrypted auth DB and Linux Secret Service/zbus APIs are blocking and may create their own Tokio runtime, so they must never execute directly inside the async control-plane request future. Token refresh and HTTP requests remain asynchronous; only credential persistence crosses the blocking boundary. The client owns a refresh single-flight guard: expired-token callers re-check credentials after acquiring it, and callers retrying the same rejected access token reuse a token saved by an earlier refresh; valid-token resolution does not acquire the guard. - **control plane** is the sole source of cursor truth: every invocation starts from `POST /agent-trace/ingestion/state`, uploads via `POST /agent-trace/ingestion/batch`, and advances a stream's cursor only from a validated batch response (`accepted == rows.len()` and `cursor == rows.last().sourceRowId`), never by inferring `cursor + rows.len()`. -The four streams (`messages`, `parts`, `diff_traces`, `agent_traces`) start concurrently after the single authoritative state response; batches and cursor-refresh calls remain sequential within each stream. Accepted-batch progress and stream-completion events may arrive as their respective futures complete. Text mode creates four aligned `indicatif` rows immediately on `stderr`, before any accepted batch can be reported, each with a 15-column stream label, a steady spinner, and a `0 rows uploaded` starting message. Accepted batches update only their stream's cumulative count; each stream changes its spinner to a shared-policy styled `✓` with its final count as soon as its own future completes, including an empty stream with zero uploaded rows. After the progress rows, the final text report contains only a status heading: `Agent Trace already synced.` when all four streams uploaded zero rows, otherwise `Agent Trace sync complete.`. Non-TTY output uses stable aligned plain snapshots without ANSI or terminal-control sequences, and `NO_COLOR` disables completion styling. JSON mode uses a no-op progress sink and retains its JSON-only stdout contract without human progress on `stderr`. +The three remote streams (`messages`, `parts`, `agent_traces`) start concurrently after the single authoritative state response; batches and cursor-refresh calls remain sequential within each stream. Accepted-batch progress and stream-completion events may arrive as their respective futures complete. Text mode creates three aligned `indicatif` rows immediately on `stderr`, before any accepted batch can be reported, each with a 15-column stream label, a steady spinner, and a `0 rows uploaded` starting message. Accepted batches update only their stream's cumulative count; each stream changes its spinner to a shared-policy styled `✓` with its final count as soon as its own future completes, including an empty stream with zero uploaded rows. After the progress rows, the final text report contains only a status heading: `Agent Trace already synced.` when the three remote streams uploaded zero rows, otherwise `Agent Trace sync complete.`. Non-TTY output uses stable aligned plain snapshots without ANSI or terminal-control sequences, and `NO_COLOR` disables completion styling. JSON mode uses a no-op progress sink and retains its JSON-only stdout contract without human progress on `stderr`. + +## Local capture streams versus remote streams + +- **Local capture streams (four):** `messages`, `parts`, `diff_traces`, `agent_traces`. +- **Remote `sce sync` streams (three):** `messages`, `parts`, `agent_traces`. + +`diff_traces` stays local with no remote upload. Its table, hook ingress, structured-patch reconstruction, post-commit patch intersection, staged-diff AI-overlap, and attribution are unaffected by sync, and sync neither reads nor changes its rows. + +The client still decodes `cursors.diffTraces` from `/state` as a mandatory field and echoes it in the `streams.diffTraces` JSON entry as a compatibility no-op (see [sync-command.md](sync-command.md#output-contract)). These protocol surfaces are retained, unused by the active sync path, until the compatibility-removal phase: `IngestionStream::DiffTraces`, `AuthenticatedControlPlaneClient::ingest_diff_traces`, `AgentTraceExportReader::read_diff_traces_after`, `AgentTraceDiffTraceExportRow`, the `diff_traces` batch request serialization, and `AgentTraceCursors.diff_traces`. + +### Release gate + +Each step must not start before the one above it is complete. + +```mermaid +flowchart TD + P2["Phase 2: SCE stops POSTing diff_traces batches
still decodes /state.cursors.diffTraces"] --> R[merge and release a new SCE version] + R --> V[verify and adopt it so supported clients no longer upload diff_traces] + V --> P3["Phase 3: Control Plane stops accepting and storing new diff_traces batches
/state keeps returning cursors.diffTraces"] + P3 --> CR["Later compatibility-removal phase:
remove /state.cursors.diffTraces and the retained client/server surfaces"] +``` + +Phase 3 only stops new `diff_traces` batch ingestion and persistence. `/state.cursors.diffTraces` remains required through the compatibility window because the Phase 2 client decodes it as mandatory; a Control Plane that dropped it would break every Phase 2 client. Removing that cursor and the retained surfaces belongs to the later compatibility-removal phase, which removes them from both sides together. The decision is recorded in [Retire diff_traces from remote sync through a staged compatibility gate](../decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md). ## No-local-persistence invariants @@ -61,3 +84,4 @@ Because every invocation starts from the control plane's authoritative `/state` - [auth-db.md](../sce/auth-db.md) — encrypted WorkOS credential storage sync authenticates through. - [CLI error-code taxonomy](../sce/cli-error-code-taxonomy.md) — the `CliError`/`UserError` typed boundary that authentication-failure classification renders through. - [Trace-sync progress stream contract](../decisions/2026-08-13-trace-sync-progress-stream-contract.md) — stderr progress/timestamps and stdout/JSON compatibility boundary. +- [Retire diff_traces from remote sync through a staged compatibility gate](../decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md) — three remote streams, retained compatibility surfaces, and the release gate. diff --git a/context/cli/sync-command.md b/context/cli/sync-command.md index d19426ae3..8f4df4865 100644 --- a/context/cli/sync-command.md +++ b/context/cli/sync-command.md @@ -55,10 +55,17 @@ flowchart LR The command resolves repository storage through `agent_trace_storage`, builds an `AuthenticatedControlPlaneClient` from stored WorkOS credentials and the resolved `control_plane_base_url`, then performs one authoritative `/state` -request before starting the four concurrent stream state machines: -`messages`, `parts`, `diff_traces`, and `agent_traces`. Batches and cursor -refreshes remain sequential within each stream, and final reporting retains the -fixed stream order. +request before starting the three concurrent remote stream state machines: +`messages`, `parts`, and `agent_traces`. Batches and cursor refreshes remain +sequential within each stream, and final reporting retains the fixed stream +order. + +The local database has four capture streams (`messages`, `parts`, +`diff_traces`, `agent_traces`); `sce sync` remotely synchronizes three of them. `diff_traces` +stays local: sync never reads its rows, sends no batch with +`"stream": "diff_traces"`, and emits no progress event for it. See +[Agent Trace sync architecture](agent-trace-sync-command.md#local-capture-streams-versus-remote-streams) +for the release gate that governs this. The control plane is the sole cursor authority. Sync creates no local cursor, `agent-trace-sync.db`, Turso Sync state, `BridgeLock`, or local data warehouse. @@ -68,11 +75,12 @@ authoritative control-plane cursors. ## Output contract The final text report contains only a completion heading after the progress -display. It says `Agent Trace already synced.` when all four streams uploaded -zero rows; otherwise it says `Agent Trace sync complete.`. During text mode, -four progress rows are created immediately -in the fixed order -`messages`, `parts`, `diff_traces`, `agent_traces`. Each row uses a 15-column +display. It says `Agent Trace already synced.` when the three remote streams +(`messages`, `parts`, `agent_traces`) uploaded zero rows; otherwise it says +`Agent Trace sync complete.`. The `diffTraces` compatibility entry does not +affect the heading. During text mode, three progress rows are created +immediately in the fixed order `messages`, `parts`, `agent_traces`; there is no +`diff_traces` row. Each row uses a 15-column stream-label field, starts at `0 rows uploaded`, and has its own steady spinner. Accepted batches update only their stream's cumulative count. A stream replaces its spinner with a styled `✓` and its final count as soon as that stream's @@ -94,6 +102,13 @@ sink, emits no progress on stderr, and emits this JSON-only stdout shape: } ``` +`streams.diffTraces` is a temporary compatibility no-op entry, retained so the +JSON shape does not change during the compatibility window. It always reports +`uploaded: 0` and `batches: 0`, with `initialCursor == finalCursor == +state.cursors.diffTraces` from the `/state` response. It is built from that +server cursor alone, even when local `diff_traces` rows exist beyond it, and it +does not mean the stream is synchronized. + Authentication refresh, conflict reconciliation, ambiguous batch recovery, terminal protocol failures, ownership rejection, and sanitized control-plane errors remain owned by `services::agent_trace_sync` and its control-plane @@ -131,3 +146,4 @@ architecture. - [CLI error-code taxonomy](../sce/cli-error-code-taxonomy.md) - [Trace-sync progress stream contract](../decisions/2026-08-13-trace-sync-progress-stream-contract.md) - [Automatic Agent Trace synchronization](agent-trace-auto-sync.md) +- [Retire diff_traces from remote sync through a staged compatibility gate](../decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md) diff --git a/context/context-map.md b/context/context-map.md index 4287d2f3f..4ad3ca484 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -18,8 +18,8 @@ Feature/domain context: - `context/cli/structured-patch-service.md` (Claude structured editor-hook derivation in `cli/src/services/structured_patch.rs`, including `Write` structured-update hunks, `Write` `tool_input.content` create fallback, `Edit` structured patches, deterministic skip reasons, `ParsedPatch` output semantics, Rust golden fixture coverage, and repository read-time enrichment that assigns persisted row `model_id` to each hunk and canonical row `session_id` to each touched line) - `context/cli/styling-service.md` (CLI text-mode output styling with `owo-colors`, TTY/`NO_COLOR` policy, shared helper API for human-facing surfaces including sync completion markers, and per-column right-to-left RGB gradient banner rendering) - `context/cli/agent-trace-auto-sync.md` (setup-enabled post-commit Agent Trace synchronization with explicit-false opt-out plus doctor readiness reporting: a newly created repo-local config supplies the explicit `true` opt-in while omitted runtime values resolve to `false`; the existing `sce sync` command is launched once through the current executable after local persistence, with detached null-standard-stream behavior, fail-open startup, canonical managed-block proof of hook readiness, stable text/JSON enabled/disabled/not-ready/not-applicable states, no daemon/queue/high-frequency trigger, and retryability through manual sync and control-plane cursor authority) -- `context/cli/sync-command.md` (the top-level `sce sync` command: repository-scoped Agent Trace storage resolution, WorkOS-authenticated four-stream control-plane synchronization through the sync-owned consumer-typed `services::sync::progress` reporter contract with sync-owned events, its generic/no-op contract and `indicatif` presentation adapter for aligned stderr progress with independent stream completion, explicit successful finalization, JSON stdout silence, and rejection of the removed `sce trace` command group) -- `context/cli/agent-trace-sync-command.md` (composed local-to-control-plane `sce sync` architecture: the `hooks/plugins → repository Agent Trace DB → AgentTraceExportReader → sce sync → HTTPS + WorkOS Bearer → control plane` data flow, the `sce auth login` / `cd ` / `sce sync` user flow, the no-local-cursor/no-`agent-trace-sync.db`/no-Turso-Sync/no-`BridgeLock`/no-local-DWH invariants, and `401`/`409`/ambiguous-batch-failure recovery semantics) +- `context/cli/sync-command.md` (the top-level `sce sync` command: repository-scoped Agent Trace storage resolution, WorkOS-authenticated three-stream control-plane synchronization (`messages`, `parts`, `agent_traces`; `diff_traces` stays local with a `streams.diffTraces` compatibility no-op entry) through the sync-owned consumer-typed `services::sync::progress` reporter contract with sync-owned events, its generic/no-op contract and `indicatif` presentation adapter for aligned stderr progress with independent stream completion, explicit successful finalization, JSON stdout silence, and rejection of the removed `sce trace` command group) +- `context/cli/agent-trace-sync-command.md` (composed local-to-control-plane `sce sync` architecture: the `hooks/plugins → repository Agent Trace DB → AgentTraceExportReader → sce sync → HTTPS + WorkOS Bearer → control plane` data flow, the `sce auth login` / `cd ` / `sce sync` user flow, the no-local-cursor/no-`agent-trace-sync.db`/no-Turso-Sync/no-`BridgeLock`/no-local-DWH invariants, the four-local-capture-streams versus three-remote-streams boundary with its `diff_traces` release gate, and `401`/`409`/ambiguous-batch-failure recovery semantics) - `context/cli/config-precedence-contract.md` (implemented `sce config` show/validate command contract, deterministic `flags > env > config file > defaults` resolution order, focused `config/resolver.rs` ownership for config discovery/merge/runtime precedence plus default-discovered invalid-file degradation, focused `config/render.rs` ownership for `show`/`validate` text+JSON output construction, canonical `$schema` acceptance for startup-loaded `sce/config.json` files, shared auth-key env/config/optional baked-default support starting with `workos_client_id`, shared runtime resolution for flat logging observability keys including config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` with `/sce/logs` defaulting plus config-file/default-only positive `log_file_retention_limit`, config-file-only `agent_trace.repository_id`/`agent_trace.repository_remote` repository-identity keys with default remote `origin`, `agent_trace.auto_sync` resolution with omitted-value fallback `false`, explicit-false opt-out, and setup-written explicit `true` bootstrap behavior for the post-commit trigger boundary, the catalog-derived `integrations.optional_workflows` optional-workflow selection key, JSON-pointer-prefixed schema-validation errors, canonical Pkl-generated `sce/config.json` schema ownership plus CLI embedding/reuse contract including `policies.attribution_hooks.enabled` default-true/explicit-false opt-out metadata, config-file selection order, `show` provenance output, and trimmed `validate` output contract) - `context/cli/capability-traits.md` (current broad CLI capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps`, `GitOps`/`ProcessGitOps`, git root/hooks resolution behavior, compile-time-typed borrowed AppContext wiring with associated-type narrow capability accessors plus `ContextWithRepoRoot` repo-root-scoped context derivation, generic command execution bounds, and test-only unimplemented stubs; current service internals do not consume fs/git traits until later lifecycle migration tasks) - `context/cli/service-lifecycle.md` (current compile-safe lifecycle seam in `cli/src/services/lifecycle.rs`, including default no-op `ServiceLifecycle` diagnose/fix/setup methods against narrow `HasRepoRoot`, lifecycle-owned health/fix/setup result types with generic setup messages, doctor/setup adapter boundaries, the static `LifecycleProvider` enum catalog/dispatcher, hook/config/local_db/auth_db/agent_trace_db lifecycle providers including setup-time repository-scoped Agent Trace DB initialization plus checkout identity diagnostics, implemented doctor aggregation over diagnose/fix providers, and implemented setup aggregation over `setup` providers in order config → local_db → auth_db → agent_trace_db → hooks when requested) @@ -84,7 +84,7 @@ Feature/domain context: - `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` with a `<= 2_000ms` default query failure budget, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) - `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, encrypted token-storage persistence, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) - `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by the fresh multi-statement baseline schema plus additive `source_instance_id` and `claude_model_state` migrations, with `repository_metadata` carrying both `repository_id` and a concurrency-safe atomic-claim `source_instance_id` (physical database identity, independent of `repository_id`), narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, parts, and the non-exported exact-scope Claude model-state register, repository-level recent diff-trace reads without checkout filtering including persisted hunk-model plus canonical touched-line-session enrichment for structured rows, on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) -- `context/sce/agent-trace-export-readers.md` (implemented `AgentTraceExportReader<'a>` in `cli/src/services/agent_trace_export/mod.rs`: the read-only local export boundary over `RepositoryAgentTraceDb` — `read_messages_after`/`read_parts_after`/`read_diff_traces_after`/`read_agent_traces_after`, each cursor/limit/JS-safe-integer validated, materialized into owned camelCase `serde::Serialize` DTOs; composes directly with `ResolvedAgentTraceStorage` without owning `source_instance_id`; no local sync cursor, no `agent-trace-sync.db`, no Turso Sync, no ETL, no DWH) +- `context/sce/agent-trace-export-readers.md` (implemented `AgentTraceExportReader<'a>` in `cli/src/services/agent_trace_export/mod.rs`: the read-only local export boundary over `RepositoryAgentTraceDb` — `read_messages_after`/`read_parts_after`/`read_diff_traces_after`/`read_agent_traces_after`, each cursor/limit/JS-safe-integer validated, materialized into owned camelCase `serde::Serialize` DTOs; composes directly with `ResolvedAgentTraceStorage` without owning `source_instance_id`; `sce sync` consumes three of the four readers and `read_diff_traces_after` is a retained compatibility surface; no local sync cursor, no `agent-trace-sync.db`, no Turso Sync, no ETL, no DWH) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) @@ -171,6 +171,7 @@ Recent decision records: - `context/decisions/2026-08-10-agent-trace-source-instance-id.md` (adds `source_instance_id` as a physical-database-lineage identity on `repository_metadata`, independent of and never derived from `repository_id`; concurrency-safe atomic claim; local storage identity only, no remote-ingestion architecture designed) - `context/decisions/2026-08-11-separate-control-plane-and-sce-web-urls.md` (separates the dedicated control-plane sync default from the SCE web and config-schema URL owner while preserving the existing override seam and ingestion contract) - `context/decisions/2026-08-13-trace-sync-progress-stream-contract.md` (keeps trace-sync progress and lifecycle timestamps on stderr while preserving stdout payload and JSON silence) +- `context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md` (`sce sync` uploads three streams while `diff_traces` stays local; retained `/state.cursors.diffTraces`, `streams.diffTraces` no-op entry, and unused protocol surfaces through the staged Phase 2 → release → Phase 3 → compatibility-removal gate) - `context/decisions/2026-08-18-consumer-typed-progress-reporter-boundary.md` (keeps the reusable reporter contract generic over consumer event types while sync owns `SyncProgressEvent`) - `context/decisions/2026-08-18-sync-owned-progress-reporter-contract.md` (makes `services::sync::progress` the sole owner of the generic progress contract, no-op reporter, sync adapter, and focused tests; no top-level progress service remains) - `context/decisions/2026-08-20-general-unexpected-user-error-catalog.md` (records the closed `UserError` catalog entry with one static log-files guidance sentence, no dynamic path interpolation or arbitrary message variant; current `sce sync` adoption is documented in `context/sce/cli-error-code-taxonomy.md`) diff --git a/context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md b/context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md new file mode 100644 index 000000000..5263d7188 --- /dev/null +++ b/context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md @@ -0,0 +1,70 @@ +# Decision: Retire diff_traces from remote sync through a staged compatibility gate + +Date: 2026-10-01 +Status: Accepted +Plan: `context/plans/retire-diff-traces-remote-sync.md` +Task: T01 + +## Context + +The local Agent Trace database captures four streams: `messages`, `parts`, `diff_traces`, and `agent_traces`. `sce sync` previously uploaded all four to the Control Plane ingestion API. `diff_traces` is being retired from the Control Plane ingestion contract, but it stays essential locally: hook ingress, structured-patch reconstruction, post-commit patch intersection, staged-diff AI-overlap, and attribution all read it. + +The client and the Control Plane are released independently. A released SCE client decodes `cursors.diffTraces` in the `POST /agent-trace/ingestion/state` response as a mandatory field, and consumers of `sce sync --format json` read `streams.diffTraces`. Removing either surface in one step would break released clients or JSON consumers. + +## Decision + +`sce sync` remotely synchronizes exactly three streams — `messages`, `parts`, `agent_traces` — and `diff_traces` remains a local-only capture stream, retired from Control Plane ingestion through a staged compatibility gate in which each step waits for the one before it: + +1. **Phase 2 (SCE client):** stop reading and POSTing `diff_traces` batches, while still decoding the mandatory `/state.cursors.diffTraces` field. +2. Merge and release a new SCE version. +3. Verify and adopt that version so supported clients no longer upload `diff_traces`. +4. **Phase 3 (Control Plane):** stop accepting and storing new `diff_traces` batches, while `/state` keeps returning `cursors.diffTraces`. +5. **Later compatibility-removal phase:** remove `/state.cursors.diffTraces` and the retained client and server compatibility surfaces from both sides together. + +During the compatibility window the client keeps `streams.diffTraces` in `sce sync --format json` as a zero-upload entry that echoes the server cursor (`uploaded: 0`, `batches: 0`, `initialCursor == finalCursor == state.cursors.diffTraces`), and keeps the unused protocol surfaces `IngestionStream::DiffTraces`, `ingest_diff_traces`, `read_diff_traces_after`, `AgentTraceDiffTraceExportRow`, and `AgentTraceCursors.diff_traces`. + +## Rationale + +Stopping the upload first means no supported client depends on the Control Plane accepting `diff_traces` by the time the server rejects it. Keeping the `/state` cursor and the JSON entry through the window means neither a Phase 2 client nor a JSON consumer sees a shape change until both sides can drop the field together. Building the JSON entry from the server cursor alone keeps sync from reading local `diff_traces` rows at all, so a local row the export reader would reject cannot fail a sync. + +## Alternatives considered + +- **Remove the `diff_traces` protocol surfaces and the `/state` cursor in the same change** — breaks released clients that decode the cursor as mandatory, and couples the client and Control Plane releases. +- **Make `AgentTraceCursors.diff_traces` optional** — alters the `/state` DTO inside the compatibility window and weakens cursor validation for no client benefit. +- **Gate the behavior behind a feature flag** — adds a second supported sync shape to test and document for a change that is meant to be one-directional. + +## Compatibility and risks + +- The `/state` request and response contract is unchanged; `cursors.diffTraces` stays required. +- `streams.diffTraces` in the JSON report no longer means the stream is synchronized. A consumer that reads `uploaded` or cursor movement there as sync progress will see a constant zero-upload entry. +- Text-mode progress shows three rows; anything that scraped a `diff_traces` progress row loses it. +- If Phase 3 lands before the Phase 2 client is adopted, older clients fail their `diff_traces` batch requests. The gate order is the mitigation. +- Historical `diff_traces` rows already stored by the Control Plane are neither backfilled nor deleted by this decision. + +## Guardrails + +- Sync must not read local `diff_traces` rows, including to build the compatibility entry. +- The local `diff_traces` table, migrations, hook ingress, and every local consumer stay untouched. +- The retained protocol surfaces keep their signatures and serialization until the compatibility-removal phase. +- Phase 3 must not remove `/state.cursors.diffTraces`. + +## Consequences + +- The governing distinction is four local capture streams versus three remotely synchronized streams. +- `AgentTraceExportReader` exposes four readers while `sce sync` consumes three; `read_diff_traces_after` has no active caller. +- The client carries unused compatibility code until the removal phase. +- The "already synced" text heading depends only on the three active streams. + +## Follow-up + +- Release and adopt the SCE version carrying Phase 2. +- Phase 3 in the Control Plane. +- The later compatibility-removal phase across both sides. + +## References + +- Plan: [`retire-diff-traces-remote-sync`](../plans/retire-diff-traces-remote-sync.md) +- Task: `T01` +- Current-state context: [`sce sync command`](../cli/sync-command.md), [`Agent Trace sync architecture`](../cli/agent-trace-sync-command.md), [`Agent Trace export readers`](../sce/agent-trace-export-readers.md), [`Agent Trace DB`](../sce/agent-trace-db.md) +- Evidence: [`sync orchestration`](../../cli/src/services/sync/sync.rs), [`sync progress`](../../cli/src/services/sync/progress.rs), [`sync rendering`](../../cli/src/services/sync/render_sync.rs) +- Related decision: [`Keep trace-sync progress on stderr while stdout remains payload-only`](2026-08-13-trace-sync-progress-stream-contract.md) diff --git a/context/glossary.md b/context/glossary.md index 8f805072c..d0a7019ea 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -72,7 +72,8 @@ - `command loop`: The `clap` derive-based parser + dispatcher in `cli/src/cli_schema.rs`, `cli/src/services/parse/command_runtime.rs`, and `cli/src/app.rs` that routes `help`, `config`, `setup`, `doctor`, `auth`, `hooks`, `policy`, `sync`, `version`, and `completion`, executes implemented command flows, emits command-local help payloads for supported subcommand trees, resolves unknown command paths to the longest valid parent's help surface, and returns deterministic actionable errors for unknown options and other invalid invocation. - `RuntimeCommand seam`: Internal static command-execution abstraction in `cli/src/services/command_registry.rs` where clap-parsed commands are represented as a `RuntimeCommand` enum with variants for `Help`, `HelpText`, `Version`, `Completion`, `Auth`, `Config`, `Setup`, `Doctor`, `Hooks`, `Policy`, and `Sync`. The enum owns `name()` and `execute(...)` dispatch, delegating behavior to service-owned command payload structs while avoiding boxed `dyn RuntimeCommand` handles. Parsed request construction lives in `cli/src/services/parse/command_runtime.rs` when user-provided options or subcommands are required. - `sce dependency baseline`: Current crate dependency set declared in `cli/Cargo.toml` (`anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `indicatif`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, `uuid`, plus target-specific keyring backends), validated through normal compile/test coverage. `[dev-dependencies]` now carries `quint-connect` (dev/test-only, for the in-progress mutation-cursor Quint Connect model-based-testing harness; see `context/plans/mutation-cursor-quint-connect.md`). -- `progress reporter contract`: Library-independent, consumer-typed sync seam in `cli/src/services/sync/progress.rs` where `ProgressReporter` carries an arbitrary consumer event type, supports explicit successful finalization and closure-based collectors, and provides `NoopProgressReporter` for callers without human progress output. The sync-owned `SyncProgressEvent` in `cli/src/services/sync/sync.rs` carries lifecycle, accepted-batch, and stream-completion payloads for the four concurrent Agent Trace streams; `services::sync::progress` also owns the fixed `indicatif` terminal adapter and focused contract tests, while `sync/command.rs` selects text versus JSON behavior. No top-level `services::progress` module exists. +- `progress reporter contract`: Library-independent, consumer-typed sync seam in `cli/src/services/sync/progress.rs` where `ProgressReporter` carries an arbitrary consumer event type, supports explicit successful finalization and closure-based collectors, and provides `NoopProgressReporter` for callers without human progress output. The sync-owned `SyncProgressEvent` in `cli/src/services/sync/sync.rs` carries lifecycle, accepted-batch, and stream-completion payloads for the three concurrent remote Agent Trace streams (`messages`, `parts`, `agent_traces`); `services::sync::progress` also owns the fixed `indicatif` terminal adapter and focused contract tests, while `sync/command.rs` selects text versus JSON behavior. No top-level `services::progress` module exists. +- `remote sync streams`: The three Agent Trace streams `sce sync` uploads to the control plane — `messages`, `parts`, `agent_traces` — as distinct from the four local capture streams, which also include the local-only `diff_traces`. `streams.diffTraces` in `sce sync --format json` is a compatibility no-op entry echoing the server cursor, not a synchronized stream. See `context/cli/agent-trace-sync-command.md`. - `local Turso adapter`: Module in `cli/src/services/local_db/mod.rs` that defines `LocalDbSpec` and exposes `LocalDb` as a `TursoDb` alias. It resolves the canonical local DB path with `local_db_path()`, currently declares zero migrations, and inherits retry-backed `new()`, `execute()`, `query()`, and `query_map()` behavior from the shared generic adapter. - `encrypted Turso adapter`: Generic adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` (connection, tokio runtime bridge, spec typing). Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; the adapter also exposes retry-backed synchronous `execute`, `query`, `query_map`, and `run_migrations` helpers with `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. diff --git a/context/overview.md b/context/overview.md index 45ddba282..fbfbb0576 100644 --- a/context/overview.md +++ b/context/overview.md @@ -87,7 +87,7 @@ The same config resolver now also owns the attribution-hooks gate used by local The config service split now includes `cli/src/services/config/resolver.rs` as the focused owner for config-file discovery, file-layer merging, env/flag/default precedence, auth-key resolution, observability resolution, attribution-hooks resolution, and default-discovered invalid-file degradation; `cli/src/services/config/mod.rs` remains the facade/rendering orchestration surface while preserving existing `services::config` imports. Generated config now includes repo-local OpenCode plugin assets: `sce-bash-policy.ts` plus `sce-agent-trace.ts` are emitted under `config/.opencode/plugins/`; the OpenCode agent-trace plugin extracts `{ sessionID, diff, time, model_id }` from user `message.updated` events with diffs, tracks per-session OpenCode client version from `session.created`/`session.updated`, and sends payloads to `sce hooks diff-trace` with `tool_name="opencode"` plus optional `tool_version`. Claude generated config registers `SessionStart` and `PostModelSwitch` for local model-state intake through `sce hooks claude-model-state`, while supported `PostToolUse Write|Edit|MultiEdit|NotebookEdit` events remain routed directly to `sce hooks diff-trace`; it does not call the retired `sce hooks session-model` route. Rust handles extraction, validation, and persistence without a TypeScript intermediary; the former `config/.claude/plugins/sce-agent-trace.ts` Bun runtime was removed in T07 of the `claude-rust-diff-trace` plan. The Rust hook validates required fields, resolves Claude `model_id` with `direct > exact transcript > exact session/agent state > bridge-derived chain state > NULL` precedence using direct metadata, matching `transcript_path`/`tool_use_id` JSONL fallback, then one exact local state lookup while keeping `tool_version` direct (without restoring `session_models`), and persists tool-prefixed `session_id` values (`oc*`for OpenCode,`cc*`for Claude,`pi*`for Pi),`model_id`, `tool_name`, and nullable `tool_version`into`diff_traces`through AgentTraceDb. Bash-policy now delegates OpenCode enforcement to the Rust`sce policy bash`command: the generated OpenCode plugin at`config/.opencode/plugins/sce-bash-policy.ts`is a thin wrapper that calls`sce policy bash --input normalized --output json`via`spawnSync`and throws on deny decisions; it no longer contains independent TypeScript policy logic. The former`bash-policy/runtime.ts`TypeScript runtime has been removed. Preset... Claude bash-policy enforcement is also generated through`.claude/settings.json`as a`PreToolUse` `Bash`command hook running`sce policy bash`, so Claude and OpenCode both delegate to the Rust policy evaluator without a Claude TypeScript runtime. Pi bash-policy enforcement is delegated the same way through a project-local Pi extension (`config/lib/pi-plugin/sce-pi-extension.ts`, emitted to `config/.pi/extensions/sce/index.ts`) whose `tool_call`handler blocks denied bash commands via`sce policy bash`and fails open when the policy check cannot run (see`context/sce/pi-extension-runtime.md`). -Local database bootstrap is now owned by `LocalDbLifecycle::setup`and`AgentTraceDbLifecycle::setup`aggregated by the setup command. Agent Trace lifecycle setup creates/reuses the current checkout ID for diagnostics and creates or migrates the repository-scoped`/sce/repos//agent-trace.db`; hook runtime uses the same repository-storage identity/path resolution without running migrations, and missing or stale schema causes hook paths to fail open with the existing`Run 'sce setup'.`guidance. Doctor validates the repository-scoped DB path/health and can bootstrap missing parent directories; outside a Git repository it reports an actionable "requires a Git repository" diagnostic instead of probing a sentinel path.`sce sync`is fully implemented: it resolves repository-scoped storage, authenticates against the control plane with stored WorkOS credentials, fetches authoritative cursors once, synchronizes the four Agent Trace capture streams concurrently while preserving sequential batches within each stream, and renders the documented concise text/JSON output (see`context/cli/sync-command.md`). The former `sce trace` command group and its database inspection surfaces are unavailable. +Local database bootstrap is now owned by `LocalDbLifecycle::setup`and`AgentTraceDbLifecycle::setup`aggregated by the setup command. Agent Trace lifecycle setup creates/reuses the current checkout ID for diagnostics and creates or migrates the repository-scoped`/sce/repos//agent-trace.db`; hook runtime uses the same repository-storage identity/path resolution without running migrations, and missing or stale schema causes hook paths to fail open with the existing`Run 'sce setup'.`guidance. Doctor validates the repository-scoped DB path/health and can bootstrap missing parent directories; outside a Git repository it reports an actionable "requires a Git repository" diagnostic instead of probing a sentinel path.`sce sync`is fully implemented: it resolves repository-scoped storage, authenticates against the control plane with stored WorkOS credentials, fetches authoritative cursors once, synchronizes three of the four local Agent Trace capture streams (`messages`, `parts`, `agent_traces`) concurrently while preserving sequential batches within each stream, leaves `diff_traces` local, and renders the documented concise text/JSON output (see`context/cli/sync-command.md`). The former `sce trace` command group and its database inspection surfaces are unavailable. The repository-root flake (`flake.nix`) applies a Rust overlay-backed stable toolchain pinned to `1.95.0`(with`rustfmt`and`clippy`), reads package/check version from the repo-root `.version`file, and builds`packages.sce`through a Crane`buildDepsOnly`+`buildPackage`pipeline. One deterministic pre-Cargo Nix derivation invokes the shared generated-input producer and supplies its validated`SCE_CLI_GENERATED_INPUT_DIR`store path to native, release, test, and Clippy Cargo derivations. Pkl is absent from those Cargo environments; dependency-only and format derivations do not receive the handoff, so canonical generation changes invalidate compiling outputs without invalidating dependency artifacts or formatting.`cli-tests`, `cli-clippy`, and `cli-fmt`remain Crane-backed check derivations. The root flake splits native and release outputs:`packages.sce`and`packages.default` build the **native** development binary (`scePackage`), while `packages.sce-release` builds the release binary (`sceReleasePackage`: static musl on Linux, native on Darwin). So `nix build .#sce`/`.#default`, `nix run . -- --help`, `nix run .#sce -- --help`, and `nix profile install github:crocoder-dev/shared-context-engineering`target the native binary, and`nix build .#sce-release`/`nix run .#sce-release -- ...`(plus`nix run .#release-artifacts`, which builds `.#sce-release`) target the release binary. On Linux the native and release outputs are distinct store paths, and the release output passes the native portability audit. `packages..ci-checks`is the explicit long-running validation tier:`nix build .#ci-checks`builds the`.#sce-release`package and, on Linux, audits the real release binary for forbidden`/nix/store/`references, so the expensive work stays out of`nix flake check`(which never builds`.#sce-release`). Git-commit embedding is **release-only**: `SCE_GIT_COMMIT`is injected via a`releaseCommitArgs` fragment applied only to the release derivations (`scePackageMusl`on Linux,`sceReleasePackageNative`on Darwin), not to`commonCargoArgs`. So native `.#sce`/`.#default`and every`nix flake check` derivation (`cli-tests`, `cli-clippy`, `cli-fmt`) build without the commit in their inputs and stay cache-reusable across commits (native `sce version`reports`unknown`), while `.#sce-release`still reports the real commit via`sce version`. `cli/build.rs` `emit_git_commit`emits`SCE_GIT_COMMIT`only when the env var is explicitly set — no`git rev-parse`fallback and no`.git/HEAD`/`.git/packed-refs`rerun watches. On Darwin the release now uses a distinct native-toolchain derivation (native toolchain + commit), so it diverges from`.#sce`to carry the commit while native stays commit-independent. diff --git a/context/plans/retire-diff-traces-remote-sync.md b/context/plans/retire-diff-traces-remote-sync.md new file mode 100644 index 000000000..33c58eb3b --- /dev/null +++ b/context/plans/retire-diff-traces-remote-sync.md @@ -0,0 +1,230 @@ +# Plan: retire-diff-traces-remote-sync + +## Change summary + +Phase 2 of retiring `diff_traces` from the Control Plane ingestion contract. +Today `cli/src/services/sync/sync.rs` starts four concurrent remote stream +state machines after the single `/state` call (`messages`, `parts`, +`diff_traces`, `agent_traces`) through `try_join_four`. This plan changes the +active sync orchestration so `sce sync` starts exactly three: `messages`, +`parts`, and `agent_traces`. The `diff_traces` stream is no longer read through +`AgentTraceExportReader::read_diff_traces_after`, no longer uploaded through +`AuthenticatedControlPlaneClient::ingest_diff_traces`, and no longer produces +progress events or a progress row. + +This replaces behavior in the sync orchestration and its presentation only. It +preserves everything else: the local `diff_traces` evidence pipeline (table, +migrations, hook ingress, structured-patch reconstruction, post-commit +intersection, staged-diff AI-overlap, attribution) is untouched, and the +low-level protocol compatibility surfaces (`IngestionStream::DiffTraces`, +`read_diff_traces_after`, `AgentTraceDiffTraceExportRow`, `ingest_diff_traces`, +the batch request DTO, `state.cursors.diff_traces` decoding) stay in place as +deferred-cleanup surfaces. The `streams.diffTraces` entry in +`sce sync --format json` is retained as a zero-upload compatibility report that +echoes the server's unchanged `diffTraces` cursor. The governing distinction +after this plan is: four local Agent Trace capture streams, three remotely +synchronized streams. + +Phase 2 stops using the stream; it does not redesign the protocol. The +orchestration change and its user-facing presentation (progress rows, text +heading, JSON compatibility report) land in one atomic task, so there is no +completed task state in which `diff_traces` is still remotely synchronized but +its progress reporting has disappeared. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the +check that proves it. `/validate` runs these checks; no task in the stack +performs final validation. + +- [x] AC1: `sce sync` starts exactly three remote stream state machines — `messages`, `parts`, `agent_traces` — and the active sync path contains no `diff_traces` stream future. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n "read_diff_traces_after|ingest_diff_traces|IngestionStream::DiffTraces =>|try_join_four" cli/src/services/sync/sync.rs` shows no call to `read_diff_traces_after`, `ingest_diff_traces`, or `try_join_four`, and the only `DiffTraces` reference outside `#[cfg(test)]` is the `cursor_for_stream` match arm; the concurrency integration test in `cli/src/services/sync/sync.rs` asserts `max_in_flight() == 3` and exactly four captured requests (one `/state`, three `/batch`). +- [x] AC2: With rows present in all four local tables, a normal sync uploads `messages`, `parts`, and `agent_traces`, and sends no ingestion batch whose body has `"stream": "diff_traces"` — on the first run and on a second incremental run. + - Validate: the full-sync integration test in `cli/src/services/sync/sync.rs` parses every captured `/agent-trace/ingestion/batch` request body and asserts the set of `stream` values is exactly `{messages, parts, agent_traces}` and that none equals `diff_traces`, after both runs; run with `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::sync`. +- [x] AC3: Sync neither reads nor changes local `diff_traces` rows: the seeded row is byte-identical after sync, and sync succeeds even when the local `diff_traces` table holds a row the export reader would reject. + - Validate: the same integration test asserts the `diff_traces` row count and column values are unchanged after both runs; a sync test seeds a `diff_traces` row with an exportable numeric field above `9_007_199_254_740_991` (which makes `read_diff_traces_after` return an error) and asserts `run_sync_against` still returns `Ok`. This behavioral proof is required in addition to the AC1 source inspection; neither replaces the other. +- [x] AC4: A `/state` response carrying `cursors.diffTraces` is still required and decoded — `AgentTraceCursors.diff_traces` stays a mandatory, non-optional field — and the initial `POST /agent-trace/ingestion/state` request is unchanged. + - Validate: existing `control_plane.rs` cursor decode/validation tests pass unmodified; the full-sync integration test asserts the first captured request is `POST /agent-trace/ingestion/state` with the unchanged `repositoryId`/`sourceInstanceId` body, and `git diff main -- cli/src/services/agent_trace_sync/control_plane.rs` shows no change to request/response DTOs (including the type of `AgentTraceCursors.diff_traces`) or cursor validation. +- [x] AC5: `sce sync --format json` retains `streams.diffTraces` with `uploaded == 0`, `batches == 0`, and `initialCursor == finalCursor == state.cursors.diffTraces`, including when local `diff_traces` rows exist beyond that cursor. + - Validate: an integration test in `cli/src/services/sync/sync.rs` answers `/state` with a non-zero `diffTraces` cursor (for example `123`) while the local table holds a newer row, and asserts `report.streams.diff_traces == StreamSyncReport { uploaded: 0, initial_cursor: 123, final_cursor: 123, batches: 0 }`; `json_shape_matches_contract` in `cli/src/services/sync/render_sync.rs` still asserts the `diffTraces` object shape. +- [x] AC6: Text-mode progress shows exactly three rows — `messages`, `parts`, `agent_traces` — with no `diff_traces` spinner or progress row, and sync emits no `BatchAccepted` or `StreamCompleted` event for `diff_traces`. + - Validate: the progress reporter test in `cli/src/services/sync/progress.rs` asserts the three labels are rendered and that the output does not contain `diff_traces`; the progress-events test in `cli/src/services/sync/sync.rs` asserts the exact event list contains events only for the three active streams. +- [x] AC7: The final text heading is `Agent Trace already synced.` exactly when the three active streams uploaded zero rows, independent of the `diffTraces` compatibility entry. + - Validate: a `render_sync.rs` test renders a report whose three active streams have `uploaded == 0` and asserts the already-synced heading, and a report with one active stream uploading asserts `Agent Trace sync complete.`. +- [x] AC8: The three active streams keep their existing behavior for concurrency across streams, sequential batches within a stream, cursor validation, `409` reconciliation, ambiguous `5xx`/transport/invalid-response reconciliation, terminal `400`/`403`/protocol failures, WorkOS authentication and refresh single-flight, batching, and no local sync cursor state. + - Validate: the existing tests `concurrent_sync_keeps_batches_sequential_within_one_stream`, `invalid_state_cursor_fails_before_any_batch_request`, `terminal_batch_status_fails_without_state_reconciliation`, `malformed_2xx_batch_response_still_reconciles_via_state`, `forbidden_state_response_fails_without_mutating_local_metadata`, the no-local-cursor-file assertion in the full-sync test, and all `agent_trace_sync` engine and `control_plane` tests pass with assertions changed only where they counted the fourth stream. +- [x] AC9: The local `diff_traces` pipeline and every protocol compatibility surface are unchanged, and no Control Plane code or contract change is required. + - Validate: `git diff main --stat` shows no change under `cli/src/services/hooks/`, `cli/src/services/agent_trace_db/`, `cli/src/services/patch.rs`, `cli/src/services/structured_patch.rs`, `cli/migrations/`, or `config/`; `nix shell nixpkgs#ripgrep -c rg -n "DiffTraces|fn read_diff_traces_after|struct AgentTraceDiffTraceExportRow|fn ingest_diff_traces|pub diff_traces" cli/src/services/agent_trace_sync cli/src/services/agent_trace_export` still finds each surface; existing diff-trace hook, intersection, and export-reader tests pass unmodified. +- [x] AC10: Durable context consistently distinguishes four local capture streams from three remotely synchronized streams, states that `AgentTraceExportReader` still exposes four readers of which `sce sync` consumes three, documents `streams.diffTraces` as a temporary compatibility no-op entry, and records the release gate with its compatibility boundary: Phase 3 stops the Control Plane accepting and storing new `diff_traces` batches while `/state` keeps returning `cursors.diffTraces`, and removing that cursor and the retained client/server compatibility surfaces belongs to a later compatibility-removal phase. This is a final consistency check across current-state context; the context itself is written by T01's task context synchronization, not by a separate task. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n -i "four[- ](concurrent |remote )?(agent trace )?stream|four-stream|four progress|all four streams" context --glob '!context/plans/**' --glob '!context/handovers/**' --glob '!context/decisions/**'` returns no statement describing four *remote/synchronized* streams; inspect `context/cli/sync-command.md` and `context/cli/agent-trace-sync-command.md` for the local-vs-remote lists, the `diffTraces` compatibility note, and the release gate, and confirm no current-state context file says or implies that Phase 3 removes `/state.cursors.diffTraces` or the whole `diff_traces` wire contract. + +### Full validation + +Repository-wide checks `/validate` runs after the last task, regardless of +which criterion they map to. + +- `nix flake check` +- `nix run .#pkl-check-generated` + +### Context sync + +T01's task context synchronization updates every file below before T01 becomes +`synced`; there is no separate context task. + +- `context/cli/sync-command.md` — three concurrent remote stream state machines, three progress rows, three-stream "already synced" rule, and `streams.diffTraces` documented as a retained compatibility no-op (`uploaded: 0`, `batches: 0`, `initialCursor == finalCursor == state.cursors.diffTraces`) that does not mean the stream is synchronized. +- `context/cli/agent-trace-sync-command.md` — local capture streams (`messages`, `parts`, `diff_traces`, `agent_traces`) versus remote `sce sync` streams (`messages`, `parts`, `agent_traces`); `diff_traces` stays local with no remote upload; the release gate, including that Phase 3 only stops new `diff_traces` batch ingestion and persistence, that `/state.cursors.diffTraces` remains required through the compatibility window, and that a later compatibility-removal phase removes the retained surfaces. +- `context/sce/agent-trace-export-readers.md` — the reader still exposes four `read_*_after` methods during the compatibility window; `sce sync` consumes three; `read_diff_traces_after` and `AgentTraceDiffTraceExportRow` are deferred-cleanup surfaces with no active caller. +- `context/sce/agent-trace-db.md` — the export-reader paragraph that currently calls all four "exported streams". +- `context/overview.md`, `context/context-map.md`, `context/glossary.md`, `context/architecture.md` — only the entries that state four-stream remote synchronization (context-map `sync-command.md` annotation, glossary `progress reporter contract`, architecture `SyncProgressEvent`/`sce sync` lines); `overview.md` is verify-only unless a stale four-stream statement is found. + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. +- This plan has one implementation task, so durable context made stale by the + implementation is never deferred to a later task. T01 completes as: + implement → run targeted verification → synchronize the durable context + listed under Context sync → record `Context synchronization: synced` → task + complete. `/validate` then runs the acceptance criteria. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/sync/sync.rs`, `cli/src/services/sync/progress.rs`, `cli/src/services/sync/render_sync.rs` (code and inline tests); lint allowances only on the retained compatibility surfaces in `cli/src/services/agent_trace_sync/control_plane.rs` and `cli/src/services/agent_trace_export/mod.rs`; the durable context files listed under Context sync. +- **Out of scope:** `crocoder-dev/control-plane`; ClickHouse or DWH behavior; backfilling or deleting historical data; the local `diff_traces` table, migrations, `RepositoryAgentTraceDb::insert_diff_trace`, `RepositoryAgentTraceDb::recent_diff_trace_patches`, `sce hooks diff-trace`, OpenCode/Claude/Pi/Codex diff-trace capture, structured-patch reconstruction, post-commit patch intersection, staged-diff AI-overlap, and Agent Trace attribution. +- **Constraints:** the initial `POST /agent-trace/ingestion/state` request and `/state` response decoding are unchanged — `cursors.diffTraces` stays required and `AgentTraceCursors.diff_traces` stays a mandatory field; the `sce sync --format json` field `streams.diffTraces` is retained; no local `diff_traces` row may be read to build that entry; the orchestration change and its progress/heading presentation change land together in one task; the three remaining streams keep identical concurrency, cursor, reconciliation, batching, and authentication semantics; comments and doc comments touched by the implementation must accurately describe three active remote streams versus four local capture streams; `cli/Cargo.toml` denies warnings, so retained-but-unused compatibility surfaces need a `#[allow(dead_code)]` allowance rather than deletion; no new dependency; no host `cargo` — use `nix develop -c ./scripts/run-cli-cargo.sh ...` and `nix flake check`; do not rewrite other stable output strings. +- **Non-goal:** removing `IngestionStream::DiffTraces`, `read_diff_traces_after`, `AgentTraceDiffTraceExportRow`, `ingest_diff_traces`, the `diff_traces` batch DTO serialization, or `AgentTraceCursors.diff_traces`; making `AgentTraceCursors.diff_traces` optional or otherwise altering the `/state` DTO; renaming `diff_traces`; introducing a feature flag; introducing a generic N-way concurrency framework. These removals belong to the later compatibility-removal phase described under Release gate. + +## Assumptions + +- The three-way join is implemented by converting the existing private `try_join_four` helper in `sync.rs` to a three-way equivalent. `tokio` is built without the `macros` feature and the crate has no `futures` dependency, so no existing primitive fits without a dependency change. +- The internal `StreamSyncReports.diff_traces` field is kept so `render_sync.rs` keeps emitting `streams.diffTraces`; it is constructed directly from `state.cursors.diff_traces` after the `/state` call, with no stream future. +- "`read_diff_traces_after` is not needed by the active sync path" is proven behaviorally by seeding a local `diff_traces` row that the reader rejects (numeric field above the JS safe-integer bound) and asserting sync still succeeds, plus the source inspection in AC1. +- Existing sync integration tests are strengthened and renamed where they say "four" rather than duplicated; `seed_one_row_per_stream` keeps seeding all four local tables. +- Releasing and adopting the new SCE version, Phase 3, and the later compatibility-removal phase (the release gate below) are follow-up work, not tasks in this plan. + +## Task stack + +- [x] T01: `Retire diff_traces from active sce sync while preserving protocol compatibility` (status:done) + - Task ID: T01 + - Scope: In, as one atomic change — + - **Sync orchestration** (`cli/src/services/sync/sync.rs`): remove the active `diff_traces` `sync_one_stream` future; convert the four-way join (`try_join_four`) to a three-way join so the active streams are exactly `messages`, `parts`, `agent_traces`; build `StreamSyncReports.diff_traces` directly from `state.cursors.diff_traces` as `{ uploaded: 0, batches: 0, initial_cursor: state.cursors.diff_traces, final_cursor: state.cursors.diff_traces }` without reading any local `diff_traces` row; add `#[allow(dead_code)]` only where the retained compatibility surfaces (`ingest_diff_traces`, `read_diff_traces_after`, and anything reachable only through them) would otherwise fail the deny-warnings build; make touched comments and doc comments describe three active remote streams versus four local capture streams. + - **Progress** (`cli/src/services/sync/progress.rs`): `STREAM_LABELS` becomes `["messages", "parts", "agent_traces"]` and every array sized from it follows; no `diff_traces` spinner or progress row. + - **Text completion heading** (`cli/src/services/sync/render_sync.rs`): `render_text` decides `Agent Trace already synced.` from `messages`, `parts`, and `agent_traces` only, ignoring the compatibility `diffTraces` report; `render_json` and the `streams.diffTraces` field are unchanged. + - **Tests** (inline, same task): strengthen the full-sync test (all four local tables seeded; one `/state` request; exactly three first-run `/batch` requests whose parsed `stream` values are exactly `messages`, `parts`, `agent_traces`; no request with `stream = "diff_traces"`; second incremental run sends no `diff_traces`; local `diff_traces` row present and unchanged; unchanged `/state` request body; no local cursor file); add the reader-rejecting-row proof (a local `diff_traces` row that `read_diff_traces_after` would reject does not make sync fail); add the non-zero `diffTraces` cursor compatibility assertion (`streams.diffTraces` echoes the unchanged server cursor with zero uploads and batches); adjust the concurrency test to three-way (`max_in_flight() == 3`), the progress-events test to events for the three active streams only, and the terminal-failure tests where they counted the fourth stream; update the progress reporter test to assert three rows and the absence of `diff_traces`; add `render_sync.rs` heading tests for both outcomes; keep `json_shape_matches_contract` passing unmodified. + - **Task context synchronization** (before the task is `synced`): update every file listed under Context sync so current-state context matches the implemented behavior — the local-capture list (`messages`, `parts`, `diff_traces`, `agent_traces`) versus the remote `sce sync` list (`messages`, `parts`, `agent_traces`); `AgentTraceExportReader` exposes four readers during the compatibility window while `sce sync` consumes three; the `streams.diffTraces` compatibility no-op contract and that it does not mean the stream is synchronized; the list of deferred-cleanup compatibility surfaces; and the release gate with its Phase 3 / later compatibility-removal boundary. + - Scope: Out — deleting or changing any compatibility surface's signature or serialization; `control_plane.rs` DTOs and cursor validation (including making `AgentTraceCursors.diff_traces` optional); the `agent_trace_sync` engine; hooks; the Agent Trace DB; any change to JSON field names or other output strings; a feature flag; a generic N-way concurrency abstraction; edits to immutable decision records; rewriting context not made stale by this change; plan files other than this one. + - Dependencies: none + - Done when: a sync against a database with rows in all four local tables sends one `/state` request and exactly three `/batch` requests, none with `"stream": "diff_traces"`; a second run sends only `/state`; local `diff_traces` rows are untouched and a reader-rejected row does not fail sync; `report.streams.diff_traces` echoes the server cursor with zero uploads and zero batches even when local rows exist beyond it; progress events and text-mode progress rows cover only `messages`, `parts`, `agent_traces` in that fixed order; the heading is computed from the three active streams; cross-stream concurrency is three-way and all per-stream cursor, reconciliation, and authentication tests pass with assertions changed only where they counted the fourth stream; the build is warning-free; no current-state context file describes four remotely synchronized streams, and `sync-command.md`, `agent-trace-sync-command.md`, and `agent-trace-export-readers.md` carry the stream lists, compatibility note, four-readers/three-consumed boundary, and release gate; `Context synchronization` is recorded as `synced`. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::sync`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_sync`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_export`; `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features`; after context synchronization, `nix shell nixpkgs#ripgrep -c rg -n -i "four[- ](concurrent |remote )?(agent trace )?stream|four-stream|four progress|all four streams" context --glob '!context/plans/**' --glob '!context/handovers/**' --glob '!context/decisions/**'`. + - Completed: 2026-10-01 + - Files changed: `cli/src/services/sync/sync.rs`, `cli/src/services/sync/progress.rs`, `cli/src/services/sync/render_sync.rs`, `context/plans/retire-diff-traces-remote-sync.md` + - Result: `run_sync_async` now joins three `sync_one_stream` futures (`messages`, `parts`, `agent_traces`) through `try_join_three`; `StreamSyncReports.diff_traces` is built directly from `state.cursors.diff_traces` with zero uploads and batches and no local read. `STREAM_LABELS` has three entries. `render_text` decides the heading from the three active streams. Tests: the full-sync test was renamed to `full_sync_uploads_three_remote_streams_and_second_run_is_naturally_incremental` and strengthened (parsed batch `stream` values, unchanged `/state` body, unchanged local `diff_traces` rows across both runs); the concurrency test was renamed to `concurrent_sync_overlaps_all_three_stream_batches_after_one_state_request` and asserts `max_in_flight() == 3` with four captured requests; added `sync_succeeds_when_local_diff_trace_row_is_rejected_by_export_reader`, `diff_traces_report_echoes_server_cursor_without_uploading_newer_local_rows`, `text_heading_is_already_synced_when_three_active_streams_uploaded_nothing`, and `text_heading_is_complete_when_any_active_stream_uploaded_rows`; the progress-events and progress-reporter tests cover three streams and assert the absence of `diff_traces`. + - Verify: + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::sync` — passed (32 passed, 0 failed). + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_sync` — passed (38 passed, 0 failed). + - `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_export` — passed (48 passed, 0 failed). + - `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features` — passed, no warnings. + - Context `rg` four-stream sweep — passed after context synchronization (no matches). + - Deviations: + - No `#[allow(dead_code)]` was added. `cli/src/services/mod.rs` already declares `agent_trace_export` and `agent_trace_sync` under module-level `#[allow(dead_code)]`, so the retained compatibility surfaces build warning-free and `control_plane.rs` and `agent_trace_export/mod.rs` are untouched. + - The reader-rejecting-row test does not call `read_diff_traces_after` itself, so AC1's source inspection of `sync.rs` stays clean; the rejection premise is proven by the unmodified export test `read_diff_traces_after_rejects_row_above_safe_integer_bound`, which seeds the same shape of row. + - Doc comments in `sync.rs` that described four synchronized streams were removed rather than reworded, per the user's instruction to keep generated comments out of code. + - Context impact: required — current-state context describes four concurrent remote streams, four progress rows, and four exported streams; the files listed under Context sync must be updated. + - Context synchronization: synced + +After T01 is `done` and `synced`, the plan proceeds directly to `/validate`. + +## Release gate + +This plan ends at the following compatibility gate. Each step must not start +before every step above it is complete. + +``` +Phase 2 (this plan) +SCE stops POSTing diff_traces batches +but still decodes /state.cursors.diffTraces + ↓ +merge + release a new SCE version + ↓ +verify/adopt that version so supported SCE clients no longer upload diff_traces + ↓ +Phase 3 +Control Plane removes/rejects stream = "diff_traces" from active batch ingestion +Control Plane stops storing new diff_traces rows +BUT /state.cursors.diffTraces remains, +for compatibility with the Phase 2 SCE client + ↓ +later compatibility-removal phase +remove: + /state.cursors.diffTraces + IngestionStream::DiffTraces + ingest_diff_traces + read_diff_traces_after export compatibility + remaining server/client protocol compatibility +``` + +Phase 3 removes `diff_traces` from active batch ingestion and prevents new +server-side `diff_traces` persistence. `POST /agent-trace/ingestion/state` +continues returning `cursors.diffTraces` during the compatibility window because +the Phase 2 SCE release still decodes that field as mandatory: a Control Plane +that dropped it would break every Phase 2 client. Removing the state cursor and +the retained client/server compatibility surfaces belongs to a later +compatibility-removal phase, which removes the field coherently from both sides. + +The two follow-up phases are therefore distinct: + +- **Phase 3:** stop accepting and storing *new* `diff_traces`. +- **Later compatibility-removal phase:** remove the retained `diff_traces` + protocol compatibility. + +## Open questions + +None. The change request fixes scope, compatibility surfaces, output contract, +test obligations, non-goals, and the release gate; the remaining choices are +local and recorded under Assumptions. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-01 + +### Commands run + +- `nix flake check` -> exit 0 (all 12 flake checks passed, including `cli-tests`, `cli-clippy`, `cli-fmt`, Bun tests, and `pkl-generated-check`) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 142 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::sync` -> exit 0 (32 passed, 0 failed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_sync` -> exit 0 (38 passed, 0 failed) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::agent_trace_export` -> exit 0 (48 passed, 0 failed) +- `nix shell nixpkgs#ripgrep -c rg -n "read_diff_traces_after|ingest_diff_traces|IngestionStream::DiffTraces =>|try_join_four" cli/src/services/sync/sync.rs` -> exit 0 (one match: the `cursor_for_stream` arm at line 565) +- `git diff main --stat` -> exit 0 (only `cli/src/services/sync/{sync,progress,render_sync}.rs` and `context/` files changed) +- `git diff main --stat -- cli/src/services/hooks/ cli/src/services/agent_trace_db/ cli/src/services/patch.rs cli/src/services/structured_patch.rs cli/migrations/ config/ cli/src/services/agent_trace_sync cli/src/services/agent_trace_export` -> exit 0 (no changes) +- `nix shell nixpkgs#ripgrep -c rg -n "DiffTraces|fn read_diff_traces_after|struct AgentTraceDiffTraceExportRow|fn ingest_diff_traces|pub diff_traces" cli/src/services/agent_trace_sync cli/src/services/agent_trace_export` -> exit 0 (every retained surface found) +- `nix shell nixpkgs#ripgrep -c rg -n -i "four[- ](concurrent |remote )?(agent trace )?stream|four-stream|four progress|all four streams" context --glob '!context/plans/**' --glob '!context/handovers/**' --glob '!context/decisions/**'` -> exit 1 (no matches, as required) + +### Success-criteria verification + +- [x] AC1: `sce sync` starts exactly three remote stream state machines and the active sync path contains no `diff_traces` stream future -> the `rg` inspection finds no `read_diff_traces_after`, `ingest_diff_traces`, or `try_join_four` in `sync.rs`; the only non-test `DiffTraces` reference is the `cursor_for_stream` arm (line 565, test module starts at 585); `concurrent_sync_overlaps_all_three_stream_batches_after_one_state_request` asserts `max_in_flight() == 3` and four captured requests, and passed. +- [x] AC2: A normal sync uploads `messages`, `parts`, `agent_traces` and sends no `diff_traces` batch on the first or second run -> `full_sync_uploads_three_remote_streams_and_second_run_is_naturally_incremental` parses every captured batch body, asserts the stream set is exactly `agent_traces`, `messages`, `parts` after both runs and that none is `diff_traces`; passed. +- [x] AC3: Sync neither reads nor changes local `diff_traces` rows, including a reader-rejected row -> the full-sync test compares a full column snapshot of `diff_traces` before and after both runs; `sync_succeeds_when_local_diff_trace_row_is_rejected_by_export_reader` seeds `time_ms = JS_MAX_SAFE_INTEGER + 1` and asserts `run_sync_against` returns `Ok`; the rejection premise is covered by the unmodified `read_diff_traces_after_rejects_row_above_safe_integer_bound`; all passed. +- [x] AC4: `/state` `cursors.diffTraces` stays required and decoded and the initial state request is unchanged -> `git diff main` shows no change under `cli/src/services/agent_trace_sync`; the `control_plane` cursor decode/validation tests passed unmodified; the full-sync test asserts the first request is `POST /agent-trace/ingestion/state` with exactly `repositoryId`/`sourceInstanceId`. +- [x] AC5: `streams.diffTraces` is retained with zero uploads and batches and echoes the server cursor -> `diff_traces_report_echoes_server_cursor_without_uploading_newer_local_rows` answers `/state` with `diffTraces: 123` while a local row `124` exists and asserts `StreamSyncReport { uploaded: 0, initial_cursor: 123, final_cursor: 123, batches: 0 }`; `json_shape_matches_contract` still asserts the `diffTraces` object shape; both passed. +- [x] AC6: Text-mode progress shows exactly three rows and no `diff_traces` events -> `progress_reporter_creates_aligned_rows_and_updates_only_the_matching_stream` asserts the three labels in order and that output does not contain `diff_traces`; `progress_events_cover_batches_empty_streams_and_fixed_order` asserts an exact event list covering only the three active streams; both passed. +- [x] AC7: The heading is `Agent Trace already synced.` exactly when the three active streams uploaded zero rows -> `text_heading_is_already_synced_when_three_active_streams_uploaded_nothing` (including a non-zero `diffTraces` upload count) and `text_heading_is_complete_when_any_active_stream_uploaded_rows` passed. +- [x] AC8: The three active streams keep their existing concurrency, cursor, reconciliation, authentication, and batching behavior -> all named tests and all `agent_trace_sync` engine and `control_plane` tests passed; the `sync.rs` test diff against `main` changes existing assertions only where they counted the fourth stream. +- [x] AC9: The local `diff_traces` pipeline and every protocol compatibility surface are unchanged -> the scoped `git diff main --stat` is empty for hooks, Agent Trace DB, patch, structured patch, migrations, `config/`, `agent_trace_sync`, and `agent_trace_export`; the `rg` inspection finds each retained surface; `nix flake check` ran the full CLI test suite unmodified. +- [x] AC10: Durable context distinguishes four local capture streams from three remote streams and records the release gate -> the four-stream `rg` sweep returns no matches; inspection of `context/cli/sync-command.md`, `context/cli/agent-trace-sync-command.md`, and `context/sce/agent-trace-export-readers.md` confirms the local-vs-remote lists, the four-readers/three-consumed boundary, the `streams.diffTraces` compatibility no-op note, and the release gate stating Phase 3 keeps `/state.cursors.diffTraces`; no current-state context file says or implies Phase 3 removes that cursor or the whole wire contract. + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- The release gate is follow-up work outside this plan: Phase 3 in the Control Plane must not start before the new SCE version is released and adopted, or older clients fail their `diff_traces` batch requests. +- The retained compatibility surfaces (`read_diff_traces_after`, `ingest_diff_traces`, `IngestionStream::DiffTraces`) have no active caller and are kept warning-free only by the existing module-level `#[allow(dead_code)]` in `cli/src/services/mod.rs`. diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index d6f8bd853..3ec843382 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -189,7 +189,7 @@ Both triggers compare `OLD.*` vs `NEW.*` for all mutable columns (excluding `upd - `fix()` bootstraps the resolved repository DB parent directory for auto-fixable parent-readiness problems, with the same global parent fallback outside repository context. - `setup()` resolves repository storage through `agent_trace_storage`, opens/creates `/sce/repos//agent-trace.db` with the repository schema, validates `repository_metadata.repository_id`, and emits setup messaging with the repository ID, `source_instance_id` (`RepositoryMetadata` from the same resolution call), and initialized DB path. Hook runtime lazy initialization remains available for repositories where setup has not run or schema metadata is incomplete. - `sce doctor` surfaces lifecycle-owned repository Agent Trace DB health in the `Configuration` section, with `[PASS]`/`[FAIL]`/`[MISS]` status tokens. Outside repository context the lifecycle reports the actionable "requires a Git repository" diagnostic instead of probing a sentinel path. JSON output includes the resolved `agent_trace_db` field; checkout identity is no longer reported (see `context/cli/checkout-identity.md`). -- `sce sync` reads repository DB rows through the read-only export boundary before sending them to the control plane. See [context/cli/sync-command.md](../cli/sync-command.md). +- `sce sync` reads `messages`, `parts`, and `agent_traces` rows through the read-only export boundary before sending them to the control plane; `diff_traces` rows are not read or uploaded. See [context/cli/sync-command.md](../cli/sync-command.md). ## Runtime writers @@ -232,7 +232,7 @@ Post-commit intersection rows are written by the active `post-commit` hook flow ## Export reader (read-only) -`cli/src/services/agent_trace_export/mod.rs` defines `AgentTraceExportReader<'a>`, a read-only incremental reader over one repository-scoped `RepositoryAgentTraceDb` (`AgentTraceExportReader::new(&db)`), composing directly with `ResolvedAgentTraceStorage` from `agent_trace_storage` without generating or owning `source_instance_id` itself. `claude_model_state` is outside this reader's four exported streams and is not synchronized. See [agent-trace-export-readers.md](agent-trace-export-readers.md) for the full reader boundary, the storage-resolver composition point, the four stream query shapes, and the explicit no-local-cursor / no-sync-db / no-Turso-Sync / no-ETL / no-DWH statement. +`cli/src/services/agent_trace_export/mod.rs` defines `AgentTraceExportReader<'a>`, a read-only incremental reader over one repository-scoped `RepositoryAgentTraceDb` (`AgentTraceExportReader::new(&db)`), composing directly with `ResolvedAgentTraceStorage` from `agent_trace_storage` without generating or owning `source_instance_id` itself. The reader exposes four `read_*_after` methods (`messages`, `parts`, `diff_traces`, `agent_traces`); `sce sync` consumes three and leaves `diff_traces` local, with `read_diff_traces_after` retained as a compatibility surface with no active caller. `claude_model_state` has no reader method and is not synchronized. See [agent-trace-export-readers.md](agent-trace-export-readers.md) for the full reader boundary, the storage-resolver composition point, the four reader query shapes, and the explicit no-local-cursor / no-sync-db / no-Turso-Sync / no-ETL / no-DWH statement. ## Staged-diff AI-overlap evidence gate diff --git a/context/sce/agent-trace-export-readers.md b/context/sce/agent-trace-export-readers.md index b57951e6c..0d71c541d 100644 --- a/context/sce/agent-trace-export-readers.md +++ b/context/sce/agent-trace-export-readers.md @@ -12,7 +12,7 @@ flowchart LR - **SCE local source DB** — the existing repository-scoped `RepositoryAgentTraceDb` (see [agent-trace-db.md](agent-trace-db.md)), written by the hook/lifecycle paths already documented there. This reader does not change its writer, schema, or migrations. - **Incremental export reader** — `AgentTraceExportReader<'a>`, described below. Read-only, stateless across calls, no network. -- **Control-plane client** — `sce sync` (see [agent-trace-sync-command.md](../cli/agent-trace-sync-command.md)) composes this reader with the authenticated control-plane HTTP client; the reader itself remains unaware of that caller. +- **Control-plane client** — `sce sync` (see [agent-trace-sync-command.md](../cli/agent-trace-sync-command.md)) composes this reader with the authenticated control-plane HTTP client; the reader itself remains unaware of that caller. `sce sync` consumes three of the four reader methods (`read_messages_after`, `read_parts_after`, `read_agent_traces_after`). ## Composition point @@ -39,7 +39,7 @@ Four methods, one per capture stream, sharing one shape: `(cursor: i64, limit: u SELECT ... FROM WHERE id > ?1 ORDER BY id ASC LIMIT ?2 ``` -against `messages`, `parts`, `diff_traces`, and `agent_traces` respectively (`read_messages_after`, `read_parts_after`, `read_diff_traces_after`, `read_agent_traces_after`). `cursor` is the last server-accepted `id` for that stream; the reader makes no gap or contiguity assumption about IDs. `claude_model_state` is local attribution state outside these four export streams and has no reader method or sync cursor. `diff_traces.patch` / `payload_type` and `agent_traces.trace_json` are returned raw and unmodified — no patch parsing, no JSON reparsing. +against `messages`, `parts`, `diff_traces`, and `agent_traces` respectively (`read_messages_after`, `read_parts_after`, `read_diff_traces_after`, `read_agent_traces_after`). `cursor` is the last server-accepted `id` for that stream; the reader makes no gap or contiguity assumption about IDs. `claude_model_state` is local attribution state outside these four reader methods and has no reader method or sync cursor. `diff_traces.patch` / `payload_type` and `agent_traces.trace_json` are returned raw and unmodified — no patch parsing, no JSON reparsing. Every call validates, before executing any query: @@ -52,6 +52,10 @@ and validates, per returned row before returning: Each stream has an owned `serde::Serialize` export-row DTO (`AgentTraceMessageExportRow`, `AgentTracePartExportRow`, `AgentTraceDiffTraceExportRow`, `AgentTraceAgentTraceExportRow`) with `#[serde(rename_all = "camelCase")]` matching the shipped control-plane ingestion contract; `sourceRowId` is the local `id` unmodified. `post_commit_patch_intersections` is not exported by any reader method. +## Compatibility window + +The reader still exposes all four `read_*_after` methods, but `sce sync` consumes only three. `read_diff_traces_after` and `AgentTraceDiffTraceExportRow` are deferred-cleanup surfaces with no active caller: `diff_traces` is a local-only capture stream that is no longer uploaded. They keep their signature, validation, and serialization until the compatibility-removal phase described in [agent-trace-sync-command.md](../cli/agent-trace-sync-command.md#release-gate). + ## What does not exist This reader introduces no local sync state and no outbound transport: diff --git a/context/sce/cli-stdout-stderr-contract.md b/context/sce/cli-stdout-stderr-contract.md index bc90ae629..cb1549a4f 100644 --- a/context/sce/cli-stdout-stderr-contract.md +++ b/context/sce/cli-stdout-stderr-contract.md @@ -27,6 +27,6 @@ See also: `context/sce/cli-error-code-taxonomy.md` for the canonical error-code - Stream routing is centralized in one app-level path to avoid per-command stream drift. - Exit code class mapping remains unchanged (`parse`, `validation`, `runtime`, `dependency`). - Observability logger records are independent from command payload output: with `log_to_file=true`, normal records are written to the configured log file and not emitted to `stderr`; with `log_to_file=false`, only error-level logger records are emitted to `stderr`. User-facing diagnostics, direct file-write diagnostics, and text-mode sync progress remain on `stderr`. -- Text-mode `sce sync` emits its aligned four-row `indicatif` progress display on `stderr` before accepted batches begin: rows start at zero with independent steady spinners, accepted batches update only the corresponding cumulative count, and each stream receives a styled completion check at its own future boundary. Redirected/non-TTY output stays plain and free of terminal-control sequences, while `NO_COLOR` disables styling. The final text report remains the command result without repository or source-instance identifiers; JSON-mode sync emits no human progress text and keeps its JSON-only payload on `stdout`, also without those identifiers. +- Text-mode `sce sync` emits its aligned three-row `indicatif` progress display on `stderr` before accepted batches begin: rows start at zero with independent steady spinners, accepted batches update only the corresponding cumulative count, and each stream receives a styled completion check at its own future boundary. Redirected/non-TTY output stays plain and free of terminal-control sequences, while `NO_COLOR` disables styling. The final text report remains the command result without repository or source-instance identifiers; JSON-mode sync emits no human progress text and keeps its JSON-only payload on `stdout`, also without those identifiers. The durable trace-sync stream choice is recorded in [Trace-sync progress stream contract](../decisions/2026-08-13-trace-sync-progress-stream-contract.md).