diff --git a/crates/bridge/src/lib.rs b/crates/bridge/src/lib.rs index b7d9298b88..583dc0ab5d 100644 --- a/crates/bridge/src/lib.rs +++ b/crates/bridge/src/lib.rs @@ -254,6 +254,13 @@ pub trait PersistenceBridge: BridgeTypes { &mut self, request: FlushFilesystemStateRequest, ) -> Result<(), Self::Error>; + /// Release any state the bridge retains for `vm_id` once that VM is + /// disposed. VM ids are monotonic and never reissued, so a disposed VM's + /// snapshot can never be loaded again; a bridge that caches snapshots in + /// memory MUST drop them here or it grows once per VM lifecycle for the + /// process lifetime (LT-022). Defaults to a no-op for bridges that persist + /// externally and hold nothing in memory. + fn forget_filesystem_state(&mut self, _vm_id: &str) {} } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/native-sidecar/src/stdio.rs b/crates/native-sidecar/src/stdio.rs index 7d2b29f100..53bad73d54 100644 --- a/crates/native-sidecar/src/stdio.rs +++ b/crates/native-sidecar/src/stdio.rs @@ -1092,7 +1092,28 @@ async fn run_async( } _ = process_event_notify.notified() => { for session in active_sessions.iter().cloned().collect::>() { - if sidecar.pump_process_events(&session.compat_ownership_scope()).await? { + // Pumping one session's process events MUST NOT be able to + // terminate the sidecar. This pump runs guest-driven work + // (JS sync RPCs, deferred fd wakeups); before this guard a + // guest-triggered kernel error here propagated out of the + // run loop to `main`, which exit(1)'d — killing every other + // VM/actor sharing the process (LT-011). Log and keep + // serving the other sessions instead. + let pumped = match sidecar + .pump_process_events(&session.compat_ownership_scope()) + .await + { + Ok(pumped) => pumped, + Err(error) => { + tracing::error!( + session = %session.session_id, + error = %error, + "process-event pump failed for session; continuing to serve other sessions" + ); + continue; + } + }; + if pumped { match event_ready_tx.try_send(()) { Ok(()) | Err(tokio::sync::mpsc::error::TrySendError::Full(())) => {} @@ -1135,9 +1156,53 @@ async fn handle_protocol_frame( } = accounted_frame; match frame { ProtocolFrame::RequestFrame(request) => { - let (dispatch, extra_responses) = + let failed_request_id = request.request_id; + let failed_ownership = request.ownership.clone(); + let dispatch_outcome = dispatch_with_prompt_interrupt(sidecar, request.clone(), stdin_rx, pending_frame) - .await?; + .await; + let (dispatch, extra_responses) = match dispatch_outcome { + Ok(value) => value, + Err(error) => { + // A single request handler failing MUST NOT terminate the + // sidecar: it is the trusted enforcement point shared by + // every VM (and, on Rivet Compute, every co-located actor) + // in this process. Previously any handler error propagated + // out of this loop to `main`, which exited(1) — so one + // guest tripping e.g. a kernel EBADF took down every + // co-located VM (cross-tenant DoS). Surface it to the + // originating caller as a typed rejection and keep serving. + let message = error.to_string(); + tracing::error!( + error = %message, + "request dispatch failed; returning rejection instead of terminating the sidecar" + ); + let rejection = response_frame( + failed_request_id, + failed_ownership, + ResponsePayload::RejectedResponse(wire::RejectedResponse { + code: String::from("ERR_AGENTOS_REQUEST_FAILED"), + message, + limit_name: None, + configured_limit: None, + current_usage: None, + requested: None, + unit: None, + scope: Some(String::from("request")), + vm_id: None, + session_generation: None, + capability_id: None, + operation: None, + configuration_path: None, + retryable: Some(false), + errno: None, + }), + ); + send_output_frame(write_tx, ProtocolFrame::ResponseFrame(rejection))?; + flush_sidecar_requests(sidecar, write_tx)?; + return Ok(()); + } + }; track_session_state( &dispatch.response.payload, active_sessions, @@ -2713,6 +2778,12 @@ impl PersistenceBridge for LocalBridge { self.snapshots.insert(request.vm_id, request.snapshot); Ok(()) } + + fn forget_filesystem_state(&mut self, vm_id: &str) { + // Without this the map retained one full encoded root filesystem per VM + // lifecycle for the sidecar's lifetime (~0.5 MiB/VM, LT-022). + self.snapshots.remove(vm_id); + } } impl ClockBridge for LocalBridge { diff --git a/crates/native-sidecar/src/vm.rs b/crates/native-sidecar/src/vm.rs index 4db9f6a525..a1fb62bbb4 100644 --- a/crates/native-sidecar/src/vm.rs +++ b/crates/native-sidecar/src/vm.rs @@ -1174,6 +1174,16 @@ where // steps' `?`, so any failure stranded the engine/extension maps (H1) and // the output-buffer map was never reclaimed at all (M6). self.reclaim_vm_tracking(session_id, vm_id); + // Drop any bridge-retained filesystem snapshot for this VM. VM ids are + // monotonic and never reissued, so the snapshot flushed during teardown + // can never be loaded again; an in-memory bridge that keeps it grows by + // one full root-filesystem blob per VM lifecycle (LT-022). Runs + // unconditionally, like the tracking reclaim above, so a failed + // teardown cannot strand the entry. + let _ = self.bridge.with_mut(|bridge| { + bridge.forget_filesystem_state(vm_id); + Ok(()) + }); let _ = fs::remove_dir_all(&vm.cwd); if let Some(staging_root) = vm.packages_staging_root.take() { let _ = fs::remove_dir_all(&staging_root);