From 63785997ecd1f37e480c501cef30511a38c3593a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 15:22:20 -0700 Subject: [PATCH 1/7] feat(isolation): add boundary protocol and Linux primitives Signed-off-by: Drew Newberry --- AGENTS.md | 3 +- Cargo.lock | 27 + Cargo.toml | 3 +- crates/openshell-binary-identity/Cargo.toml | 18 + crates/openshell-binary-identity/README.md | 17 + crates/openshell-binary-identity/src/lib.rs | 457 +++ crates/openshell-core/src/proto/mod.rs | 4 + .../src/provider_credentials.rs | 86 +- .../openshell-isolation-interface/Cargo.toml | 21 + .../src/boundary_protocol.rs | 1223 ++++++++ .../src/contract.rs | 154 +- .../openshell-isolation-interface/src/lib.rs | 15 +- .../src/linux/child_seccomp.rs | 514 ++++ .../src/linux/landlock.rs | 201 ++ .../src/linux/mod.rs | 15 + .../src/linux/proc_fd.rs | 165 ++ .../src/linux/seccomp_notify.rs | 934 +++++++ .../src/linux/socket_registry.rs | 461 ++++ .../src/linux/task_memory.rs | 411 +++ .../src/linux/workload_launcher.rs | 191 ++ .../src/mediation.rs | 168 ++ .../src/remote.rs | 2447 +++++++++++++++++ .../tests/backend_conformance.rs | 45 +- proto/isolation_boundary.proto | 20 + 24 files changed, 7580 insertions(+), 20 deletions(-) create mode 100644 crates/openshell-binary-identity/Cargo.toml create mode 100644 crates/openshell-binary-identity/README.md create mode 100644 crates/openshell-binary-identity/src/lib.rs create mode 100644 crates/openshell-isolation-interface/src/boundary_protocol.rs create mode 100644 crates/openshell-isolation-interface/src/linux/child_seccomp.rs create mode 100644 crates/openshell-isolation-interface/src/linux/landlock.rs create mode 100644 crates/openshell-isolation-interface/src/linux/mod.rs create mode 100644 crates/openshell-isolation-interface/src/linux/proc_fd.rs create mode 100644 crates/openshell-isolation-interface/src/linux/seccomp_notify.rs create mode 100644 crates/openshell-isolation-interface/src/linux/socket_registry.rs create mode 100644 crates/openshell-isolation-interface/src/linux/task_memory.rs create mode 100644 crates/openshell-isolation-interface/src/linux/workload_launcher.rs create mode 100644 crates/openshell-isolation-interface/src/mediation.rs create mode 100644 crates/openshell-isolation-interface/src/remote.rs create mode 100644 proto/isolation_boundary.proto diff --git a/AGENTS.md b/AGENTS.md index edd028ee80..d0049c85a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-conformance-cli/` | Conformance CLI | Distributable `list` and `run` entrypoint for gateway conformance | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Container supervision, policy-enforced egress routing | +| `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | | `crates/openshell-isolation-interface/` | Isolation backend interface | RFC 0012 `IsolationBackend` trait + types; the supervisor-facing runtime contract for the boundary | | `crates/openshell-policy/` | Policy engine | Filesystem, network, process, and inference constraints | | `crates/openshell-router/` | Privacy router | Privacy-aware LLM routing | @@ -67,7 +68,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution | | `crates/openshell-supervisor-middleware-builtins/` | Built-in middleware | First-party in-process middleware implementations | | `crates/openshell-supervisor-network/` | Network supervisor | Proxying, L7 enforcement, policy evaluation, and inference routing | -| `crates/openshell-supervisor-process/` | Process supervisor | Process lifecycle, namespace, and bypass monitoring | +| `crates/openshell-supervisor-process/` | Supervisor process runtime | Gateway sessions, SSH access, and remote sandbox process control | | `crates/openshell-vfio/` | VFIO support | PCI and GPU passthrough preparation and lifecycle | | `python/openshell/` | Python SDK | Python bindings and CLI packaging | | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types | diff --git a/Cargo.lock b/Cargo.lock index 79086ec1b1..32e64ab9cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3759,6 +3759,14 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openshell-binary-identity" +version = "0.0.0" +dependencies = [ + "openshell-isolation-interface", + "sha2 0.10.9", +] + [[package]] name = "openshell-bootstrap" version = "0.0.0" @@ -4174,8 +4182,25 @@ name = "openshell-isolation-interface" version = "0.0.0" dependencies = [ "async-trait", + "hyper-util", + "libc", "openshell-core", + "rcgen", + "rustix 1.1.4", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2 0.10.9", + "socket2", + "thiserror 2.0.18", "tokio", + "tokio-rustls", + "tokio-stream", + "tonic", + "tower 0.5.3", + "tracing", + "uuid", ] [[package]] @@ -5561,6 +5586,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8632,6 +8658,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index 47418d0fc5..aa06583e60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,12 +33,13 @@ hyper-util = { version = "0.1", features = ["tokio", "server-auto"] } http = "1.2" http-body = "1.0" http-body-util = "0.1" +h2 = "0.4" # TLS tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/crates/openshell-binary-identity/Cargo.toml b/crates/openshell-binary-identity/Cargo.toml new file mode 100644 index 0000000000..a8b8714be4 --- /dev/null +++ b/crates/openshell-binary-identity/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-binary-identity" +description = "Trusted executable identity resolution for OpenShell isolation backends" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +sha2 = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-binary-identity/README.md b/crates/openshell-binary-identity/README.md new file mode 100644 index 0000000000..9f11201b7e --- /dev/null +++ b/crates/openshell-binary-identity/README.md @@ -0,0 +1,17 @@ +# Binary identity + +`openshell-binary-identity` provides shared executable-identity resolution for +RFC 0012 isolation backends. Runtime-specific observers remain in their backend: +Docker obtains an authoritative thread ID from seccomp notification, while the +co-located Linux path maps an accepted socket to its owning processes. + +Given an authoritative Linux PID and an optional trusted process-tree root, the +crate reads the executable path from procfs, hashes the live `/proc//exe` +object, and collects bounded executable ancestry and diagnostic command-line +paths. Resolution failures are returned as `ResolveError` so the caller can +deny the associated connection. + +The crate does not intercept connections, authenticate remote observers, or +evaluate policy. The isolation backend remains responsible for binding the +resolved identity to the active boundary and exact accepted connection before +constructing `MediatedConnection`. diff --git a/crates/openshell-binary-identity/src/lib.rs b/crates/openshell-binary-identity/src/lib.rs new file mode 100644 index 0000000000..6e5c347a8b --- /dev/null +++ b/crates/openshell-binary-identity/src/lib.rs @@ -0,0 +1,457 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared executable-identity resolution for RFC 0012 isolation backends. +//! +//! Runtime-specific observation remains inside each isolation backend. Once an +//! observer has an authoritative PID in its procfs view, this crate +//! canonicalizes the executable path, hashes the live executable object, and +//! collects its process ancestry. Backends bind the returned identity to the +//! intercepted connection before constructing a `MediatedConnection`. + +#[cfg(target_os = "linux")] +use openshell_isolation_interface::contract::Sha256Digest; +use openshell_isolation_interface::contract::{BinaryIdentity, ResolveError}; +#[cfg(target_os = "linux")] +use std::collections::HashMap; +#[cfg(target_os = "linux")] +use std::sync::{Mutex, OnceLock}; + +#[cfg(target_os = "linux")] +const EXECUTABLE_DIGEST_CACHE_CAPACITY: usize = 1_024; + +#[cfg(target_os = "linux")] +static EXECUTABLE_DIGEST_CACHE: OnceLock>> = + OnceLock::new(); + +/// Resolves executable identity from a Linux procfs process identifier. +/// +/// The configured scope bounds ancestry and cmdline collection to the observed +/// PID namespace or a known workload process tree. +#[derive(Clone, Copy, Debug)] +pub struct ProcfsIdentityResolver { + ancestry_scope: AncestryScope, +} + +#[derive(Clone, Copy, Debug)] +enum AncestryScope { + PidNamespace, + ProcessTree(u32), +} + +impl Default for ProcfsIdentityResolver { + fn default() -> Self { + Self::for_pid_namespace() + } +} + +impl ProcfsIdentityResolver { + /// Build a resolver that discovers a nested PID namespace's init process + /// and never reports host-runtime ancestors outside that namespace. + #[must_use] + pub const fn for_pid_namespace() -> Self { + Self { + ancestry_scope: AncestryScope::PidNamespace, + } + } + + /// Build a resolver bounded by the workload's trusted process-tree root. + #[must_use] + pub const fn for_process_tree(ancestor_root: u32) -> Self { + Self { + ancestry_scope: AncestryScope::ProcessTree(ancestor_root), + } + } + + /// Resolve the identity for an authoritative process ID. + pub fn resolve(self, pid: u32) -> Result { + #[cfg(target_os = "linux")] + { + let ancestor_root = match self.ancestry_scope { + AncestryScope::PidNamespace => nested_pid_namespace_init(pid), + AncestryScope::ProcessTree(root) => Some(root), + }; + resolve_linux_process(pid, ancestor_root) + } + + #[cfg(not(target_os = "linux"))] + { + match self.ancestry_scope { + AncestryScope::PidNamespace => {} + AncestryScope::ProcessTree(ancestor_root) => { + let _ = ancestor_root; + } + } + let _ = pid; + Err(ResolveError::Failed( + "procfs binary identity is only available on Linux".to_string(), + )) + } + } +} + +#[cfg(target_os = "linux")] +fn resolve_linux_process( + pid: u32, + ancestor_root: Option, +) -> Result { + let (snapshot, mut executable) = open_process_snapshot(pid)?; + let binary_path = snapshot.binary_path.clone(); + let executable_key = snapshot.executable_cache_key(); + let cached_digest = cached_executable_digest(executable_key); + let binary_digest = cached_digest.map_or_else(|| hash_executable(pid, &mut executable), Ok)?; + let ancestor_processes = collect_ancestor_processes(&snapshot, ancestor_root); + let ancestors = ancestor_processes + .iter() + .map(|snapshot| snapshot.binary_path.clone()) + .collect::>(); + + let mut excluded_paths = ancestors.clone(); + excluded_paths.push(binary_path.clone()); + let cmdline_paths = cmdline_absolute_paths(&snapshot.cmdline) + .into_iter() + .chain( + ancestor_processes + .iter() + .flat_map(|snapshot| cmdline_absolute_paths(&snapshot.cmdline)), + ) + .filter(|path| !excluded_paths.contains(path)) + .fold(Vec::new(), |mut paths, path| { + if !paths.contains(&path) { + paths.push(path); + } + paths + }); + + validate_process_snapshot(pid, &snapshot)?; + for ancestor in &ancestor_processes { + validate_process_snapshot(ancestor.pid, ancestor)?; + } + if cached_digest.is_none() { + cache_executable_digest(executable_key, binary_digest); + } + + Ok(BinaryIdentity { + binary_path, + binary_digest: Some(binary_digest), + ancestors, + cmdline_paths, + }) +} + +#[cfg(target_os = "linux")] +#[derive(Debug, PartialEq, Eq)] +struct ProcessSnapshot { + pid: u32, + parent_pid: u32, + binary_path: std::path::PathBuf, + executable_device: u64, + executable_inode: u64, + executable_size: u64, + executable_mtime: i64, + executable_mtime_nsec: i64, + executable_ctime: i64, + executable_ctime_nsec: i64, + start_time: u64, + cmdline: Vec, +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct ExecutableCacheKey { + device: u64, + inode: u64, + size: u64, + mtime: i64, + mtime_nsec: i64, + ctime: i64, + ctime_nsec: i64, +} + +#[cfg(target_os = "linux")] +impl ProcessSnapshot { + fn executable_cache_key(&self) -> ExecutableCacheKey { + ExecutableCacheKey { + device: self.executable_device, + inode: self.executable_inode, + size: self.executable_size, + mtime: self.executable_mtime, + mtime_nsec: self.executable_mtime_nsec, + ctime: self.executable_ctime, + ctime_nsec: self.executable_ctime_nsec, + } + } +} + +#[cfg(target_os = "linux")] +fn executable_digest_cache() -> &'static Mutex> { + EXECUTABLE_DIGEST_CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(target_os = "linux")] +fn cached_executable_digest(key: ExecutableCacheKey) -> Option { + executable_digest_cache() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&key) + .copied() +} + +#[cfg(target_os = "linux")] +fn cache_executable_digest(key: ExecutableCacheKey, digest: Sha256Digest) { + let mut cache = executable_digest_cache() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if cache.len() >= EXECUTABLE_DIGEST_CACHE_CAPACITY { + cache.clear(); + } + cache.insert(key, digest); +} + +#[cfg(target_os = "linux")] +fn open_process_snapshot(pid: u32) -> Result<(ProcessSnapshot, std::fs::File), ResolveError> { + use std::os::unix::fs::MetadataExt as _; + + let path = format!("/proc/{pid}/exe"); + let binary_path = executable_path(pid)?; + let executable = std::fs::File::open(&path) + .map_err(|error| ResolveError::Failed(format!("open {path}: {error}")))?; + let metadata = executable + .metadata() + .map_err(|error| ResolveError::Failed(format!("stat {path}: {error}")))?; + let (parent_pid, start_time) = process_stat(pid)?; + let snapshot = ProcessSnapshot { + pid, + parent_pid, + binary_path, + executable_device: metadata.dev(), + executable_inode: metadata.ino(), + executable_size: metadata.size(), + executable_mtime: metadata.mtime(), + executable_mtime_nsec: metadata.mtime_nsec(), + executable_ctime: metadata.ctime(), + executable_ctime_nsec: metadata.ctime_nsec(), + start_time, + cmdline: read_process_cmdline(pid)?, + }; + validate_process_snapshot(pid, &snapshot)?; + Ok((snapshot, executable)) +} + +#[cfg(target_os = "linux")] +fn validate_process_snapshot(pid: u32, expected: &ProcessSnapshot) -> Result<(), ResolveError> { + use std::os::unix::fs::MetadataExt as _; + + let path = format!("/proc/{pid}/exe"); + let metadata = std::fs::metadata(&path) + .map_err(|error| ResolveError::Failed(format!("stat {path}: {error}")))?; + let (parent_pid, start_time) = process_stat(pid)?; + let current = ProcessSnapshot { + pid, + parent_pid, + binary_path: executable_path(pid)?, + executable_device: metadata.dev(), + executable_inode: metadata.ino(), + executable_size: metadata.size(), + executable_mtime: metadata.mtime(), + executable_mtime_nsec: metadata.mtime_nsec(), + executable_ctime: metadata.ctime(), + executable_ctime_nsec: metadata.ctime_nsec(), + start_time, + cmdline: read_process_cmdline(pid)?, + }; + if ¤t == expected { + Ok(()) + } else { + Err(ResolveError::Failed(format!( + "process {pid} changed while its executable identity was collected" + ))) + } +} + +#[cfg(target_os = "linux")] +fn process_stat(pid: u32) -> Result<(u32, u64), ResolveError> { + let path = format!("/proc/{pid}/stat"); + let stat = std::fs::read_to_string(&path) + .map_err(|error| ResolveError::Failed(format!("read {path}: {error}")))?; + let fields = stat + .rsplit_once(") ") + .map(|(_, fields)| fields) + .ok_or_else(|| ResolveError::Failed(format!("parse {path}: missing command field")))?; + let mut fields = fields.split_whitespace(); + let _state = fields.next(); + let parent_pid = fields + .next() + .ok_or_else(|| ResolveError::Failed(format!("parse {path}: missing parent PID")))? + .parse() + .map_err(|error| ResolveError::Failed(format!("parse {path} parent PID: {error}")))?; + let start_time = fields + .nth(17) + .ok_or_else(|| ResolveError::Failed(format!("parse {path}: missing start time")))? + .parse() + .map_err(|error| ResolveError::Failed(format!("parse {path} start time: {error}")))?; + Ok((parent_pid, start_time)) +} + +#[cfg(target_os = "linux")] +fn read_process_cmdline(pid: u32) -> Result, ResolveError> { + let path = format!("/proc/{pid}/cmdline"); + std::fs::read(&path).map_err(|error| ResolveError::Failed(format!("read {path}: {error}"))) +} + +#[cfg(target_os = "linux")] +fn executable_path(pid: u32) -> Result { + use std::ffi::OsString; + use std::io::ErrorKind; + use std::os::unix::ffi::{OsStrExt as _, OsStringExt as _}; + + const DELETED_SUFFIX: &[u8] = b" (deleted)"; + + let link = format!("/proc/{pid}/exe"); + let target = std::fs::read_link(&link) + .map_err(|error| ResolveError::Failed(format!("read {link}: {error}")))?; + let target_missing = + matches!(std::fs::metadata(&target), Err(error) if error.kind() == ErrorKind::NotFound); + let bytes = target.as_os_str().as_bytes(); + + if target_missing && bytes.ends_with(DELETED_SUFFIX) { + let stripped = bytes[..bytes.len() - DELETED_SUFFIX.len()].to_vec(); + return Ok(std::path::PathBuf::from(OsString::from_vec(stripped))); + } + + Ok(target) +} + +#[cfg(target_os = "linux")] +fn hash_executable(pid: u32, executable: &mut std::fs::File) -> Result { + use sha2::{Digest as _, Sha256}; + use std::io::Read as _; + + let path = format!("/proc/{pid}/exe"); + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 8 * 1024]; + loop { + let length = executable + .read(&mut buffer) + .map_err(|error| ResolveError::Failed(format!("hash {path}: {error}")))?; + if length == 0 { + break; + } + digest.update(&buffer[..length]); + } + format!("{:x}", digest.finalize()).parse() +} + +#[cfg(target_os = "linux")] +fn collect_ancestor_processes( + process: &ProcessSnapshot, + ancestor_root: Option, +) -> Vec { + const MAX_DEPTH: usize = 64; + + if ancestor_root == Some(process.pid) { + return Vec::new(); + } + + let mut ancestors = Vec::new(); + let mut parent = process.parent_pid; + for _ in 0..MAX_DEPTH { + if parent == 0 + || ancestors + .iter() + .any(|current: &ProcessSnapshot| current.pid == parent) + { + break; + } + + // PID 1 is host or guest init rather than workload ancestry unless it + // is the explicitly supplied process-tree root. + if parent == 1 && ancestor_root != Some(1) { + break; + } + + let Ok((snapshot, _executable)) = open_process_snapshot(parent) else { + break; + }; + let next_parent = snapshot.parent_pid; + ancestors.push(snapshot); + if ancestor_root == Some(parent) || parent == 1 { + break; + } + parent = next_parent; + } + ancestors +} + +#[cfg(target_os = "linux")] +fn parent_pid(pid: u32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/status")) + .ok()? + .lines() + .find_map(|line| line.strip_prefix("PPid:"))? + .trim() + .parse() + .ok() +} + +#[cfg(target_os = "linux")] +fn nested_pid_namespace_init(pid: u32) -> Option { + const MAX_DEPTH: usize = 64; + + let mut current = pid; + for _ in 0..MAX_DEPTH { + if namespace_pid(current) == Some(1) { + // Host PID 1 is outside every workload. A nested namespace init + // has a distinct host PID and is a valid workload ancestry root. + return (current != 1).then_some(current); + } + current = parent_pid(current).filter(|parent| *parent > 0 && *parent != current)?; + } + None +} + +#[cfg(target_os = "linux")] +fn namespace_pid(pid: u32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/status")) + .ok()? + .lines() + .find_map(|line| line.strip_prefix("NSpid:"))? + .split_whitespace() + .next_back()? + .parse() + .ok() +} + +#[cfg(target_os = "linux")] +fn cmdline_absolute_paths(cmdline: &[u8]) -> Vec { + cmdline + .split(|byte| *byte == 0) + .filter(|argument| argument.first() == Some(&b'/')) + .map(|argument| std::path::PathBuf::from(String::from_utf8_lossy(argument).into_owned())) + .collect() +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn resolves_current_process_from_live_executable() { + let identity = ProcfsIdentityResolver::for_pid_namespace() + .resolve(std::process::id()) + .expect("resolve current process"); + + assert!(identity.binary_path.is_absolute()); + assert!(identity.binary_digest.is_some()); + } + + #[test] + fn process_tree_root_does_not_escape_into_host_ancestry() { + let pid = std::process::id(); + let identity = ProcfsIdentityResolver::for_process_tree(pid) + .resolve(pid) + .expect("resolve process-tree root"); + + assert!(identity.ancestors.is_empty()); + } +} diff --git a/crates/openshell-core/src/proto/mod.rs b/crates/openshell-core/src/proto/mod.rs index d3b3405813..f008551d50 100644 --- a/crates/openshell-core/src/proto/mod.rs +++ b/crates/openshell-core/src/proto/mod.rs @@ -63,6 +63,10 @@ pub mod inference { pub use super::generated::openshell::inference::v1; } +pub mod isolation { + pub use super::generated::openshell::isolation::v1; +} + pub mod middleware { pub use super::generated::openshell::middleware::v1; } diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 2b1537a21b..d9055fa319 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -359,6 +359,17 @@ impl ProviderCredentialState { /// here so SDKs can read them at startup. /// 3. Everything else stays as placeholders for proxy-time resolution. pub fn child_env_with_gcp_resolved(&self) -> HashMap { + self.child_env_snapshot_with_gcp_resolved().1 + } + + /// Return the current revision and its workload-facing environment from + /// one state snapshot. + /// + /// Remote isolation boundaries use the pair as a revisioned update. The + /// revision must describe the exact environment sent across the boundary, + /// so callers must not obtain the two values through separate lock + /// acquisitions. + pub fn child_env_snapshot_with_gcp_resolved(&self) -> (u64, HashMap) { use crate::google_cloud; let inner = self @@ -376,7 +387,7 @@ impl ProviderCredentialState { .any(|key| env.contains_key(*key) && inner.non_secret_environment_keys.contains(*key)); if !has_gcp_metadata && !has_gcp_config { - return env; + return (inner.current.revision, env); } if has_gcp_metadata { @@ -414,7 +425,44 @@ impl ProviderCredentialState { } } - env + (inner.current.revision, env) + } + + /// Compare and install a workload-facing environment snapshot. + /// + /// Provider environment revisions are opaque content identities, not + /// ordered counters. The expected revision makes retries idempotent while + /// rejecting updates based on a stale view of the boundary state. + pub fn compare_and_install_child_env_snapshot( + &self, + expected_revision: u64, + revision: u64, + mut child_env: HashMap, + ) -> u64 { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + if revision == inner.current.revision || expected_revision != inner.current.revision { + return inner.current.revision; + } + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.non_secret_environment_keys.clear(); + inner.static_credential_bindings.clear(); + inner.known_static_credential_keys.clear(); + inner.static_credential_identity_epochs.clear(); + revision } /// Return the GCP token placeholder and its remaining lifetime in seconds. @@ -2117,6 +2165,40 @@ mod tests { ); } + #[test] + fn child_env_snapshot_update_uses_opaque_revision_cas() { + let state = ProviderCredentialState::from_child_env_snapshot( + 4, + HashMap::from([("TOKEN".to_string(), "four".to_string())]), + ); + + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 6, + HashMap::from([("TOKEN".to_string(), "six".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 5, + HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot(6, 2, HashMap::new()), + 2, + "opaque revisions may move numerically backwards" + ); + + let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); + assert_eq!(revision, 2); + assert!(env.is_empty(), "an empty snapshot must revoke the old env"); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index 647f19dad4..97f2d64e36 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -13,7 +13,28 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } async-trait = "0.1" +hyper-util = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } +tokio-stream = { workspace = true } +tonic = { workspace = true } +tower = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } +socket2 = { workspace = true } +tracing = { workspace = true } +rcgen = { workspace = true } +sha2 = { workspace = true } +uuid = { workspace = true } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { workspace = true, features = ["fs", "process"] } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs new file mode 100644 index 0000000000..310afacda6 --- /dev/null +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -0,0 +1,1223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Versioned control protocol shared by every remote isolation boundary. +//! +//! Drivers choose and provision the transport, but they do not redefine the +//! process lifecycle, streaming, identity, or authentication messages. The +//! control and boundary roles exchange these length-delimited JSON frames over +//! a private Unix socket, authenticated TCP connection, or virtio-vsock stream. + +use std::fmt; +use std::io; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use crate::AgentSpec; +use crate::contract::Sha256Digest; +use crate::contract::{ + BackendError, BinaryIdentity, BoundaryExitStatus, BoundarySignal, DriverFenceEvidence, + ExecSpec, ResolveError, SandboxConfirmEvidence, TopologyDescriptor, +}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use rcgen::{CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; +pub const STREAM_STDIN: u8 = 0; +pub const STREAM_STDOUT: u8 = 1; +pub const STREAM_STDERR: u8 = 2; +pub const STREAM_EXIT: u8 = 3; +pub const STREAM_STDIN_CLOSED: u8 = 4; +/// Supervisor decision for a staged seccomp-mediated TCP open. +pub const STREAM_NETWORK_DECISION: u8 = 5; +/// Supervisor response for one sandbox-local DNS relay exchange. +pub const STREAM_DNS_RESPONSE: u8 = 6; +/// Boundary acknowledgement that a mediated DNS response was committed. +pub const STREAM_DNS_ACK: u8 = 7; +pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; +/// Control-side endpoint for a driver-provisioned boundary. +/// Supervisor-side mutual-TLS identity for one sandbox generation. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryClientTls { + /// DNS identity required from the sandbox certificate. + pub server_name: String, + /// Per-generation trust anchor for the sandbox certificate. + pub ca_certificate_pem: String, + /// Supervisor-only client certificate chain. + pub certificate_chain_pem: String, + /// Supervisor-only client private key. + pub private_key_pem: String, +} + +impl fmt::Debug for BoundaryClientTls { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryClientTls") + .field("server_name", &self.server_name) + .field("ca_certificate_pem", &"") + .field("certificate_chain_pem", &"") + .field("private_key_pem", &"") + .finish() + } +} + +/// Sandbox-side mutual-TLS files staged by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryServerTls { + /// Sandbox server certificate chain. + pub certificate_chain_path: PathBuf, + /// Sandbox server private key. + pub private_key_path: PathBuf, + /// Trust anchor used to require the generation-specific supervisor leaf. + pub client_ca_certificate_path: PathBuf, +} + +/// Complete per-generation material returned only to a trusted driver. +#[derive(Clone)] +pub struct BoundaryMutualTlsMaterial { + pub server_name: String, + pub ca_certificate_pem: String, + pub sandbox_certificate_pem: String, + pub sandbox_private_key_pem: String, + pub supervisor_certificate_pem: String, + pub supervisor_private_key_pem: String, +} + +/// Generate distinct server- and client-authentication leaves under a fresh CA. +pub fn generate_boundary_mutual_tls_material() -> Result { + const SERVER_NAME: &str = "sandbox.openshell.internal"; + let ca_key = KeyPair::generate() + .map_err(|error| BackendError::Descriptor(format!("generate boundary CA key: {error}")))?; + let mut ca_params = CertificateParams::default(); + ca_params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params + .distinguished_name + .push(DnType::CommonName, "OpenShell sandbox channel CA"); + ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + let ca = ca_params.self_signed(&ca_key).map_err(|error| { + BackendError::Descriptor(format!("generate boundary CA certificate: {error}")) + })?; + + let sandbox_key = KeyPair::generate().map_err(|error| { + BackendError::Descriptor(format!("generate sandbox channel key: {error}")) + })?; + let mut sandbox_params = CertificateParams::new(vec![SERVER_NAME.to_string()]) + .map_err(|error| BackendError::Descriptor(format!("build sandbox certificate: {error}")))?; + sandbox_params + .distinguished_name + .push(DnType::CommonName, "OpenShell sandbox"); + sandbox_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let sandbox = sandbox_params + .signed_by(&sandbox_key, &ca, &ca_key) + .map_err(|error| { + BackendError::Descriptor(format!("sign sandbox channel certificate: {error}")) + })?; + + let supervisor_key = KeyPair::generate().map_err(|error| { + BackendError::Descriptor(format!("generate supervisor channel key: {error}")) + })?; + let mut supervisor_params = CertificateParams::default(); + supervisor_params + .distinguished_name + .push(DnType::CommonName, "OpenShell supervisor"); + supervisor_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let supervisor = supervisor_params + .signed_by(&supervisor_key, &ca, &ca_key) + .map_err(|error| { + BackendError::Descriptor(format!("sign supervisor channel certificate: {error}")) + })?; + + Ok(BoundaryMutualTlsMaterial { + server_name: SERVER_NAME.to_string(), + ca_certificate_pem: ca.pem(), + sandbox_certificate_pem: sandbox.pem(), + sandbox_private_key_pem: sandbox_key.serialize_pem(), + supervisor_certificate_pem: supervisor.pem(), + supervisor_private_key_pem: supervisor_key.serialize_pem(), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryTransport { + /// Mutual TLS over a private Unix socket, including libkrun's host mapping. + Unix { + socket_path: PathBuf, + tls: BoundaryClientTls, + }, + /// Mutual TLS over a runtime-scoped TCP endpoint. + TlsTcp { + address: std::net::SocketAddr, + tls: BoundaryClientTls, + }, + /// Mutual TLS over Linux host `AF_VSOCK`. + Vsock { + guest_cid: u32, + control_port: u32, + tls: BoundaryClientTls, + }, +} + +/// Boundary-side listener provisioned by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryListener { + /// Mutual TLS over a private Unix socket shared with a companion. + Unix { + socket_path: PathBuf, + tls: BoundaryServerTls, + }, + /// Mutual TLS over TCP. An unspecified IP is valid for the sandbox bind. + TlsTcp { + address: std::net::SocketAddr, + tls: BoundaryServerTls, + }, + /// Mutual TLS over guest `AF_VSOCK`. + Vsock { + control_port: u32, + tls: BoundaryServerTls, + }, +} + +/// Protected descriptor consumed by `openshell-supervisor`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryTopology { + /// Stable identity of the boundary, normally the sandbox ID. + pub boundary_id: String, + /// Immutable driver-owned workload generation. + pub generation: String, + /// Fresh session epoch shared with the sandbox bootstrap. + pub session_epoch: String, + /// Immutable numeric identity already applied to the sandbox workload. + pub workload_identity: crate::contract::ResolvedWorkloadIdentity, + /// Driver-provisioned control endpoint. + pub transport: BoundaryTransport, + /// Multiplex logical exchanges over one authenticated gRPC connection. + pub multiplexed: bool, + /// Trusted dial target for well-known host-gateway aliases, when the + /// network supervisor cannot use the boundary's resolver view. + #[serde(default)] + pub host_gateway_ip: Option, + /// Driver-specific immutable resource coordinates bound at attach (for + /// example pod UID, VM generation, or container ID). + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Concrete outer-fence evidence validated by the driver. + pub driver_fence: DriverFenceEvidence, + /// Per-boundary authentication secret; never exposed to workload code. + pub bootstrap_token: String, +} + +impl fmt::Debug for BoundaryTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryTopology") + .field("boundary_id", &self.boundary_id) + .field("generation", &self.generation) + .field("session_epoch", &"") + .field("transport", &self.transport) + .field("multiplexed", &self.multiplexed) + .field("host_gateway_ip", &self.host_gateway_ip) + .field("resource_claims", &self.resource_claims) + .field("driver_fence", &self.driver_fence) + .field("bootstrap_token", &"") + .finish() + } +} + +impl BoundaryTopology { + /// Encode this topology as the shared RFC 0012 descriptor admitted for + /// `backend_name`. + pub fn descriptor( + &self, + backend_name: impl Into, + ) -> Result { + let payload = serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))?; + Ok(TopologyDescriptor { + backend_name: backend_name.into(), + payload, + }) + } +} + +/// Protected bootstrap configuration consumed by `openshell-sandbox`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryConfig { + /// Stable identity expected in every authenticated request. + pub boundary_id: String, + /// Immutable driver-owned workload generation. + pub generation: String, + /// Fresh session epoch for this sandbox/supervisor relationship. + pub session_epoch: String, + /// Per-boundary authentication secret. + pub bootstrap_token: String, + /// Driver-provisioned listener. + pub listener: BoundaryListener, + /// Serve the protected protocol as multiplexed gRPC streams. + pub multiplexed: bool, + /// Immutable coordinates the boundary requires from the control-side + /// topology descriptor before accepting attachment. + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Driver-provisioned, read-only runtime evidence for resource claims. + /// + /// Each entry maps a claim key to an absolute file whose trimmed contents + /// must equal the corresponding value in `resource_claims` before the + /// boundary opens its listener. Kubernetes uses this to bind a one-use + /// bootstrap bundle to the admitted workload Pod UID exposed by the + /// Downward API. Other drivers may leave the map empty. + #[serde(default)] + pub resource_claim_files: std::collections::BTreeMap, + /// Exact identity already applied by the runtime to the sandbox process. + pub workload_identity: crate::contract::ResolvedWorkloadIdentity, + /// Concrete outer-fence evidence validated by the driver. + pub driver_fence: DriverFenceEvidence, + /// Driver-resolved environment exposed only to workload processes. + #[serde(default)] + pub child_env: std::collections::HashMap, +} + +impl fmt::Debug for BoundaryConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryConfig") + .field("boundary_id", &self.boundary_id) + .field("generation", &self.generation) + .field("session_epoch", &"") + .field("bootstrap_token", &"") + .field("listener", &self.listener) + .field("multiplexed", &self.multiplexed) + .field("resource_claims", &self.resource_claims) + .field("resource_claim_files", &self.resource_claim_files) + .field("workload_identity", &self.workload_identity) + .field("driver_fence", &self.driver_fence) + .field("child_env_keys", &self.child_env.keys().collect::>()) + .finish() + } +} + +impl BoundaryConfig { + /// Serialize the protected driver-owned boundary configuration. + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode boundary config: {error}"))) + } +} + +/// Validate driver-specific immutable coordinates before a boundary binds them. +/// +/// Claim values are opaque to the common protocol, but empty or +/// whitespace-bearing identifiers cannot safely distinguish runtime objects. +pub fn validate_resource_claims( + claims: &std::collections::BTreeMap, +) -> Result<(), BackendError> { + for (key, value) in claims { + if key.is_empty() || key.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor( + "boundary resource-claim keys must be non-empty and contain no whitespace" + .to_string(), + )); + } + if value.is_empty() || value.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor(format!( + "boundary resource claim {key:?} must be non-empty and contain no whitespace" + ))); + } + } + Ok(()) +} + +pub async fn write_stream_frame( + writer: &mut (impl AsyncWrite + Unpin), + channel: u8, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame exceeds limit", + )); + } + writer.write_u8(channel).await?; + writer + .write_u32(payload.len().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame length overflow", + ) + })?) + .await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_stream_frame( + reader: &mut (impl AsyncRead + Unpin), +) -> io::Result)>> { + let channel = match reader.read_u8().await { + Ok(channel) => channel, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + }; + let declared = reader.read_u32().await? as usize; + if declared > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("boundary stream frame is too large: {declared} bytes"), + )); + } + let mut payload = vec![0; declared]; + reader.read_exact(&mut payload).await?; + Ok(Some((channel, payload))) +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + /// Cryptographically random idempotency key scoped to one sandbox generation. + pub request_id: String, + /// SHA-256 of the canonically serialized request payload. + pub payload_digest: String, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl RequestEnvelope { + /// Build a request envelope with a fresh idempotency key and normalized + /// payload digest. + pub fn new( + boundary_id: String, + bootstrap_token: String, + request: Request, + ) -> Result { + let payload_digest = request_payload_digest(&request)?; + Ok(Self { + request_id: uuid::Uuid::new_v4().to_string(), + payload_digest, + boundary_id, + bootstrap_token, + request, + }) + } + + /// Verify that the request body still matches the immutable digest bound + /// to this idempotency key. + pub fn validate_payload_digest(&self) -> Result<(), FrameError> { + let actual = request_payload_digest(&self.request)?; + if actual == self.payload_digest { + Ok(()) + } else { + Err(FrameError::PayloadDigestMismatch) + } + } +} + +fn request_payload_digest(request: &Request) -> Result { + // Round-tripping through Value canonicalizes every JSON object by key. In + // particular, this makes HashMap-backed provider environments stable + // across process restarts and independently serialized retries. + let normalized = serde_json::to_value(request).map_err(FrameError::Serialize)?; + let payload = serde_json::to_vec(&normalized).map_err(FrameError::Serialize)?; + let digest = Sha256::digest(payload); + Ok(format!("{digest:x}")) +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach { + policy: Box, + resource_claims: std::collections::BTreeMap, + }, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + }, + UpdateProviderEnvironment { + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + }, + AttachProcess { + process_id: String, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, + Exec { + spec: ExecSpecWire, + }, + ExecSignal { + process_id: String, + signal: SignalWire, + }, + Resize { + process_id: String, + cols: u16, + rows: u16, + }, + PortForward { + host: std::net::IpAddr, + port: u16, + }, + /// Upgrade one authenticated logical stream into the persistent DNS data + /// plane. + OpenMediation, + AcceptNetwork, + AcceptDns, +} + +impl Request { + /// Whether this control-path request changes generation-owned sandbox + /// state and therefore must be replayed from the idempotency ledger. + #[must_use] + pub const fn is_replayable_mutation(&self) -> bool { + matches!( + self, + Self::Attach { .. } + | Self::Confirm + | Self::StartAgent { .. } + | Self::UpdateProviderEnvironment { .. } + | Self::Exec { .. } + | Self::Signal { .. } + | Self::Terminate { .. } + | Self::ExecSignal { .. } + | Self::Resize { .. } + ) + } +} + +impl fmt::Debug for Request { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Attach { + policy: _, + resource_claims, + } => formatter + .debug_struct("Attach") + .field("policy", &"") + .field("resource_claims", resource_claims) + .finish(), + Self::Confirm => formatter.write_str("Confirm"), + Self::StartAgent { + sandbox_id, + spec, + policy: _, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => formatter + .debug_struct("StartAgent") + .field("sandbox_id", sandbox_id) + .field("spec", spec) + .field("policy", &"") + .field("ca_cert_present", &ca_cert.is_some()) + .field("ca_bundle_present", &ca_bundle.is_some()) + .field("provider_env_revision", provider_env_revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => formatter + .debug_struct("UpdateProviderEnvironment") + .field("expected_revision", expected_revision) + .field("revision", revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::Wait { process_id } => formatter + .debug_struct("Wait") + .field("process_id", process_id) + .finish(), + Self::AttachProcess { process_id } => formatter + .debug_struct("AttachProcess") + .field("process_id", process_id) + .finish(), + Self::Signal { process_id, signal } => formatter + .debug_struct("Signal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Terminate { process_id } => formatter + .debug_struct("Terminate") + .field("process_id", process_id) + .finish(), + Self::Exec { spec } => formatter.debug_tuple("Exec").field(spec).finish(), + Self::ExecSignal { process_id, signal } => formatter + .debug_struct("ExecSignal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Resize { + process_id, + cols, + rows, + } => formatter + .debug_struct("Resize") + .field("process_id", process_id) + .field("cols", cols) + .field("rows", rows) + .finish(), + Self::PortForward { host, port } => formatter + .debug_struct("PortForward") + .field("host", host) + .field("port", port) + .finish(), + Self::OpenMediation => formatter.write_str("OpenMediation"), + Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), + Self::AcceptDns => formatter.write_str("AcceptDns"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: String, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached { + snapshot: SessionSnapshotWire, + }, + Confirmed { + /// Measured capability-free posture produced before workload launch. + evidence: Box, + }, + Started { + process_id: String, + provider_env_revision: u64, + }, + ProviderEnvironmentUpdated { + revision: u64, + }, + ProcessAttached { + terminal: bool, + }, + Exited { + status: ExitStatusWire, + }, + Signaled, + Terminated, + ExecStarted { + process_id: String, + pty: bool, + }, + Resized, + PortConnected, + MediationReady, + NetworkConnected { + identity: BinaryIdentityWire, + destination: std::net::SocketAddr, + socket: crate::contract::NetworkSocketMetadata, + policy_generation: u64, + timing: MediationTimingWire, + }, + DnsQuery { + request: Vec, + transport: crate::contract::DnsTransport, + identity: BinaryIdentityWire, + timing: MediationTimingWire, + }, + Error { + kind: String, + message: String, + }, +} + +/// Sandbox-monotonic timing carried across the boundary protocol. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MediationTimingWire { + pub notification_to_queue_us: u64, + pub queue_wait_us: u64, +} + +/// Boundary-owned process/session state returned on every supervisor attach. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SessionSnapshotWire { + pub generation: String, + pub processes: Vec, +} + +/// Stable generation-scoped process state available to a replacement supervisor. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSnapshotWire { + pub process_id: String, + pub kind: ProcessKindWire, + pub terminal: bool, + pub status: Option, + pub retained_output: OutputWindowWire, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessKindWire { + Main, + Exec, +} + +/// Sequence range retained by the sandbox output ring. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutputWindowWire { + pub first_sequence: u64, + pub next_sequence: u64, + pub truncated: bool, +} + +/// Completion of one sandbox-local DNS relay exchange. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", content = "value", rename_all = "snake_case")] +pub enum DnsQueryResultWire { + Response(Vec), + Error(String), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryIdentityWire { + pub binary_path: Option, + pub binary_digest: Option, + pub ancestors: Vec, + pub cmdline_paths: Vec, + pub resolve_error: Option, +} + +impl From> for BinaryIdentityWire { + fn from(identity: Result) -> Self { + match identity { + Ok(identity) => Self { + binary_path: Some(identity.binary_path), + binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + ancestors: identity.ancestors, + cmdline_paths: identity.cmdline_paths, + resolve_error: None, + }, + Err(error) => Self { + binary_path: None, + binary_digest: None, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: Some(error.to_string()), + }, + } + } +} + +impl BinaryIdentityWire { + pub fn into_result(self) -> Result { + if let Some(error) = self.resolve_error { + return Err(ResolveError::Failed(error)); + } + let binary_path = self.binary_path.ok_or_else(|| { + ResolveError::Failed("boundary identity omitted binary path".to_string()) + })?; + let binary_digest = self + .binary_digest + .map(|digest| digest.parse::()) + .transpose()?; + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors: self.ancestors, + cmdline_paths: self.cmdline_paths, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecSpecWire { + pub program: String, + pub args: Vec, + pub env: Vec<(String, String)>, + pub workdir: Option, + pub pty: bool, +} + +impl From for ExecSpecWire { + fn from(spec: ExecSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +impl From for ExecSpec { + fn from(spec: ExecSpecWire) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + // Exhaustively destructure the policy so adding a `SandboxPolicy` + // field is a compile error here instead of a silently dropped field + // across the host-to-guest trust boundary. + let SandboxPolicy { + version, + filesystem, + network, + landlock, + process, + } = policy; + let FilesystemPolicy { + read_only, + read_write, + include_workdir, + } = filesystem; + let NetworkPolicy { mode, proxy } = network; + let LandlockPolicy { compatibility } = landlock; + let ProcessPolicy { + run_as_user, + run_as_group, + } = process; + Self { + version, + read_only, + read_write, + include_workdir, + network: NetworkModeWire::from(mode), + proxy_addr: proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(compatibility), + run_as_user, + run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +impl From for BoundarySignal { + fn from(signal: SignalWire) -> Self { + match signal { + SignalWire::Term => Self::Term, + SignalWire::Kill => Self::Kill, + SignalWire::Int => Self::Int, + SignalWire::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +impl From for ExitStatusWire { + fn from(status: BoundaryExitStatus) -> Self { + match status { + BoundaryExitStatus::Exited(code) => Self::Exited(code), + BoundaryExitStatus::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub async fn read_frame_async(reader: &mut R) -> Result +where + R: AsyncRead + Unpin, + T: DeserializeOwned, +{ + use tokio::io::AsyncReadExt as _; + + let mut header = [0_u8; 4]; + reader.read_exact(&mut header).await?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..]).await?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("control request payload digest does not match its envelope")] + PayloadDigestMismatch, + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: "4e94636d-54f8-4d85-8e4e-58954fb5af0a".to_string(), + payload_digest: String::new(), + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some(b"test certificate".to_vec()), + ca_bundle: Some(b"test bundle".to_vec()), + provider_env_revision: 7, + provider_env: std::collections::HashMap::from([( + "OPENAI_API_KEY".to_string(), + "test credential".to_string(), + )]), + }, + }; + let request = RequestEnvelope { + payload_digest: request_payload_digest(&request.request).expect("request digest"), + ..request + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + assert!(!debug.contains("test credential")); + assert!(!debug.contains("test certificate")); + assert!(!debug.contains("test bundle")); + assert!(debug.contains("OPENAI_API_KEY")); + assert!(request.validate_payload_digest().is_ok()); + } + + #[test] + fn request_digest_is_stable_across_map_order_and_detects_mutation() { + let mut first = std::collections::HashMap::new(); + first.insert("B".to_string(), "2".to_string()); + first.insert("A".to_string(), "1".to_string()); + let mut second = std::collections::HashMap::new(); + second.insert("A".to_string(), "1".to_string()); + second.insert("B".to_string(), "2".to_string()); + let build = |provider_env| Request::UpdateProviderEnvironment { + expected_revision: 1, + revision: 2, + provider_env, + }; + assert_eq!( + request_payload_digest(&build(first)).expect("first digest"), + request_payload_digest(&build(second)).expect("second digest") + ); + + let mut envelope = RequestEnvelope::new( + "sandbox-1".to_string(), + "token".to_string(), + build(std::collections::HashMap::new()), + ) + .expect("request envelope"); + envelope.request = Request::Terminate { + process_id: "different".to_string(), + }; + assert!(matches!( + envelope.validate_payload_digest(), + Err(FrameError::PayloadDigestMismatch) + )); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn resource_claims_reject_empty_or_ambiguous_identities() { + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + String::new() + ),])) + .is_err() + ); + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod uid".to_string(), + "uid-1".to_string() + ),])) + .is_err() + ); + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "uid-1".to_string(), + )])) + .expect("opaque resource identity should be valid"); + } +} diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index 96623d42d9..9e7f3480eb 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -36,8 +36,10 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use std::time::{Duration, Instant}; use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::sync::oneshot; @@ -189,7 +191,7 @@ impl VerifiedTopologyDescriptor { } /// Exact non-root identity selected before the immutable workload is created. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ResolvedWorkloadIdentity { /// Effective and real user ID used by sandbox and all workload children. pub uid: u32, @@ -222,6 +224,7 @@ impl ResolvedWorkloadIdentity { "workload identity source and resource digest are required".to_string(), )); } + supplementary_gids.retain(|supplementary_gid| *supplementary_gid != gid); supplementary_gids.sort_unstable(); supplementary_gids.dedup(); Ok(Self { @@ -380,7 +383,7 @@ pub trait BoundBoundary: Send { } /// Capability masks measured from `/proc//status`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct CapabilityEvidence { pub inheritable: u64, pub permitted: u64, @@ -402,7 +405,7 @@ impl CapabilityEvidence { } /// Active seccomp notification and socket-broker evidence. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[allow( clippy::struct_excessive_bools, reason = "each independently measured kernel operation is reported explicitly" @@ -419,8 +422,88 @@ pub struct SeccompEvidence { pub cancellation: bool, } +/// Driver-owned evidence that the mandatory outer network fence is installed. +/// +/// The sandbox cannot observe the Docker daemon, Kubernetes API, or VM device +/// model directly. Drivers therefore bind the exact fence they validated into +/// both protected bootstrap halves. The sandbox reports that value back during +/// confirmation, and the supervisor rejects any mismatch before agent launch. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "backend", rename_all = "kebab-case", deny_unknown_fields)] +pub enum DriverFenceEvidence { + Docker { + container_id: String, + network_mode: String, + unexpected_networks: Vec, + }, + Kubernetes { + network_policy_uid: String, + network_policy_resource_version: String, + ingress_isolated: bool, + egress_isolated: bool, + egress_rule_count: u32, + }, + Vm { + generation: String, + network_device_count: u32, + }, +} + +impl DriverFenceEvidence { + #[must_use] + pub const fn backend_name(&self) -> &'static str { + match self { + Self::Docker { .. } => "docker", + Self::Kubernetes { .. } => "kubernetes-proxy-pod", + Self::Vm { .. } => "vm", + } + } + + /// Validate the concrete fence properties and bind them to the selected + /// isolation backend. + pub fn validate_for_backend(&self, backend_name: &str) -> Result<(), BackendError> { + let valid = match self { + Self::Docker { + container_id, + network_mode, + unexpected_networks, + } => { + backend_name == "docker" + && !container_id.is_empty() + && network_mode == "none" + && unexpected_networks.is_empty() + } + Self::Kubernetes { + network_policy_uid, + network_policy_resource_version, + ingress_isolated, + egress_isolated, + egress_rule_count, + } => { + backend_name == "kubernetes-proxy-pod" + && !network_policy_uid.is_empty() + && !network_policy_resource_version.is_empty() + && *ingress_isolated + && *egress_isolated + && *egress_rule_count == 0 + } + Self::Vm { + generation, + network_device_count, + } => backend_name == "vm" && !generation.is_empty() && *network_device_count == 0, + }; + if valid { + Ok(()) + } else { + Err(BackendError::Confirm(format!( + "driver fence evidence is incomplete or does not match backend {backend_name:?}" + ))) + } + } +} + /// Measured sandbox-owned evidence produced before agent launch. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[allow( clippy::struct_excessive_bools, reason = "confirmation preserves independently measured security results" @@ -444,13 +527,15 @@ pub struct SandboxConfirmEvidence { pub tcp_deny_round_trip: bool, pub authenticated_supervisor: bool, pub session_epoch: String, - pub direct_egress_blocked: bool, + pub driver_fence: DriverFenceEvidence, pub resource_claims: BTreeMap, } impl SandboxConfirmEvidence { /// Validate the security-critical evidence required before launch. pub fn validate(&self, expected: &ResolvedWorkloadIdentity) -> Result<(), BackendError> { + self.driver_fence + .validate_for_backend(self.driver_fence.backend_name())?; let complete = &self.identity == expected && self.capabilities.is_empty() && self.no_new_privileges @@ -473,7 +558,6 @@ impl SandboxConfirmEvidence { && self.tcp_allow_round_trip && self.tcp_deny_round_trip && self.authenticated_supervisor - && self.direct_egress_blocked && !self.generation.is_empty() && !self.session_epoch.is_empty(); if complete { @@ -582,6 +666,13 @@ pub enum BoundarySignal { /// however many times it is called; a local PID is never the process handle. #[async_trait] pub trait BoundaryProcess: Send + Sync { + /// Attach to the admitted process's retained standard I/O. The boundary + /// remains the process owner and may permit only one control attachment. + async fn attach(&self) -> Result { + Err(BackendError::Unsupported( + "process attachment is not supported".to_string(), + )) + } /// Await terminal status (stable across repeated calls). async fn wait(&self) -> Result; /// Deliver a signal to the process or its group. @@ -595,6 +686,18 @@ pub type BoundaryInput = Box; /// A boxed async reader from a boundary process's stdout or stderr. pub type BoundaryOutput = Box; +/// A control-side attachment to the admitted process's retained I/O. +pub struct ProcessAttachment { + /// Stdin writer. + pub stdin: BoundaryInput, + /// Stdout reader, or the PTY-merged output stream. + pub stdout: BoundaryOutput, + /// Stderr reader, distinct from stdout for non-PTY processes. + pub stderr: Option, + /// PTY control, present when the admitted process owns a terminal. + pub terminal: Option>, +} + /// A PTY attached to an exec session. #[async_trait] pub trait BoundaryTerminal: Send + Sync { @@ -761,7 +864,7 @@ impl FromStr for Sha256Digest { } /// Immutable socket metadata supplied with a pending external TCP open. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct NetworkSocketMetadata { /// Kernel socket cookie captured for the exact open-file description. pub socket_cookie: u64, @@ -772,7 +875,7 @@ pub struct NetworkSocketMetadata { } /// Typed supervisor decision for one pending TCP open. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum NetworkOpenResult { /// L4 authorization and a bounded relay handler are ready. L7 policy still /// applies to bytes after the local connection commits. @@ -781,6 +884,31 @@ pub enum NetworkOpenResult { Denied { errno: i32 }, } +/// Timing captured while one mediated operation crosses the sandbox boundary. +/// +/// The durations are measured in the sandbox's monotonic clock. The supervisor +/// timestamp is local to the supervisor and is intentionally not serialized. +#[derive(Debug, Clone)] +pub struct MediationTiming { + /// Time from receiving the sandbox syscall notification to queueing it for + /// the transport. + pub sandbox_notification_to_queue: Duration, + /// Time spent waiting in the sandbox-side mediation queue. + pub sandbox_queue_wait: Duration, + /// Time at which the supervisor received the operation. + pub supervisor_received_at: Instant, +} + +impl Default for MediationTiming { + fn default() -> Self { + Self { + sandbox_notification_to_queue: Duration::ZERO, + sandbox_queue_wait: Duration::ZERO, + supervisor_received_at: Instant::now(), + } + } +} + /// A staged workload TCP open delivered before its local relay is committed. /// /// An `Err` identity must be denied and audited. The supervisor owns @@ -797,6 +925,8 @@ pub struct PendingNetworkOpen { pub socket: NetworkSocketMetadata, /// Policy generation under which the request was created. pub policy_generation: u64, + /// Monotonic stage timing for performance diagnostics. + pub timing: MediationTiming, /// Single-use completion channel back to the sandbox broker. pub result: oneshot::Sender, } @@ -805,8 +935,8 @@ pub struct PendingNetworkOpen { /// mediation service wherever that service runs. /// /// It may wrap a dedicated listener or a demultiplexed view over shared -/// transport; how it reaches a co-located proxy, a sidecar, or a shared -/// mediation service is backend-private. A trusted backend component associates +/// transport; how it reaches the separate supervisor is backend-private. A +/// trusted backend component associates /// every returned connection with its active boundary without relying solely on /// a transport tuple or workload-provided identifier. An `Err` from `accept` /// means the source itself is unusable and fails the boundary closed. @@ -817,7 +947,7 @@ pub trait NetworkMediationSource: Send + Sync { } /// DNS transport used by one workload exchange. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum DnsTransport { /// One DNS wire datagram without a TCP length prefix. Udp, @@ -837,6 +967,8 @@ pub struct MediatedDnsQuery { /// holder sent a datagram. Consumers must never treat unavailable /// identity as a binary-policy grant. pub binary_identity: Result, + /// Monotonic stage timing for performance diagnostics. + pub timing: MediationTiming, /// Single-use response channel owned by the backend adapter. pub response: oneshot::Sender, BackendError>>, } diff --git a/crates/openshell-isolation-interface/src/lib.rs b/crates/openshell-isolation-interface/src/lib.rs index 151fcb4ea8..e0d1bd424f 100644 --- a/crates/openshell-isolation-interface/src/lib.rs +++ b/crates/openshell-isolation-interface/src/lib.rs @@ -7,9 +7,8 @@ //! the supervisor role drives it through one contract. The supervisor-facing //! contract lives in [`contract`]: an object-safe, runtime-selectable backend //! plus a fixed chain of boxed lifecycle states the supervisor advances without -//! branching on where the boundary sits. The same calls work whether the -//! boundary lives in the agent's container (the in-pod backend) or further out -//! (a microVM, a node daemon, a separate pod). +//! branching on placement. Each driver places a sandbox boundary beside the +//! workload and connects it to a separate supervisor. //! //! The backend establishes standing enforcement before untrusted code runs and //! ensures launch-time controls are in force before each process's first @@ -47,4 +46,14 @@ pub struct AgentSpec { pub interactive: bool, } +/// Versioned control-to-boundary wire types shared by every backend. +pub mod boundary_protocol; pub mod contract; +/// Persistent network mediation stream framing shared by sandbox and supervisor. +pub mod mediation; +/// Reusable control-side implementation for a remote boundary endpoint. +pub mod remote; + +/// Linux-only primitives shared by capability-free sandbox implementations. +#[cfg(target_os = "linux")] +pub mod linux; diff --git a/crates/openshell-isolation-interface/src/linux/child_seccomp.rs b/crates/openshell-isolation-interface/src/linux/child_seccomp.rs new file mode 100644 index 0000000000..21d5005e9d --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/child_seccomp.rs @@ -0,0 +1,514 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Prepared seccomp self-protection for same-UID workload children. +//! +//! The program is built before `fork` and installed by the workload launcher +//! after the child has inherited the network user-notification filter. It does +//! not allocate while installing and deliberately leaves mediated networking +//! syscalls alone so the older `USER_NOTIF` action can still win. + +#![allow(unsafe_code)] + +use std::io; + +const SECCOMP_SET_MODE_FILTER: libc::c_uint = 1; +const SECCOMP_RET_KILL_PROCESS: u32 = 0x8000_0000; +const SECCOMP_RET_ERRNO: u32 = 0x0005_0000; +const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; + +const BPF_LD_W_ABS: u16 = 0x20; +const BPF_JMP_JEQ_K: u16 = 0x15; +const BPF_JMP_JSET_K: u16 = 0x45; +const BPF_RET_K: u16 = 0x06; + +const SECCOMP_DATA_NR_OFFSET: u32 = 0; +const SECCOMP_DATA_ARCH_OFFSET: u32 = 4; +const SECCOMP_DATA_ARGS_OFFSET: u32 = 16; +#[cfg(target_arch = "x86_64")] +const X32_SYSCALL_BIT: u32 = 0x4000_0000; + +const CLOSE_RANGE_UNSHARE_FLAG: u32 = 1 << 1; +const F_SETOWN_COMMAND: u32 = 8; +const F_SETSIG_COMMAND: u32 = 10; +const F_SETOWN_EX_COMMAND: u32 = 15; +const FIOSETOWN_REQUEST: u32 = 0x8901; +const SIOCSPGRP_REQUEST: u32 = 0x8902; +const CLONE_NAMESPACE_FLAGS: u32 = (libc::CLONE_NEWCGROUP + | libc::CLONE_NEWIPC + | libc::CLONE_NEWNET + | libc::CLONE_NEWNS + | libc::CLONE_NEWPID + | libc::CLONE_NEWUSER + | libc::CLONE_NEWUTS) as u32; + +/// A prebuilt child filter that can be installed without heap allocation. +pub struct ChildHardeningProgram { + instructions: Vec, +} + +impl ChildHardeningProgram { + /// Install this filter on the calling thread only. + /// + /// The caller must invoke this from the post-fork child after all + /// sandbox-wide TSYNC work and the launcher's `NEW_LISTENER` filter. + pub fn install(&mut self) -> io::Result<()> { + set_no_new_privileges()?; + let len = u16::try_from(self.instructions.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "child seccomp filter is too large", + ) + })?; + let mut program = libc::sock_fprog { + len, + filter: self.instructions.as_mut_ptr(), + }; + // SAFETY: `program` references the prebuilt cBPF instruction vector + // for the complete syscall. No TSYNC flag is used. + let result = unsafe { + libc::syscall( + libc::SYS_seccomp, + SECCOMP_SET_MODE_FILTER, + 0, + std::ptr::addr_of_mut!(program), + ) + }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + + /// Number of cBPF instructions, exposed for admission diagnostics. + #[must_use] + pub fn instruction_count(&self) -> usize { + self.instructions.len() + } +} + +/// Build the same-UID workload self-protection program before `fork`. +/// +/// `sandbox_tgid` is the sandbox PID as visible from its workload namespace. +/// The filter blocks all direct thread-targeting through `tkill`, and blocks +/// process-directed operations that name the trusted sandbox leader. Worker +/// threads share that TGID and are therefore covered by `tgkill` and the +/// process-level APIs. +pub fn prepare(sandbox_tgid: u32) -> io::Result { + if sandbox_tgid == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sandbox TGID must be nonzero", + )); + } + + let mut instructions = vec![ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_ARCH_OFFSET), + jump(BPF_JMP_JEQ_K, native_audit_arch(), 1, 0), + stmt(BPF_RET_K, SECCOMP_RET_KILL_PROCESS), + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + ]; + #[cfg(target_arch = "x86_64")] + instructions.extend([ + jump(BPF_JMP_JSET_K, X32_SYSCALL_BIT, 0, 1), + stmt(BPF_RET_K, SECCOMP_RET_KILL_PROCESS), + ]); + + for syscall in [ + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_pidfd_getfd, + libc::SYS_pidfd_send_signal, + libc::SYS_kcmp, + libc::SYS_process_madvise, + libc::SYS_process_mrelease, + libc::SYS_tkill, + libc::SYS_unshare, + libc::SYS_setns, + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + libc::SYS_capset, + libc::SYS_setuid, + libc::SYS_setgid, + libc::SYS_setreuid, + libc::SYS_setregid, + libc::SYS_setresuid, + libc::SYS_setresgid, + libc::SYS_setfsuid, + libc::SYS_setfsgid, + libc::SYS_setgroups, + libc::SYS_sethostname, + libc::SYS_setdomainname, + libc::SYS_setpriority, + libc::SYS_ioprio_set, + ] { + append_unconditional_deny(&mut instructions, syscall)?; + } + + // Modern launchers fall back from clone3 and pidfd_open only for ENOSYS. + // Returning EPERM here breaks otherwise portable process creation. The + // fallback paths remain constrained: namespace creation is denied from + // clone's scalar flags and no pidfd can be acquired. + append_unconditional_errno(&mut instructions, libc::SYS_clone3, libc::ENOSYS)?; + append_unconditional_errno(&mut instructions, libc::SYS_pidfd_open, libc::ENOSYS)?; + append_argument_masked_deny(&mut instructions, libc::SYS_clone, 0, CLONE_NAMESPACE_FLAGS)?; + + for (syscall, argument) in [ + (libc::SYS_kill, 0), + (libc::SYS_tgkill, 0), + (libc::SYS_rt_sigqueueinfo, 0), + (libc::SYS_rt_tgsigqueueinfo, 0), + ] { + append_argument_equal_deny(&mut instructions, syscall, argument, sandbox_tgid)?; + } + for syscall in [libc::SYS_kill, libc::SYS_rt_sigqueueinfo] { + // PID zero targets the caller's process group. Deny it even though + // OpenShell normally gives each workload a dedicated process group: + // an untrusted child can otherwise rejoin a trusted group first. + append_argument_equal_deny(&mut instructions, syscall, 0, 0)?; + } + // Negative PID arguments target process groups or every signalable + // process. The workload never needs that authority and must not be able + // to include the trusted sandbox workers in a broad signal operation. + append_argument_masked_deny(&mut instructions, libc::SYS_kill, 0, 1 << 31)?; + append_argument_masked_deny(&mut instructions, libc::SYS_rt_sigqueueinfo, 0, 1 << 31)?; + + for syscall in [ + libc::SYS_prlimit64, + libc::SYS_sched_setaffinity, + libc::SYS_sched_setparam, + libc::SYS_sched_setscheduler, + ] { + append_argument_nonzero_deny(&mut instructions, syscall, 0)?; + } + + // The ordinary workload filter is installed after this program and owns + // the final ban on further seccomp installation. Blocking it here would + // prevent the sandbox from completing the prepared filter stack. + append_argument_masked_deny( + &mut instructions, + libc::SYS_close_range, + 2, + CLOSE_RANGE_UNSHARE_FLAG, + )?; + + for command in [F_SETOWN_COMMAND, F_SETSIG_COMMAND, F_SETOWN_EX_COMMAND] { + append_argument_equal_deny(&mut instructions, libc::SYS_fcntl, 1, command)?; + } + for request in [FIOSETOWN_REQUEST, SIOCSPGRP_REQUEST] { + append_argument_equal_deny(&mut instructions, libc::SYS_ioctl, 1, request)?; + } + + instructions.push(stmt(BPF_RET_K, SECCOMP_RET_ALLOW)); + Ok(ChildHardeningProgram { instructions }) +} + +fn append_unconditional_deny( + instructions: &mut Vec, + syscall: i64, +) -> io::Result<()> { + let syscall = syscall_number(syscall)?; + instructions.extend([ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + jump(BPF_JMP_JEQ_K, syscall, 0, 1), + errno(libc::EPERM), + ]); + Ok(()) +} + +fn append_unconditional_errno( + instructions: &mut Vec, + syscall: i64, + error: i32, +) -> io::Result<()> { + let syscall = syscall_number(syscall)?; + instructions.extend([ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + jump(BPF_JMP_JEQ_K, syscall, 0, 1), + errno(error), + ]); + Ok(()) +} + +fn append_argument_equal_deny( + instructions: &mut Vec, + syscall: i64, + argument: u32, + value: u32, +) -> io::Result<()> { + let syscall = syscall_number(syscall)?; + instructions.extend([ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + jump(BPF_JMP_JEQ_K, syscall, 0, 3), + stmt(BPF_LD_W_ABS, argument_word_offset(argument)), + jump(BPF_JMP_JEQ_K, value, 0, 1), + errno(libc::EPERM), + ]); + Ok(()) +} + +fn append_argument_nonzero_deny( + instructions: &mut Vec, + syscall: i64, + argument: u32, +) -> io::Result<()> { + let syscall = syscall_number(syscall)?; + instructions.extend([ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + jump(BPF_JMP_JEQ_K, syscall, 0, 3), + stmt(BPF_LD_W_ABS, argument_word_offset(argument)), + jump(BPF_JMP_JEQ_K, 0, 1, 0), + errno(libc::EPERM), + ]); + Ok(()) +} + +fn append_argument_masked_deny( + instructions: &mut Vec, + syscall: i64, + argument: u32, + mask: u32, +) -> io::Result<()> { + let syscall = syscall_number(syscall)?; + instructions.extend([ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + jump(BPF_JMP_JEQ_K, syscall, 0, 3), + stmt(BPF_LD_W_ABS, argument_word_offset(argument)), + jump(BPF_JMP_JSET_K, mask, 0, 1), + errno(libc::EPERM), + ]); + Ok(()) +} + +fn syscall_number(syscall: i64) -> io::Result { + u32::try_from(syscall) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "negative syscall number")) +} + +const fn argument_word_offset(argument: u32) -> u32 { + SECCOMP_DATA_ARGS_OFFSET + argument * 8 +} + +const fn errno(value: i32) -> libc::sock_filter { + stmt(BPF_RET_K, SECCOMP_RET_ERRNO | value.cast_unsigned()) +} + +#[cfg(target_arch = "x86_64")] +const fn native_audit_arch() -> u32 { + 0xc000_003e +} + +#[cfg(target_arch = "aarch64")] +const fn native_audit_arch() -> u32 { + 0xc000_00b7 +} + +const fn stmt(code: u16, value: u32) -> libc::sock_filter { + libc::sock_filter { + code, + jt: 0, + jf: 0, + k: value, + } +} + +const fn jump(code: u16, value: u32, jt: u8, jf: u8) -> libc::sock_filter { + libc::sock_filter { + code, + jt, + jf, + k: value, + } +} + +fn set_no_new_privileges() -> io::Result<()> { + // SAFETY: PR_SET_NO_NEW_PRIVS is a one-way scalar transition. + if unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) } < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_zero_sandbox_tgid() { + assert_eq!( + prepare(0).err().expect("zero TGID must fail").kind(), + io::ErrorKind::InvalidInput + ); + } + + #[test] + fn filter_blocks_same_uid_sandbox_control() { + // SAFETY: the child uses only raw syscalls after fork and exits with + // `_exit`, so it does not run copied Rust cleanup state. + let child = unsafe { libc::fork() }; + assert!(child >= 0, "fork: {}", io::Error::last_os_error()); + if child == 0 { + let sandbox_tgid = unsafe { libc::getppid() }; + let Ok(mut filter) = prepare(u32::try_from(sandbox_tgid).unwrap_or(0)) else { + unsafe { libc::_exit(1) }; + }; + if filter.install().is_err() { + unsafe { libc::_exit(2) }; + } + let mut local = 0_u8; + let mut remote = 0_u8; + let local_iov = libc::iovec { + iov_base: std::ptr::addr_of_mut!(local).cast(), + iov_len: 1, + }; + let remote_iov = libc::iovec { + iov_base: std::ptr::addr_of_mut!(remote).cast(), + iov_len: 1, + }; + let process_vm = unsafe { + libc::process_vm_readv( + sandbox_tgid, + std::ptr::addr_of!(local_iov), + 1, + std::ptr::addr_of!(remote_iov), + 1, + 0, + ) + }; + if process_vm != -1 || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) { + unsafe { libc::_exit(3) }; + } + if unsafe { libc::kill(sandbox_tgid, 0) } != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(4) }; + } + if unsafe { libc::kill(0, 0) } != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(10) }; + } + let sandbox_group = -unsafe { libc::getpgrp() }; + if unsafe { libc::kill(sandbox_group, 0) } != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(7) }; + } + if unsafe { libc::syscall(libc::SYS_prlimit64, sandbox_tgid, libc::RLIMIT_CORE, 0, 0) } + != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(5) }; + } + if unsafe { libc::fcntl(libc::STDIN_FILENO, libc::F_SETOWN, sandbox_tgid) } != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(8) }; + } + let mut owner = sandbox_tgid; + if unsafe { + libc::ioctl( + libc::STDIN_FILENO, + libc::c_ulong::from(FIOSETOWN_REQUEST), + &raw mut owner, + ) + } != -1 + || io::Error::last_os_error().raw_os_error() != Some(libc::EPERM) + { + unsafe { libc::_exit(9) }; + } + unsafe { libc::_exit(0) }; + } + + let mut status = 0; + // SAFETY: `child` names our live direct child and status is writable. + assert_eq!(unsafe { libc::waitpid(child, &raw mut status, 0) }, child); + assert!(libc::WIFEXITED(status)); + assert_eq!(libc::WEXITSTATUS(status), 0); + } + + #[test] + fn filter_preserves_thread_and_process_creation() { + if std::env::var_os("OPENSHELL_CHILD_SECCOMP_CREATION_PROBE").is_some() { + let mut filter = prepare(std::process::id().saturating_add(1)) + .expect("prepare child hardening filter"); + filter.install().expect("install child hardening filter"); + + // A direct clone3 request must report ENOSYS so libc can use its + // established clone fallback. + let result = + unsafe { libc::syscall(libc::SYS_clone3, std::ptr::null::(), 0) }; + assert_eq!(result, -1); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ENOSYS) + ); + + // Process launchers such as uv also probe pidfd_open and require + // ENOSYS to select their non-pidfd fallback. + let result = unsafe { libc::syscall(libc::SYS_pidfd_open, libc::getpid(), 0) }; + assert_eq!(result, -1); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ENOSYS) + ); + + let joined = std::thread::spawn(|| 17_u8) + .join() + .expect("pthread-style child must start"); + assert_eq!(joined, 17); + assert!( + std::process::Command::new("/bin/true") + .status() + .expect("posix-spawn-style child must start") + .success() + ); + + let namespaced = unsafe { + libc::syscall( + libc::SYS_clone, + u64::from(CLONE_NAMESPACE_FLAGS & libc::CLONE_NEWUSER as u32) + | u64::from(libc::SIGCHLD as u32), + 0, + 0, + 0, + 0, + ) + }; + assert_eq!(namespaced, -1); + assert_eq!(io::Error::last_os_error().raw_os_error(), Some(libc::EPERM)); + return; + } + + let output = + std::process::Command::new(std::env::current_exe().expect("current test executable")) + .arg("--exact") + .arg("linux::child_seccomp::tests::filter_preserves_thread_and_process_creation") + .arg("--nocapture") + .env("OPENSHELL_CHILD_SECCOMP_CREATION_PROBE", "1") + .output() + .expect("run isolated child-hardening probe"); + assert!( + output.status.success(), + "isolated probe failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/landlock.rs b/crates/openshell-isolation-interface/src/linux/landlock.rs new file mode 100644 index 0000000000..28823ecba5 --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/landlock.rs @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Race-resistant handles for an explicit Landlock root allow-list. + +#![allow(unsafe_code)] +use std::collections::BTreeSet; +use std::ffi::{OsStr, OsString}; +use std::io; +use std::os::fd::OwnedFd; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use rustix::fs::{AtFlags, Mode, OFlags, Stat, fstat, open, openat, statat}; + +const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1; + +/// Query the Landlock ABI admitted by the active kernel and outer seccomp +/// profile without installing a ruleset. +pub fn abi_version() -> io::Result { + // SAFETY: the VERSION operation requires a null ruleset pointer and zero + // size and returns one scalar ABI version. + let result = unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0, + LANDLOCK_CREATE_RULESET_VERSION, + ) + }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + u32::try_from(result).map_err(|_| io::Error::other("Landlock ABI does not fit u32")) + } +} +/// One verified immediate child of the sandbox root. +pub struct RootEntryHandle { + name: OsString, + fd: OwnedFd, + stat: Stat, +} + +impl RootEntryHandle { + /// Immediate-root entry name. + #[must_use] + pub fn name(&self) -> &OsStr { + &self.name + } + + /// Open, no-follow handle suitable for a later Landlock `PathBeneath` rule. + #[must_use] + pub fn fd(&self) -> &OwnedFd { + &self.fd + } + + /// Device number captured when the entry was opened. + #[must_use] + pub fn device(&self) -> u64 { + self.stat.st_dev + } + + /// Inode number captured when the entry was opened. + #[must_use] + pub fn inode(&self) -> u64 { + self.stat.st_ino + } +} + +/// Open exactly the named root entries while proving that none is the private +/// sandbox hierarchy, a symlink, or a raced replacement. +/// +/// Unnamed root entries are deliberately not returned and therefore cannot be +/// admitted accidentally. The caller obtains the allow-list names from trusted +/// image/driver policy, not by blindly allowing everything present in `/`. +pub fn open_root_allowlist( + root: &Path, + allowed_names: &BTreeSet, + private_name: &OsStr, +) -> io::Result> { + validate_component(private_name)?; + if allowed_names.contains(private_name) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "private sandbox root cannot appear in the Landlock allow-list", + )); + } + + let root_fd = open( + root, + OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?; + let mut result = Vec::with_capacity(allowed_names.len()); + for name in allowed_names { + validate_component(name)?; + let before = statat(&root_fd, name, AtFlags::SYMLINK_NOFOLLOW)?; + if before.st_mode & libc::S_IFMT == libc::S_IFLNK { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Landlock root entry {} is a symlink", + Path::new(name).display() + ), + )); + } + let fd = openat( + &root_fd, + name, + OFlags::PATH | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?; + let after = fstat(&fd)?; + if before.st_dev != after.st_dev + || before.st_ino != after.st_ino + || before.st_mode & libc::S_IFMT != after.st_mode & libc::S_IFMT + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Landlock root entry {} changed while it was opened", + Path::new(name).display() + ), + )); + } + result.push(RootEntryHandle { + name: name.clone(), + fd, + stat: after, + }); + } + Ok(result) +} + +fn validate_component(name: &OsStr) -> io::Result<()> { + let bytes = name.as_bytes(); + if bytes.is_empty() || bytes == b"." || bytes == b".." || bytes.contains(&b'/') { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("invalid immediate-root entry {}", Path::new(name).display()), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::os::unix::fs::symlink; + use std::sync::atomic::{AtomicU64, Ordering}; + + use super::*; + + static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); + + fn temp_root() -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "openshell-landlock-root-{}-{}", + std::process::id(), + NEXT_TEMP.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).expect("create temp root"); + path + } + + #[test] + fn opens_only_explicit_entries_and_omits_private_root() { + let root = temp_root(); + fs::create_dir(root.join("bin")).expect("create bin"); + fs::create_dir(root.join("sandbox")).expect("create workspace"); + fs::create_dir(root.join(".openshell")).expect("create private root"); + fs::create_dir(root.join("unexpected")).expect("create unexpected root"); + + let allowed = BTreeSet::from([OsString::from("bin"), OsString::from("sandbox")]); + let entries = open_root_allowlist(&root, &allowed, OsStr::new(".openshell")) + .expect("open allow-list"); + assert_eq!( + entries + .iter() + .map(|entry| entry.name().to_owned()) + .collect::>(), + vec![OsString::from("bin"), OsString::from("sandbox")] + ); + + fs::remove_dir_all(root).expect("remove temp root"); + } + + #[test] + fn rejects_private_entry_and_symlink() { + let root = temp_root(); + fs::create_dir(root.join(".openshell")).expect("create private root"); + symlink(".openshell", root.join("runtime")).expect("create symlink"); + + let private = BTreeSet::from([OsString::from(".openshell")]); + assert!(open_root_allowlist(&root, &private, OsStr::new(".openshell")).is_err()); + let symlinked = BTreeSet::from([OsString::from("runtime")]); + assert!(open_root_allowlist(&root, &symlinked, OsStr::new(".openshell")).is_err()); + + fs::remove_dir_all(root).expect("remove temp root"); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/mod.rs b/crates/openshell-isolation-interface/src/linux/mod.rs new file mode 100644 index 0000000000..ceba35e1f1 --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/mod.rs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Audited Linux primitives used by the capability-free sandbox. +//! +//! This module intentionally contains mechanisms, not sandbox orchestration. +//! The in-workload sandbox owns lifecycle, policy, and failure handling. + +pub mod child_seccomp; +pub mod landlock; +pub mod proc_fd; +pub mod seccomp_notify; +pub mod socket_registry; +pub mod task_memory; +pub mod workload_launcher; diff --git a/crates/openshell-isolation-interface/src/linux/proc_fd.rs b/crates/openshell-isolation-interface/src/linux/proc_fd.rs new file mode 100644 index 0000000000..13f1856e54 --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/proc_fd.rs @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Strict `/proc//fd` socket identity helpers. + +#![allow(unsafe_code)] + +use std::fs; +use std::io; +use std::os::fd::RawFd; + +/// Snapshot socket inodes installed in process descriptor tables other than +/// `excluded_pid`. +/// +/// Inaccessible or concurrently disappearing entries are +/// skipped. Callers use this only to reclaim bounded mediation metadata; a +/// later operation on an unregistered descriptor fails closed. +pub fn installed_socket_inodes_excluding( + excluded_pid: u32, +) -> io::Result> { + let mut inodes = std::collections::BTreeSet::new(); + for process in fs::read_dir("/proc")? { + let Ok(process) = process else { continue }; + let Some(name) = process.file_name().to_str().map(str::to_owned) else { + continue; + }; + let Ok(pid) = name.parse::() else { + continue; + }; + if pid == excluded_pid { + continue; + } + let Ok(descriptors) = fs::read_dir(process.path().join("fd")) else { + continue; + }; + for descriptor in descriptors.flatten() { + let Ok(target) = fs::read_link(descriptor.path()) else { + continue; + }; + let Some(target) = target.to_str() else { + continue; + }; + let Some(digits) = target + .strip_prefix("socket:[") + .and_then(|value| value.strip_suffix(']')) + else { + continue; + }; + if let Ok(inode) = digits.parse::() { + inodes.insert(inode); + } + } + } + Ok(inodes) +} + +/// Return the socket inode currently installed at `fd` in `tid`'s descriptor +/// table. +/// +/// The result is only a snapshot. Callers must revalidate the seccomp +/// notification, task generation, and any retained socket cookie before a +/// state-changing operation. +pub fn socket_inode(tid: u32, fd: RawFd) -> io::Result { + if tid == 0 || fd < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "TID must be nonzero and FD must be nonnegative", + )); + } + let target = fs::read_link(format!("/proc/{tid}/fd/{fd}"))?; + let target = target.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "procfs descriptor target is not UTF-8", + ) + })?; + let digits = target + .strip_prefix("socket:[") + .and_then(|value| value.strip_suffix(']')) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "procfs descriptor is not a socket", + ) + })?; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "procfs socket inode has an invalid representation", + )); + } + digits.parse::().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("procfs socket inode does not fit u64: {error}"), + ) + }) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + + use super::*; + + #[test] + fn identifies_socket_and_rejects_regular_file() { + let mut pair = [-1; 2]; + // SAFETY: pair points to storage for exactly two returned descriptors. + let result = unsafe { + libc::socketpair( + libc::AF_UNIX, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + 0, + pair.as_mut_ptr(), + ) + }; + assert_eq!(result, 0, "socketpair: {}", io::Error::last_os_error()); + // SAFETY: successful socketpair returned two independently owned FDs. + let left = unsafe { OwnedFd::from_raw_fd(pair[0]) }; + // SAFETY: successful socketpair returned two independently owned FDs. + let _right = unsafe { OwnedFd::from_raw_fd(pair[1]) }; + assert!(socket_inode(std::process::id(), left.as_raw_fd()).unwrap() > 0); + + let file = File::open("/dev/null").expect("open regular descriptor"); + assert_eq!( + socket_inode(std::process::id(), file.as_raw_fd()) + .expect_err("regular descriptor") + .kind(), + io::ErrorKind::InvalidInput + ); + } + + #[test] + fn installed_socket_snapshot_can_exclude_the_broker() { + let mut pair = [-1; 2]; + // SAFETY: pair points to storage for exactly two returned descriptors. + let result = unsafe { + libc::socketpair( + libc::AF_UNIX, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + 0, + pair.as_mut_ptr(), + ) + }; + assert_eq!(result, 0, "socketpair: {}", io::Error::last_os_error()); + // SAFETY: successful socketpair returned two independently owned FDs. + let left = unsafe { OwnedFd::from_raw_fd(pair[0]) }; + // SAFETY: successful socketpair returned two independently owned FDs. + let _right = unsafe { OwnedFd::from_raw_fd(pair[1]) }; + let inode = socket_inode(std::process::id(), left.as_raw_fd()).unwrap(); + + assert!( + installed_socket_inodes_excluding(u32::MAX) + .unwrap() + .contains(&inode) + ); + assert!( + !installed_socket_inodes_excluding(std::process::id()) + .unwrap() + .contains(&inode) + ); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs new file mode 100644 index 0000000000..081224abac --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs @@ -0,0 +1,934 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal, typed wrappers for Linux seccomp user notification. +//! +//! The wrappers validate notification IDs around every operation and keep raw +//! UAPI structures private. Production policy and queueing belong to the +//! sandbox crate; this module owns only the kernel ABI and active conformance +//! probe. + +#![allow(unsafe_code)] + +use std::io; +use std::mem::size_of; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +const SECCOMP_SET_MODE_FILTER: libc::c_uint = 1; +const SECCOMP_GET_NOTIF_SIZES: libc::c_uint = 3; +const SECCOMP_FILTER_FLAG_NEW_LISTENER: libc::c_ulong = 1 << 3; +const SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV: libc::c_ulong = 1 << 5; + +const SECCOMP_RET_KILL_PROCESS: u32 = 0x8000_0000; +const SECCOMP_RET_USER_NOTIF: u32 = 0x7fc0_0000; +const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; + +const BPF_LD_W_ABS: u16 = 0x20; +const BPF_JMP_JEQ_K: u16 = 0x15; +#[cfg(target_arch = "x86_64")] +const BPF_JMP_JSET_K: u16 = 0x45; +const BPF_ALU_AND_K: u16 = 0x54; +const BPF_RET_K: u16 = 0x06; + +const SECCOMP_DATA_NR_OFFSET: u32 = 0; +const SECCOMP_DATA_ARCH_OFFSET: u32 = 4; +const SECCOMP_DATA_ARGS_OFFSET: u32 = 16; +#[cfg(target_arch = "x86_64")] +const X32_SYSCALL_BIT: u32 = 0x4000_0000; + +const SECCOMP_ADDFD_FLAG_SEND: u32 = 1 << 1; +const SECCOMP_USER_NOTIF_FLAG_CONTINUE: u32 = 1; + +const CONNECTED_SEND_FLAGS: u32 = + (libc::MSG_DONTWAIT | libc::MSG_EOR | libc::MSG_MORE | libc::MSG_NOSIGNAL | libc::MSG_OOB) + as u32; +const PROBE_NOTIFICATION_TIMEOUT: Duration = Duration::from_secs(5); + +const IOC_NRBITS: u32 = 8; +const IOC_TYPEBITS: u32 = 8; +const IOC_SIZEBITS: u32 = 14; +const IOC_NRSHIFT: u32 = 0; +const IOC_TYPESHIFT: u32 = IOC_NRSHIFT + IOC_NRBITS; +const IOC_SIZESHIFT: u32 = IOC_TYPESHIFT + IOC_TYPEBITS; +const IOC_DIRSHIFT: u32 = IOC_SIZESHIFT + IOC_SIZEBITS; +const IOC_WRITE: u32 = 1; +const IOC_READ: u32 = 2; +const SECCOMP_IOC_MAGIC: u32 = b'!' as u32; + +#[allow(clippy::cast_possible_truncation)] +const fn ioc(direction: u32, number: u32, size: usize) -> libc::c_ulong { + ((direction << IOC_DIRSHIFT) + | (SECCOMP_IOC_MAGIC << IOC_TYPESHIFT) + | (number << IOC_NRSHIFT) + | ((size as u32) << IOC_SIZESHIFT)) as libc::c_ulong +} + +const fn iowr(number: u32) -> libc::c_ulong { + ioc(IOC_READ | IOC_WRITE, number, size_of::()) +} + +const fn iow(number: u32) -> libc::c_ulong { + ioc(IOC_WRITE, number, size_of::()) +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct SeccompData { + nr: i32, + arch: u32, + instruction_pointer: u64, + args: [u64; 6], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct RawNotification { + id: u64, + pid: u32, + flags: u32, + data: SeccompData, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct RawResponse { + id: u64, + val: i64, + error: i32, + flags: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct RawAddFd { + id: u64, + flags: u32, + srcfd: u32, + newfd: u32, + newfd_flags: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct RawNotificationSizes { + notification: u16, + response: u16, + data: u16, +} + +const SECCOMP_IOCTL_NOTIF_RECV: libc::c_ulong = iowr::(0); +const SECCOMP_IOCTL_NOTIF_SEND: libc::c_ulong = iowr::(1); +const SECCOMP_IOCTL_NOTIF_ID_VALID: libc::c_ulong = iow::(2); +const SECCOMP_IOCTL_NOTIF_ADDFD: libc::c_ulong = iow::(3); + +/// One validated seccomp user-notification request. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Notification { + /// Kernel-unique notification identifier. + pub id: u64, + /// Notifying Linux thread ID. + pub tid: u32, + /// Native syscall number. + pub syscall: i32, + /// Raw syscall arguments. + pub args: [u64; 6], +} + +/// Result of exercising the unprivileged notification API under the active +/// kernel, outer seccomp profile, and LSM posture. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NotificationProbeReport { + /// Whether `SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` was accepted. + pub wait_killable_recv: bool, + features: NotificationProbeFeatures, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct NotificationProbeFeatures(u8); + +impl NotificationProbeReport { + /// Whether ID validation and response delivery completed. + #[must_use] + pub fn notification_round_trip(self) -> bool { + self.features.0 & 1 != 0 + } + + /// Whether atomic ADDFD-SEND injected a close-on-exec descriptor. + #[must_use] + pub fn addfd_send(self) -> bool { + self.features.0 & 2 != 0 + } + + /// Whether process-VM read and write syscalls are admitted for same-process + /// memory, before the stronger child-credential probe runs in a driver. + #[must_use] + pub fn task_memory_copy(self) -> bool { + self.features.0 & 4 != 0 + } + + /// Whether connected null-destination `sendto` bypassed notification while + /// destination-bearing and unsafe-flag variants remained mediated. + #[must_use] + pub fn connected_send_fast_path(self) -> bool { + self.features.0 & 8 != 0 + } +} + +/// Owned listener returned by `SECCOMP_FILTER_FLAG_NEW_LISTENER`. +pub struct NotificationListener { + fd: OwnedFd, + wait_killable_recv: bool, +} + +impl NotificationListener { + /// Raw listener descriptor for readiness integration and diagnostics. + #[must_use] + pub fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } + + /// Whether the listener was installed with killable receive waits. + #[must_use] + pub fn wait_killable_recv(&self) -> bool { + self.wait_killable_recv + } + + /// Receive the next kernel notification. + pub fn receive(&self) -> io::Result { + let mut raw = RawNotification::default(); + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_RECV, + std::ptr::addr_of_mut!(raw).cast(), + )?; + Ok(Notification { + id: raw.id, + tid: raw.pid, + syscall: raw.data.nr, + args: raw.data.args, + }) + } + + /// Verify that a notification still refers to a blocked live task. + pub fn validate_id(&self, id: u64) -> io::Result<()> { + let mut id = id; + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_ID_VALID, + std::ptr::addr_of_mut!(id).cast(), + )?; + Ok(()) + } + + /// Return a successful scalar result to the notifying syscall. + pub fn respond_value(&self, id: u64, value: i64) -> io::Result<()> { + self.validate_id(id)?; + let mut response = RawResponse { + id, + val: value, + error: 0, + flags: 0, + }; + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_SEND, + std::ptr::addr_of_mut!(response).cast(), + )?; + Ok(()) + } + + /// Return `errno` to the notifying syscall. + pub fn respond_errno(&self, id: u64, errno: i32) -> io::Result<()> { + if errno <= 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "seccomp response errno must be positive", + )); + } + self.validate_id(id)?; + let mut response = RawResponse { + id, + val: 0, + error: -errno, + flags: 0, + }; + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_SEND, + std::ptr::addr_of_mut!(response).cast(), + )?; + Ok(()) + } + + /// Continue a verified local-kernel operation in the notifying task. + /// + /// Callers must not use this for an external INET operation or where a + /// mutable workload pointer is part of the authorization decision. + pub fn respond_continue(&self, id: u64) -> io::Result<()> { + self.validate_id(id)?; + let mut response = RawResponse { + id, + val: 0, + error: 0, + flags: SECCOMP_USER_NOTIF_FLAG_CONTINUE, + }; + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_SEND, + std::ptr::addr_of_mut!(response).cast(), + )?; + Ok(()) + } + + /// Atomically inject `source` and complete the notifying syscall with the + /// allocated target FD. The target receives `O_CLOEXEC` when requested. + pub fn add_fd_and_send( + &self, + notification_id: u64, + source: RawFd, + close_on_exec: bool, + ) -> io::Result { + self.validate_id(notification_id)?; + let srcfd = u32::try_from(source) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "source FD is negative"))?; + let mut addfd = RawAddFd { + id: notification_id, + flags: SECCOMP_ADDFD_FLAG_SEND, + srcfd, + newfd: 0, + newfd_flags: if close_on_exec { + u32::try_from(libc::O_CLOEXEC).expect("O_CLOEXEC fits u32") + } else { + 0 + }, + }; + ioctl_ptr( + self.fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_ADDFD, + std::ptr::addr_of_mut!(addfd).cast(), + ) + .and_then(|fd| { + RawFd::try_from(fd).map_err(|_| io::Error::other("injected FD does not fit RawFd")) + }) + } +} + +/// Install a non-TSYNC listener filter on the calling thread. +/// +/// Only the named syscalls notify. Unexpected architectures are killed, x32 +/// syscalls are killed on x86-64, and all other native syscalls are allowed. +pub fn install_listener(syscalls: &[i64]) -> io::Result { + if syscalls.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "at least one notified syscall is required", + )); + } + verify_notification_sizes()?; + set_no_new_privileges()?; + + install_listener_with_flags(syscalls, true).map_err(|error| { + if error.raw_os_error() == Some(libc::EINVAL) { + io::Error::new( + io::ErrorKind::Unsupported, + "seccomp WAIT_KILLABLE_RECV is required (Linux 5.19 or newer)", + ) + } else { + error + } + }) +} + +/// Install the capability-free workload networking listener on the calling +/// launcher thread. +/// +/// The filter mediates every syscall that can create, select, or materially +/// reconfigure an INET endpoint. Connected `send()`/null-destination +/// `sendto()` retains the audited cBPF fast path. +pub fn install_workload_listener() -> io::Result { + install_listener(&[ + libc::SYS_socket, + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + libc::SYS_sendto, + libc::SYS_sendmsg, + libc::SYS_sendmmsg, + libc::SYS_getpeername, + libc::SYS_setsockopt, + ]) +} + +/// Run a no-capability conformance probe. +/// +/// This uses the production launcher-thread shape: the listener is created on +/// one dedicated thread and moved to an unfiltered broker thread through an +/// in-process channel. +pub fn probe_notification_api() -> io::Result { + let wait_killable_recv = probe_scalar_round_trip()?; + probe_addfd_send()?; + probe_task_memory_copy()?; + probe_connected_sendto_fast_path()?; + Ok(NotificationProbeReport { + wait_killable_recv, + features: NotificationProbeFeatures(1 | 2 | 4 | 8), + }) +} + +fn probe_scalar_round_trip() -> io::Result { + const PROBE_VALUE: libc::c_long = 0x5a17; + let (sender, receiver) = mpsc::sync_channel(1); + let launcher = thread::spawn(move || -> io::Result { + let listener = install_listener(&[libc::SYS_getppid])?; + let wait_killable = listener.wait_killable_recv(); + sender + .send((listener, wait_killable)) + .map_err(|_| io::Error::other("notification broker disappeared"))?; + // SAFETY: getppid has no pointer arguments. The installed filter causes + // the kernel to block here until the broker validates and responds. + Ok(unsafe { libc::syscall(libc::SYS_getppid) }) + }); + + let (listener, wait_killable) = receiver + .recv() + .map_err(|_| io::Error::other("notification launcher disappeared"))?; + let notification = match receive_probe_notification(&listener) { + Ok(notification) => notification, + Err(error) => { + let _ = launcher.join(); + return Err(error); + } + }; + if i64::from(notification.syscall) != libc::SYS_getppid { + return Err(io::Error::other("unexpected scalar probe syscall")); + } + listener.respond_value(notification.id, PROBE_VALUE)?; + let observed = launcher + .join() + .map_err(|_| io::Error::other("notification launcher panicked"))??; + if observed != PROBE_VALUE { + return Err(io::Error::other("seccomp response value was not delivered")); + } + Ok(wait_killable) +} + +fn probe_addfd_send() -> io::Result<()> { + let (sender, receiver) = mpsc::sync_channel(1); + let launcher = thread::spawn(move || -> io::Result<()> { + let listener = install_listener(&[libc::SYS_socket])?; + sender + .send(listener) + .map_err(|_| io::Error::other("ADDFD broker disappeared"))?; + // SAFETY: arguments are scalar constants; the intercepted syscall is + // completed by ADDFD-SEND and returns the injected descriptor number. + let injected = unsafe { + libc::syscall( + libc::SYS_socket, + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + if injected < 0 { + return Err(io::Error::last_os_error()); + } + let injected = RawFd::try_from(injected) + .map_err(|_| io::Error::other("injected descriptor does not fit RawFd"))?; + // SAFETY: ADDFD-SEND returned one newly owned descriptor to this task. + let injected = unsafe { OwnedFd::from_raw_fd(injected) }; + // SAFETY: `injected` was returned as an open descriptor by the kernel. + let descriptor_flags = unsafe { libc::fcntl(injected.as_raw_fd(), libc::F_GETFD) }; + if descriptor_flags < 0 { + return Err(io::Error::last_os_error()); + } + if descriptor_flags & libc::FD_CLOEXEC == 0 { + return Err(io::Error::other("ADDFD did not preserve close-on-exec")); + } + let mut value = 0_u64; + // SAFETY: eventfd reads exactly one u64 into a valid aligned pointer. + let read = unsafe { + libc::read( + injected.as_raw_fd(), + std::ptr::addr_of_mut!(value).cast(), + size_of::(), + ) + }; + let word_size = isize::try_from(size_of::()).expect("u64 size fits isize"); + if read != word_size || value != 7 { + return Err(io::Error::other("injected eventfd was not usable")); + } + Ok(()) + }); + + let listener = receiver + .recv() + .map_err(|_| io::Error::other("ADDFD launcher disappeared"))?; + let notification = match receive_probe_notification(&listener) { + Ok(notification) => notification, + Err(error) => { + let _ = launcher.join(); + return Err(error); + } + }; + if i64::from(notification.syscall) != libc::SYS_socket { + return Err(io::Error::other("unexpected ADDFD probe syscall")); + } + // SAFETY: eventfd has no pointer arguments and returns an owned descriptor. + let source = unsafe { libc::eventfd(7, libc::EFD_CLOEXEC) }; + if source < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: eventfd returned a new owned descriptor. + let source = unsafe { OwnedFd::from_raw_fd(source) }; + listener.add_fd_and_send(notification.id, source.as_raw_fd(), true)?; + launcher + .join() + .map_err(|_| io::Error::other("ADDFD launcher panicked"))??; + Ok(()) +} + +fn probe_task_memory_copy() -> io::Result<()> { + let source = 0x1122_3344_5566_7788_u64; + let tid = std::process::id(); + let mut source_bytes = [0_u8; size_of::()]; + super::task_memory::read_exact(tid, std::ptr::addr_of!(source) as u64, &mut source_bytes)?; + let mut copied = u64::from_ne_bytes(source_bytes); + if copied != source { + return Err(io::Error::other("task-memory probe read wrong value")); + } + + let replacement = 0xaabb_ccdd_eeff_0011_u64; + super::task_memory::write_exact( + tid, + std::ptr::addr_of_mut!(copied) as u64, + &replacement.to_ne_bytes(), + )?; + if copied != replacement { + return Err(io::Error::other("task-memory probe wrote wrong value")); + } + Ok(()) +} + +fn probe_connected_sendto_fast_path() -> io::Result<()> { + let mut pair = [-1; 2]; + // SAFETY: `pair` points to storage for exactly two returned descriptors. + let result = unsafe { + libc::socketpair( + libc::AF_UNIX, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + 0, + pair.as_mut_ptr(), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful socketpair returned two independently owned FDs. + let sender_fd = unsafe { OwnedFd::from_raw_fd(pair[0]) }; + // SAFETY: successful socketpair returned two independently owned FDs. + let receiver_fd = unsafe { OwnedFd::from_raw_fd(pair[1]) }; + + let (sender, receiver) = mpsc::sync_channel(1); + let launcher = thread::spawn(move || -> io::Result<()> { + let listener = install_listener(&[libc::SYS_sendto])?; + sender + .send(listener) + .map_err(|_| io::Error::other("sendto broker disappeared"))?; + + let direct = b"direct"; + // SAFETY: the buffer is live and a null destination on this connected + // socket is equivalent to send(). The filter must allow this call + // without a broker round trip. + let sent = unsafe { + libc::sendto( + sender_fd.as_raw_fd(), + direct.as_ptr().cast(), + direct.len(), + libc::MSG_NOSIGNAL, + std::ptr::null(), + 0, + ) + }; + if sent != isize::try_from(direct.len()).expect("probe length fits isize") { + return Err(io::Error::last_os_error()); + } + + let mut payload = [0_u8; 6]; + // SAFETY: the receive buffer is live for its full declared length. + let read = unsafe { + libc::read( + receiver_fd.as_raw_fd(), + payload.as_mut_ptr().cast(), + payload.len(), + ) + }; + if read != isize::try_from(payload.len()).expect("probe length fits isize") + || &payload != direct + { + return Err(io::Error::other( + "connected sendto fast path did not relay data", + )); + } + + let destination = libc::sockaddr_un { + sun_family: libc::sa_family_t::try_from(libc::AF_UNIX) + .expect("AF_UNIX fits sa_family_t"), + sun_path: [0; 108], + }; + // SAFETY: all pointers refer to live values. This deliberately + // destination-bearing call must be denied by the broker. + let result = unsafe { + libc::sendto( + sender_fd.as_raw_fd(), + direct.as_ptr().cast(), + direct.len(), + 0, + std::ptr::addr_of!(destination).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr family size fits socklen_t"), + ) + }; + if result != -1 || io::Error::last_os_error().raw_os_error() != Some(libc::EACCES) { + return Err(io::Error::other( + "destination-bearing sendto bypassed notification", + )); + } + + // A null destination with Fast Open must not use the connected-send + // fast path either. + // SAFETY: the live buffer and null address form a valid syscall; the + // broker supplies the expected denial. + let result = unsafe { + libc::sendto( + sender_fd.as_raw_fd(), + direct.as_ptr().cast(), + direct.len(), + libc::MSG_FASTOPEN, + std::ptr::null(), + 0, + ) + }; + if result != -1 || io::Error::last_os_error().raw_os_error() != Some(libc::EOPNOTSUPP) { + return Err(io::Error::other( + "MSG_FASTOPEN sendto bypassed notification", + )); + } + Ok(()) + }); + + let listener = receiver + .recv() + .map_err(|_| io::Error::other("sendto launcher disappeared"))?; + let destination = match receive_probe_notification(&listener) { + Ok(notification) => notification, + Err(error) => { + let _ = launcher.join(); + return Err(error); + } + }; + if i64::from(destination.syscall) != libc::SYS_sendto + || destination.args[4] == 0 + || destination.args[5] == 0 + { + return Err(io::Error::other( + "destination-bearing sendto notification was malformed", + )); + } + listener.respond_errno(destination.id, libc::EACCES)?; + + let fast_open = match receive_probe_notification(&listener) { + Ok(notification) => notification, + Err(error) => { + let _ = launcher.join(); + return Err(error); + } + }; + if i64::from(fast_open.syscall) != libc::SYS_sendto + || fast_open.args[4] != 0 + || fast_open.args[5] != 0 + || fast_open.args[3] & u64::from(libc::MSG_FASTOPEN as u32) == 0 + { + return Err(io::Error::other( + "Fast Open sendto notification was malformed", + )); + } + listener.respond_errno(fast_open.id, libc::EOPNOTSUPP)?; + + launcher + .join() + .map_err(|_| io::Error::other("sendto launcher panicked"))??; + Ok(()) +} + +fn receive_probe_notification(listener: &NotificationListener) -> io::Result { + let mut descriptor = libc::pollfd { + fd: listener.as_raw_fd(), + events: libc::POLLIN | libc::POLLHUP, + revents: 0, + }; + let timeout = i32::try_from(PROBE_NOTIFICATION_TIMEOUT.as_millis()) + .expect("probe timeout fits poll milliseconds"); + // SAFETY: descriptor points to one live pollfd for the duration of poll. + let ready = unsafe { libc::poll(&raw mut descriptor, 1, timeout) }; + if ready < 0 { + return Err(io::Error::last_os_error()); + } + if ready == 0 { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "seccomp notification probe timed out", + )); + } + if descriptor.revents & libc::POLLIN == 0 { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "seccomp notification probe listener closed", + )); + } + listener.receive() +} + +fn install_listener_with_flags( + syscalls: &[i64], + wait_killable_recv: bool, +) -> io::Result { + let mut program = build_filter(syscalls)?; + let length = u16::try_from(program.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "seccomp filter is too large"))?; + let mut fprog = libc::sock_fprog { + len: length, + filter: program.as_mut_ptr(), + }; + let flags = SECCOMP_FILTER_FLAG_NEW_LISTENER + | if wait_killable_recv { + SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV + } else { + 0 + }; + // SAFETY: `fprog` points to a live classic-BPF program for the duration of + // the syscall. The returned nonnegative value is a newly owned FD. + let result = unsafe { + libc::syscall( + libc::SYS_seccomp, + SECCOMP_SET_MODE_FILTER, + flags, + std::ptr::addr_of_mut!(fprog), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + let fd = RawFd::try_from(result) + .map_err(|_| io::Error::other("seccomp listener FD does not fit RawFd"))?; + // SAFETY: successful NEW_LISTENER returns one newly owned descriptor. + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + Ok(NotificationListener { + fd, + wait_killable_recv, + }) +} + +fn build_filter(syscalls: &[i64]) -> io::Result> { + let mut program = vec![ + stmt(BPF_LD_W_ABS, SECCOMP_DATA_ARCH_OFFSET), + jump(BPF_JMP_JEQ_K, native_audit_arch(), 1, 0), + stmt(BPF_RET_K, SECCOMP_RET_KILL_PROCESS), + stmt(BPF_LD_W_ABS, SECCOMP_DATA_NR_OFFSET), + ]; + + #[cfg(target_arch = "x86_64")] + program.extend([ + jump(BPF_JMP_JSET_K, X32_SYSCALL_BIT, 0, 1), + stmt(BPF_RET_K, SECCOMP_RET_KILL_PROCESS), + ]); + + let mut syscalls = syscalls.to_vec(); + syscalls.sort_unstable(); + syscalls.dedup(); + for syscall in syscalls { + if syscall == libc::SYS_sendto { + append_sendto_filter(&mut program)?; + continue; + } + let syscall = u32::try_from(syscall) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "negative syscall number"))?; + program.extend([ + jump(BPF_JMP_JEQ_K, syscall, 0, 1), + stmt(BPF_RET_K, SECCOMP_RET_USER_NOTIF), + ]); + } + program.push(stmt(BPF_RET_K, SECCOMP_RET_ALLOW)); + Ok(program) +} + +fn append_sendto_filter(program: &mut Vec) -> io::Result<()> { + const SPECIAL_LENGTH: u8 = 20; + let syscall = u32::try_from(libc::SYS_sendto) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "negative sendto syscall"))?; + program.push(jump(BPF_JMP_JEQ_K, syscall, 0, SPECIAL_LENGTH)); + + for offset in [ + argument_word_offset(4, 0), + argument_word_offset(4, 1), + argument_word_offset(5, 0), + argument_word_offset(5, 1), + argument_word_offset(3, 1), + ] { + program.extend([ + stmt(BPF_LD_W_ABS, offset), + jump(BPF_JMP_JEQ_K, 0, 1, 0), + stmt(BPF_RET_K, SECCOMP_RET_USER_NOTIF), + ]); + } + program.extend([ + stmt(BPF_LD_W_ABS, argument_word_offset(3, 0)), + stmt(BPF_ALU_AND_K, !CONNECTED_SEND_FLAGS), + jump(BPF_JMP_JEQ_K, 0, 1, 0), + stmt(BPF_RET_K, SECCOMP_RET_USER_NOTIF), + stmt(BPF_RET_K, SECCOMP_RET_ALLOW), + ]); + Ok(()) +} + +const fn argument_word_offset(argument: u32, word: u32) -> u32 { + SECCOMP_DATA_ARGS_OFFSET + argument * 8 + word * 4 +} + +#[cfg(target_arch = "x86_64")] +const fn native_audit_arch() -> u32 { + 0xc000_003e +} + +#[cfg(target_arch = "aarch64")] +const fn native_audit_arch() -> u32 { + 0xc000_00b7 +} + +const fn stmt(code: u16, value: u32) -> libc::sock_filter { + libc::sock_filter { + code, + jt: 0, + jf: 0, + k: value, + } +} + +const fn jump(code: u16, value: u32, jt: u8, jf: u8) -> libc::sock_filter { + libc::sock_filter { + code, + jt, + jf, + k: value, + } +} + +fn set_no_new_privileges() -> io::Result<()> { + // SAFETY: PR_SET_NO_NEW_PRIVS accepts scalar arguments and only tightens + // the calling thread's privilege behavior. + let result = unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +fn verify_notification_sizes() -> io::Result<()> { + let mut sizes = RawNotificationSizes::default(); + // SAFETY: the kernel writes only the fixed-size `RawNotificationSizes` + // object supplied here. + let result = unsafe { + libc::syscall( + libc::SYS_seccomp, + SECCOMP_GET_NOTIF_SIZES, + 0, + std::ptr::addr_of_mut!(sizes), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + for (name, kernel, local) in [ + ( + "notification", + usize::from(sizes.notification), + size_of::(), + ), + ( + "response", + usize::from(sizes.response), + size_of::(), + ), + ("data", usize::from(sizes.data), size_of::()), + ] { + if kernel != local { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("kernel seccomp {name} size {kernel} differs from supported size {local}"), + )); + } + } + Ok(()) +} + +fn ioctl_ptr(fd: RawFd, request: libc::c_ulong, argument: *mut libc::c_void) -> io::Result { + // SAFETY: every caller supplies the UAPI structure encoded into `request`, + // alive and writable for the ioctl duration. + // `libc::ioctl` models the request as `c_ulong` for glibc and `c_int` + // for musl. Linux UAPI request values fit both representations. + #[cfg(target_env = "musl")] + let request = u32::try_from(request).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "ioctl request exceeds 32 bits") + })?; + #[cfg(target_env = "musl")] + let request = libc::c_int::from_ne_bytes(request.to_ne_bytes()); + let result = unsafe { libc::ioctl(fd, request, argument) }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(i64::from(result)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filter_rejects_empty_syscall_set() { + let error = install_listener(&[]).err().expect("empty filter must fail"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + + #[test] + fn active_notification_probe_passes() { + let report = probe_notification_api().expect("active notification probe"); + assert!(report.notification_round_trip()); + assert!(report.addfd_send()); + assert!(report.task_memory_copy()); + assert!(report.connected_send_fast_path()); + } + + #[test] + fn errno_response_rejects_nonpositive_values() { + // The input validation occurs before the listener FD is used. + // SAFETY: dup takes one valid descriptor and returns a new descriptor + // or a negative error without modifying memory. + let duplicated = unsafe { libc::dup(libc::STDERR_FILENO) }; + assert!(duplicated >= 0, "duplicate stderr for validation test"); + let listener = NotificationListener { + // SAFETY: successful dup returned a new owned descriptor. + fd: unsafe { OwnedFd::from_raw_fd(duplicated) }, + wait_killable_recv: false, + }; + let error = listener + .respond_errno(1, 0) + .expect_err("zero errno must fail"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/socket_registry.rs b/crates/openshell-isolation-interface/src/linux/socket_registry.rs new file mode 100644 index 0000000000..966612144e --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/socket_registry.rs @@ -0,0 +1,461 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded registry for socket-time seccomp virtualization. + +#![allow(unsafe_code)] + +use std::collections::BTreeMap; +use std::io; +use std::mem::size_of; +use std::net::SocketAddr; +use std::os::fd::{AsRawFd, BorrowedFd, OwnedFd, RawFd}; + +use rustix::fs::fstat; + +use super::proc_fd; + +/// Stable identity for one mediated socket within a listener generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SocketIdentity { + /// Generation of the seccomp listener that created the socket. + pub listener_generation: u64, + /// Socket inode observed from the source descriptor. + pub inode: u64, + /// Kernel `SO_COOKIE` value. + pub cookie: u64, +} + +/// Supported INET address family. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InetFamily { + /// `AF_INET`. + V4, + /// `AF_INET6`. + V6, +} + +/// Supported INET socket kind and protocol. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InetKind { + /// TCP stream socket. + Tcp, + /// UDP datagram socket restricted to the DNS relay. + DnsUdp, +} + +/// Immutable socket metadata captured before ADDFD-SEND. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SocketMetadata { + /// Address family. + pub family: InetFamily, + /// Socket kind/protocol. + pub kind: InetKind, + /// Whether the injected descriptor must be close-on-exec. + pub close_on_exec: bool, + /// Whether the socket's open-file description is nonblocking. + pub nonblocking: bool, + /// Task generation that created the socket. + pub creator_generation: u64, +} + +/// Stable state of one socket open-file description. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SocketState { + /// Created but not bound or connected. + Created, + /// Explicitly bound by the workload. + Bound { local: SocketAddr }, + /// Connected through an external supervisor relay. + Connected { original_peer: SocketAddr }, + /// Connected directly to an allowed workload loopback endpoint. + Local { peer: SocketAddr }, + /// UDP socket pinned to the exact local DNS relay. + DnsUdp { relay: SocketAddr }, + /// TCP socket pinned to the exact local DNS relay. + DnsTcp { relay: SocketAddr }, + /// Workload-owned listening socket. + Listening { local: SocketAddr }, + /// Stream accepted from a verified local peer. + AcceptedLocal { peer: SocketAddr }, + /// A committed relay failed after connection. + Failed { errno: i32 }, +} + +/// One committed registry entry. +#[derive(Debug)] +pub struct SocketEntry { + identity: SocketIdentity, + metadata: SocketMetadata, + state: SocketState, + retained_preconnect: Option, +} + +impl SocketEntry { + /// Stable socket identity. + #[must_use] + pub fn identity(&self) -> SocketIdentity { + self.identity + } + + /// Immutable creation metadata. + #[must_use] + pub fn metadata(&self) -> SocketMetadata { + self.metadata + } + + /// Current stable state. + #[must_use] + pub fn state(&self) -> &SocketState { + &self.state + } + + /// Retained source descriptor used to perform pre-connect operations on + /// the exact injected open-file description. + pub fn retained_preconnect(&self) -> io::Result<&OwnedFd> { + self.retained_preconnect.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotConnected, + "socket no longer has a retained pre-connect descriptor", + ) + }) + } + + /// Replace the stable state. Callers perform policy and notification + /// revalidation before invoking this commit primitive. + pub fn set_state(&mut self, state: SocketState) { + self.state = state; + } + + /// Close the temporary source descriptor after a connection commits. + pub fn release_preconnect(&mut self) { + self.retained_preconnect = None; + } + + /// Verify that the retained source still has the registered cookie and + /// inode. + pub fn validate_retained_identity(&self) -> io::Result<()> { + let retained = self.retained_preconnect()?; + let identity = socket_identity(retained.as_raw_fd(), self.identity.listener_generation)?; + if identity == self.identity { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "retained socket identity changed", + )) + } + } +} + +/// Tentative socket metadata that is invisible until ADDFD-SEND succeeds. +#[derive(Debug)] +pub struct TentativeSocket { + identity: SocketIdentity, + metadata: SocketMetadata, + source: OwnedFd, +} + +impl TentativeSocket { + /// Stable identity used to correlate the ADDFD transaction. + #[must_use] + pub fn identity(&self) -> SocketIdentity { + self.identity + } + + /// Source descriptor passed to ADDFD-SEND. + #[must_use] + pub fn source_fd(&self) -> RawFd { + self.source.as_raw_fd() + } +} + +/// Bounded committed socket registry. +pub struct SocketRegistry { + listener_generation: u64, + capacity: usize, + entries: BTreeMap, +} + +impl SocketRegistry { + /// Create an empty registry for one nonzero listener generation. + pub fn new(listener_generation: u64, capacity: usize) -> io::Result { + if listener_generation == 0 || capacity == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "listener generation and registry capacity must be nonzero", + )); + } + Ok(Self { + listener_generation, + capacity, + entries: BTreeMap::new(), + }) + } + + /// Number of committed entries. + #[must_use] + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether no sockets are committed. + #[must_use] + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Whether another socket would exceed the configured bound. + #[must_use] + pub fn is_full(&self) -> bool { + self.entries.len() >= self.capacity + } + + /// Retain only sockets that remain installed in a workload descriptor + /// table. The trusted broker's temporary source descriptors are excluded + /// from `installed` by the caller. + pub fn retain_installed(&mut self, installed: &std::collections::BTreeSet) { + self.entries + .retain(|inode, _entry| installed.contains(inode)); + } + + /// Stage a newly created source descriptor without publishing it. + pub fn stage(&self, source: OwnedFd, metadata: SocketMetadata) -> io::Result { + if self.entries.len() >= self.capacity { + return Err(io::Error::from_raw_os_error(libc::EMFILE)); + } + let identity = socket_identity(source.as_raw_fd(), self.listener_generation)?; + if self.entries.contains_key(&identity.inode) { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "socket inode is already registered", + )); + } + Ok(TentativeSocket { + identity, + metadata, + source, + }) + } + + /// Publish a tentative socket only after ADDFD-SEND has succeeded. + pub fn commit(&mut self, tentative: TentativeSocket) -> io::Result { + self.commit_with_state(tentative, SocketState::Created) + } + + /// Publish a tentative socket in a caller-proven initial state. + /// + /// Accepted sockets are created and classified by the trusted broker, so + /// they enter the registry directly as [`SocketState::AcceptedLocal`] + /// rather than pretending to be unconnected. + pub fn commit_with_state( + &mut self, + tentative: TentativeSocket, + state: SocketState, + ) -> io::Result { + if self.entries.len() >= self.capacity { + return Err(io::Error::from_raw_os_error(libc::EMFILE)); + } + if tentative.identity.listener_generation != self.listener_generation + || self.entries.contains_key(&tentative.identity.inode) + { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "socket identity cannot be committed to this registry", + )); + } + let identity = tentative.identity; + let retain_source = matches!( + state, + SocketState::Created | SocketState::Bound { .. } | SocketState::Listening { .. } + ); + self.entries.insert( + identity.inode, + SocketEntry { + identity, + metadata: tentative.metadata, + state, + retained_preconnect: retain_source.then_some(tentative.source), + }, + ); + Ok(identity) + } + + /// Resolve a notifying task's installed descriptor to a committed entry. + pub fn resolve(&self, tid: u32, fd: RawFd) -> io::Result<&SocketEntry> { + let inode = proc_fd::socket_inode(tid, fd)?; + let entry = self.entries.get(&inode).ok_or_else(|| { + io::Error::new( + io::ErrorKind::PermissionDenied, + "socket inode is not registered for this sandbox", + ) + })?; + if entry.retained_preconnect.is_some() { + entry.validate_retained_identity()?; + } + Ok(entry) + } + + /// Mutable form of [`Self::resolve`]. + pub fn resolve_mut(&mut self, tid: u32, fd: RawFd) -> io::Result<&mut SocketEntry> { + let inode = proc_fd::socket_inode(tid, fd)?; + let entry = self.entries.get_mut(&inode).ok_or_else(|| { + io::Error::new( + io::ErrorKind::PermissionDenied, + "socket inode is not registered for this sandbox", + ) + })?; + if entry.retained_preconnect.is_some() { + entry.validate_retained_identity()?; + } + Ok(entry) + } + + /// Remove metadata after descendant-FD collection proves no installed + /// alias remains. + pub fn remove_inode(&mut self, inode: u64) -> bool { + self.entries.remove(&inode).is_some() + } +} + +fn socket_identity(fd: RawFd, listener_generation: u64) -> io::Result { + // SAFETY: `fd` remains open for this function; the borrow never escapes. + let borrowed = unsafe { BorrowedFd::borrow_raw(fd) }; + let stat = fstat(borrowed)?; + if stat.st_mode & libc::S_IFMT != libc::S_IFSOCK { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "registry source descriptor is not a socket", + )); + } + let mut cookie = 0_u64; + let mut length = + libc::socklen_t::try_from(size_of::()).expect("SO_COOKIE length fits socklen_t"); + // SAFETY: getsockopt writes at most the supplied u64 and socklen_t. + let result = unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_COOKIE, + std::ptr::addr_of_mut!(cookie).cast(), + std::ptr::addr_of_mut!(length), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if usize::try_from(length).ok() != Some(size_of::()) || cookie == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "kernel returned an invalid SO_COOKIE", + )); + } + Ok(SocketIdentity { + listener_generation, + inode: stat.st_ino, + cookie, + }) +} + +#[cfg(test)] +mod tests { + use std::os::fd::FromRawFd; + + use super::*; + + fn tcp_socket() -> OwnedFd { + // SAFETY: socket returns one newly owned descriptor on success. + let fd = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + assert!(fd >= 0, "socket: {}", io::Error::last_os_error()); + // SAFETY: successful socket returned one owned descriptor. + unsafe { OwnedFd::from_raw_fd(fd) } + } + + fn metadata() -> SocketMetadata { + SocketMetadata { + family: InetFamily::V4, + kind: InetKind::Tcp, + close_on_exec: true, + nonblocking: false, + creator_generation: 11, + } + } + + #[test] + fn tentative_entry_is_invisible_until_commit() { + let mut registry = SocketRegistry::new(7, 1).unwrap(); + let socket = tcp_socket(); + let fd = socket.as_raw_fd(); + let tentative = registry.stage(socket, metadata()).unwrap(); + assert!(registry.is_empty()); + assert_eq!( + registry + .resolve(std::process::id(), fd) + .expect_err("tentative socket must be invisible") + .kind(), + io::ErrorKind::PermissionDenied + ); + + let identity = registry.commit(tentative).unwrap(); + let entry = registry.resolve(std::process::id(), fd).unwrap(); + assert_eq!(entry.identity(), identity); + assert_eq!(entry.metadata(), metadata()); + assert_eq!(entry.state(), &SocketState::Created); + assert_eq!(registry.len(), 1); + + assert_eq!( + registry + .stage(tcp_socket(), metadata()) + .expect_err("quota must fail before injection") + .raw_os_error(), + Some(libc::EMFILE) + ); + } + + #[test] + fn dup_alias_resolves_to_same_open_file_description() { + let mut registry = SocketRegistry::new(9, 4).unwrap(); + let socket = tcp_socket(); + let original_fd = socket.as_raw_fd(); + // SAFETY: dup returns a new descriptor for the same open-file + // description or a negative error. + let alias_fd = unsafe { libc::dup(original_fd) }; + assert!(alias_fd >= 0, "dup: {}", io::Error::last_os_error()); + // SAFETY: successful dup returned one owned descriptor. + let alias = unsafe { OwnedFd::from_raw_fd(alias_fd) }; + + let tentative = registry.stage(socket, metadata()).unwrap(); + let identity = registry.commit(tentative).unwrap(); + assert_eq!( + registry + .resolve(std::process::id(), alias.as_raw_fd()) + .unwrap() + .identity(), + identity + ); + } + + #[test] + fn collection_reclaims_only_uninstalled_socket_metadata() { + let mut registry = SocketRegistry::new(12, 2).unwrap(); + let first = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + let second = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + assert!(registry.is_full()); + + registry.retain_installed(&std::collections::BTreeSet::from([first.inode])); + + assert_eq!(registry.len(), 1); + assert!(registry.remove_inode(first.inode)); + assert!(!registry.remove_inode(second.inode)); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/task_memory.rs b/crates/openshell-isolation-interface/src/linux/task_memory.rs new file mode 100644 index 0000000000..4a0422a006 --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/task_memory.rs @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded, exact access to a notifying task's memory. +//! +//! Seccomp user-notification arguments contain addresses in the notifying +//! task. Callers must copy pointer-bearing inputs once into trusted memory and +//! must never treat a partial copy as valid. + +#![allow(unsafe_code)] + +use std::io; +use std::mem::size_of; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::fs::FileExt as _; + +/// Maximum number of task-memory bytes copied by one operation. +pub const MAX_TASK_MEMORY_COPY: usize = 64 * 1024; + +/// Read exactly `destination.len()` bytes from `address` in `tid`. +/// +/// Empty and oversized requests, null addresses, and partial reads fail +/// closed. The caller must still revalidate the notification and task +/// generation after the copy. +pub fn read_exact(tid: u32, address: u64, destination: &mut [u8]) -> io::Result<()> { + validate_request(tid, address, destination.len())?; + let pid = libc::pid_t::try_from(tid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "TID does not fit pid_t"))?; + let remote_address = usize::try_from(address).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "remote address does not fit usize", + ) + })?; + let local = libc::iovec { + iov_base: destination.as_mut_ptr().cast(), + iov_len: destination.len(), + }; + let remote = libc::iovec { + iov_base: remote_address as *mut libc::c_void, + iov_len: destination.len(), + }; + + // SAFETY: the local iovec spans the caller-provided live buffer. The + // remote address is untrusted but bounded; the kernel validates it in the + // target process and returns EFAULT or a short count when unavailable. + let copied = retry_eintr(|| unsafe { + libc::process_vm_readv( + pid, + std::ptr::addr_of!(local), + 1, + std::ptr::addr_of!(remote), + 1, + 0, + ) + }); + match copied { + Ok(copied) => require_exact(copied, destination.len(), "task-memory read"), + Err(error) if syscall_profile_denied(&error) => { + read_exact_from_proc_mem(tid, address, destination) + } + Err(error) => Err(error), + } +} + +/// Write exactly all of `source` to `address` in `tid`. +/// +/// This is used only for syscall outputs such as `getpeername` and +/// `sendmmsg.msg_len`. Revalidate the notification, task generation, and +/// destination layout immediately before calling it. +pub fn write_exact(tid: u32, address: u64, source: &[u8]) -> io::Result<()> { + validate_request(tid, address, source.len())?; + let pid = libc::pid_t::try_from(tid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "TID does not fit pid_t"))?; + let remote_address = usize::try_from(address).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "remote address does not fit usize", + ) + })?; + let local = libc::iovec { + iov_base: source.as_ptr().cast_mut().cast(), + iov_len: source.len(), + }; + let remote = libc::iovec { + iov_base: remote_address as *mut libc::c_void, + iov_len: source.len(), + }; + + // SAFETY: the local iovec spans the caller-provided live buffer. The + // remote address is untrusted but bounded; the kernel validates that it is + // writable in the target process. + let copied = retry_eintr(|| unsafe { + libc::process_vm_writev( + pid, + std::ptr::addr_of!(local), + 1, + std::ptr::addr_of!(remote), + 1, + 0, + ) + }); + match copied { + Ok(copied) => require_exact(copied, source.len(), "task-memory write"), + Err(error) if syscall_profile_denied(&error) => { + write_exact_to_proc_mem(tid, address, source) + } + Err(error) => Err(error), + } +} + +fn syscall_profile_denied(error: &io::Error) -> bool { + matches!( + error.raw_os_error(), + Some(libc::EPERM | libc::EACCES | libc::ENOSYS) + ) +} + +fn read_exact_from_proc_mem(tid: u32, address: u64, destination: &mut [u8]) -> io::Result<()> { + let file = std::fs::File::open(format!("/proc/{tid}/mem"))?; + let copied = file.read_at(destination, address)?; + require_exact(copied, destination.len(), "proc task-memory read") +} + +fn write_exact_to_proc_mem(tid: u32, address: u64, source: &[u8]) -> io::Result<()> { + let file = std::fs::OpenOptions::new() + .write(true) + .open(format!("/proc/{tid}/mem"))?; + let copied = file.write_at(source, address)?; + require_exact(copied, source.len(), "proc task-memory write") +} + +/// Prove same-UID parent-to-child read and write access under the active Yama, +/// LSM, and outer seccomp posture. +/// +/// Call this only from a single-threaded probe process. The child executes +/// raw, allocation-free syscalls between `fork` and `_exit`. +pub fn probe_child_access() -> io::Result<()> { + const INITIAL: u64 = 0x1122_3344_5566_7788; + const REPLACEMENT: u64 = 0xaabb_ccdd_eeff_0011; + // SAFETY: mmap creates one private anonymous page owned by this process. + let mapping = unsafe { + libc::mmap( + std::ptr::null_mut(), + size_of::(), + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + return Err(io::Error::last_os_error()); + } + let mapping_address = mapping as u64; + // SAFETY: mapping spans at least one aligned u64-sized region. + unsafe { mapping.cast::().write(INITIAL) }; + + // SAFETY: eventfd returns independently owned descriptors on success. + let ready = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if ready < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + // SAFETY: successful eventfd returned one owned descriptor. + let ready = unsafe { OwnedFd::from_raw_fd(ready) }; + // SAFETY: eventfd returns independently owned descriptors on success. + let proceed = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if proceed < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + // SAFETY: successful eventfd returned one owned descriptor. + let proceed = unsafe { OwnedFd::from_raw_fd(proceed) }; + + // SAFETY: the caller promises this probe process is single-threaded. The + // child performs only raw syscalls and memory operations before `_exit`. + let child = unsafe { libc::fork() }; + if child < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + if child == 0 { + // The sandbox remains nondumpable, but an exec'd workload must be + // observable by its same-UID ancestor. This child contains no trusted + // parent address space secrets beyond this synthetic probe value. + // SAFETY: these calls use live inherited eventfds and scalar prctl + // arguments. No Rust cleanup runs in the child. + unsafe { + if libc::prctl(libc::PR_SET_DUMPABLE, 1, 0, 0, 0) < 0 + || write_eventfd(ready.as_raw_fd()).is_err() + || read_eventfd(proceed.as_raw_fd()).is_err() + || mapping.cast::().read() != REPLACEMENT + { + libc::_exit(1); + } + libc::_exit(0); + } + } + + let outcome = (|| { + read_eventfd(ready.as_raw_fd())?; + let mut observed = [0_u8; size_of::()]; + read_exact( + u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?, + mapping_address, + &mut observed, + )?; + if u64::from_ne_bytes(observed) != INITIAL { + return Err(io::Error::other( + "cross-child memory read returned wrong data", + )); + } + write_exact( + u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?, + mapping_address, + &REPLACEMENT.to_ne_bytes(), + )?; + write_eventfd(proceed.as_raw_fd())?; + let mut status = 0; + // SAFETY: child is a live direct child and status points to storage. + if unsafe { libc::waitpid(child, std::ptr::addr_of_mut!(status), 0) } != child { + return Err(io::Error::last_os_error()); + } + if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 { + return Err(io::Error::other("cross-child memory probe failed in child")); + } + Ok(()) + })(); + + if outcome.is_err() { + // SAFETY: a failed parent-side operation may leave this direct child + // blocked on eventfd. SIGKILL and waitpid guarantee cleanup. + unsafe { + libc::kill(child, libc::SIGKILL); + libc::waitpid(child, std::ptr::null_mut(), 0); + } + } + // SAFETY: mapping is the live region returned above and no child remains. + unsafe { libc::munmap(mapping, size_of::()) }; + outcome +} + +fn read_eventfd(fd: libc::c_int) -> io::Result<()> { + let mut value = 0_u64; + // SAFETY: eventfd reads exactly one u64 into live storage. + let result = unsafe { libc::read(fd, std::ptr::addr_of_mut!(value).cast(), size_of::()) }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + require_exact( + usize::try_from(result).map_err(|_| io::Error::other("eventfd read length invalid"))?, + size_of::(), + "eventfd read", + ) +} + +fn write_eventfd(fd: libc::c_int) -> io::Result<()> { + let value = 1_u64; + // SAFETY: eventfd reads exactly one u64 from live storage. + let result = unsafe { libc::write(fd, std::ptr::addr_of!(value).cast(), size_of::()) }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + require_exact( + usize::try_from(result).map_err(|_| io::Error::other("eventfd write length invalid"))?, + size_of::(), + "eventfd write", + ) +} +fn validate_request(tid: u32, address: u64, length: usize) -> io::Result<()> { + if tid == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "task-memory TID must be nonzero", + )); + } + if address == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "task-memory address must be nonzero", + )); + } + if length == 0 || length > MAX_TASK_MEMORY_COPY { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("task-memory copy length must be between 1 and {MAX_TASK_MEMORY_COPY} bytes"), + )); + } + let start = usize::try_from(address) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "remote address is too large"))?; + start.checked_add(length - 1).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "task-memory address range overflows", + ) + })?; + Ok(()) +} + +fn retry_eintr(mut operation: impl FnMut() -> isize) -> io::Result { + loop { + let result = operation(); + if result >= 0 { + return usize::try_from(result) + .map_err(|_| io::Error::other("task-memory result does not fit usize")); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } +} + +fn require_exact(copied: usize, expected: usize, operation: &str) -> io::Result<()> { + if copied == expected { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{operation} was partial: copied {copied} of {expected} bytes"), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_and_writes_exact_same_process_memory() { + let source = 0x1122_3344_5566_7788_u64; + let mut destination = 0_u64; + let mut bytes = [0_u8; size_of::()]; + + read_exact( + std::process::id(), + std::ptr::addr_of!(source) as u64, + &mut bytes, + ) + .expect("read source"); + assert_eq!(u64::from_ne_bytes(bytes), source); + + let replacement = 0xaabb_ccdd_eeff_0011_u64; + write_exact( + std::process::id(), + std::ptr::addr_of_mut!(destination) as u64, + &replacement.to_ne_bytes(), + ) + .expect("write destination"); + assert_eq!(destination, replacement); + } + + #[test] + fn proc_mem_fallback_reads_and_writes_exact_memory() { + let source = 0x0102_0304_0506_0708_u64; + let mut destination = 0_u64; + let mut bytes = [0_u8; size_of::()]; + read_exact_from_proc_mem( + std::process::id(), + std::ptr::addr_of!(source) as u64, + &mut bytes, + ) + .expect("read through proc mem"); + assert_eq!(u64::from_ne_bytes(bytes), source); + + write_exact_to_proc_mem( + std::process::id(), + std::ptr::addr_of_mut!(destination) as u64, + &source.to_ne_bytes(), + ) + .expect("write through proc mem"); + assert_eq!(destination, source); + } + + #[test] + fn rejects_invalid_ranges_before_syscall() { + let mut byte = [0_u8; 1]; + assert_eq!( + read_exact(0, 1, &mut byte).expect_err("zero TID").kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + read_exact(std::process::id(), 0, &mut byte) + .expect_err("null address") + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + read_exact(std::process::id(), 1, &mut []) + .expect_err("empty copy") + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + validate_request(std::process::id(), 1, MAX_TASK_MEMORY_COPY + 1) + .expect_err("oversized copy") + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + validate_request(std::process::id(), u64::MAX, 2) + .expect_err("overflowing range") + .kind(), + io::ErrorKind::InvalidInput + ); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/workload_launcher.rs b/crates/openshell-isolation-interface/src/linux/workload_launcher.rs new file mode 100644 index 0000000000..9197cf28f0 --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/workload_launcher.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One-listener launch thread for capability-free workload descendants. +//! +//! Seccomp filters are per-thread. This launcher installs the networking +//! listener without TSYNC, then serializes every fork/exec operation on that +//! thread. Children inherit the filter while the sandbox's broker and +//! lifecycle threads remain unfiltered. The listener moves to the caller over +//! an in-process channel; no descriptor handoff syscall or reusable exception +//! is needed. + +use std::io; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::thread; + +use super::seccomp_notify::{NotificationListener, install_workload_listener}; + +type LaunchJob = Box; + +/// Serialized child-launch executor whose thread owns the inherited listener +/// filter. +#[derive(Clone)] +pub struct WorkloadLauncher { + jobs: mpsc::SyncSender, + alive: Arc, +} + +impl WorkloadLauncher { + /// Execute one prebuilt spawn operation on the filtered launcher thread. + /// + /// The closure must only perform audited launch work. It must not open an + /// INET socket itself: the launcher is trusted and deliberately has no + /// notification broker. + pub fn execute( + &self, + operation: impl FnOnce() -> T + Send + 'static, + ) -> io::Result { + if !self.alive.load(Ordering::Acquire) { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "workload launcher is not running", + )); + } + let (result_tx, result_rx) = mpsc::sync_channel(1); + self.jobs + .send(Box::new(move || { + let _ = result_tx.send(operation()); + })) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "workload launcher stopped"))?; + result_rx.recv().map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "workload launcher dropped the spawn result", + ) + }) + } + + /// Whether the launch thread is still able to accept work. + #[must_use] + pub fn is_alive(&self) -> bool { + self.alive.load(Ordering::Acquire) + } +} + +/// Start the only workload launcher and return its listener to an unfiltered +/// sandbox thread. +pub fn start() -> io::Result<(WorkloadLauncher, NotificationListener)> { + let (jobs_tx, jobs_rx) = mpsc::sync_channel::(64); + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let alive = Arc::new(AtomicBool::new(true)); + let thread_alive = alive.clone(); + thread::Builder::new() + .name("openshell-workload-launcher".to_string()) + .spawn(move || { + match install_workload_listener() { + Ok(listener) => { + if ready_tx.send(Ok(listener)).is_err() { + thread_alive.store(false, Ordering::Release); + return; + } + } + Err(error) => { + let _ = ready_tx.send(Err(io::Error::new( + error.kind(), + format!("install workload listener: {error}"), + ))); + thread_alive.store(false, Ordering::Release); + return; + } + } + while let Ok(job) = jobs_rx.recv() { + job(); + } + thread_alive.store(false, Ordering::Release); + }) + .map_err(|error| io::Error::other(format!("start workload launcher thread: {error}")))?; + + let listener = ready_rx.recv().map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "workload launcher exited before publishing its listener", + ) + })??; + Ok(( + WorkloadLauncher { + jobs: jobs_tx, + alive, + }, + listener, + )) +} + +#[cfg(test)] +#[allow(unsafe_code)] +mod tests { + use std::mem::size_of; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + use super::*; + + #[test] + fn one_listener_mediates_launcher_and_inherited_child() { + let (launcher, listener) = start().expect("start launcher"); + let executable = std::env::current_exe().expect("test executable"); + let mut child = launcher + .execute(move || { + let mut command = std::process::Command::new(executable); + command + .arg("--exact") + .arg("linux::workload_launcher::tests::inherited_listener_child") + .arg("--nocapture") + .env("OPENSHELL_WORKLOAD_LAUNCHER_CHILD", "1"); + command.spawn() + }) + .expect("launcher result") + .expect("spawn child"); + let notification = listener.receive().expect("receive child socket"); + assert_eq!(i64::from(notification.syscall), libc::SYS_socket); + assert!( + std::path::Path::new(&format!("/proc/{}/task/{}", child.id(), notification.tid)) + .exists() + ); + // SAFETY: eventfd returns one newly owned descriptor on success. + let eventfd = unsafe { libc::eventfd(7, libc::EFD_CLOEXEC) }; + assert!(eventfd >= 0, "eventfd: {}", io::Error::last_os_error()); + // SAFETY: successful eventfd returned one owned descriptor. + let eventfd = unsafe { OwnedFd::from_raw_fd(eventfd) }; + listener + .add_fd_and_send(notification.id, eventfd.as_raw_fd(), true) + .expect("inject child descriptor"); + assert!(child.wait().expect("wait child").success()); + assert!(launcher.is_alive()); + assert!(listener.as_raw_fd() >= 0); + } + + #[test] + fn inherited_listener_child() { + if std::env::var_os("OPENSHELL_WORKLOAD_LAUNCHER_CHILD").is_none() { + return; + } + // SAFETY: the inherited listener intercepts this scalar socket call + // and returns the descriptor injected by the parent test. + let descriptor = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + assert!(descriptor >= 0, "socket: {}", io::Error::last_os_error()); + let mut value = 0_u64; + // SAFETY: the broker injected an eventfd and `value` is live storage. + let read = unsafe { + libc::read( + descriptor, + std::ptr::addr_of_mut!(value).cast(), + size_of::(), + ) + }; + // SAFETY: descriptor is owned by this process. + unsafe { libc::close(descriptor) }; + assert_eq!( + read, + isize::try_from(size_of::()).expect("u64 size fits") + ); + assert_eq!(value, 7); + } +} diff --git a/crates/openshell-isolation-interface/src/mediation.rs b/crates/openshell-isolation-interface/src/mediation.rs new file mode 100644 index 0000000000..8d809c7807 --- /dev/null +++ b/crates/openshell-isolation-interface/src/mediation.rs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Framing for the persistent sandbox-to-supervisor DNS data plane. +//! +//! TCP connections use independent streams on the authenticated HTTP/2 +//! transport so one busy connection cannot head-of-line block another. + +use std::io; +use std::net::SocketAddr; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; + +use crate::boundary_protocol::{BinaryIdentityWire, MediationTimingWire}; +use crate::contract::{DnsTransport, NetworkSocketMetadata}; + +const HEADER_BYTES: usize = 13; +const MAX_METADATA_BYTES: usize = 256 * 1024; +const MAX_FRAME_BYTES: usize = MAX_METADATA_BYTES; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum MediationFrameKind { + NetworkOpen = 1, + NetworkDecision = 2, + NetworkData = 3, + StreamClosed = 4, + DnsQuery = 5, + DnsResponse = 6, + NetworkEof = 7, +} + +impl TryFrom for MediationFrameKind { + type Error = io::Error; + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(Self::NetworkOpen), + 2 => Ok(Self::NetworkDecision), + 3 => Ok(Self::NetworkData), + 4 => Ok(Self::StreamClosed), + 5 => Ok(Self::DnsQuery), + 6 => Ok(Self::DnsResponse), + 7 => Ok(Self::NetworkEof), + _ => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unknown mediation frame kind {value}"), + )), + } + } +} + +#[derive(Debug)] +pub struct MediationFrame { + pub kind: MediationFrameKind, + pub stream_id: u64, + pub payload: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NetworkOpenWire { + pub identity: BinaryIdentityWire, + pub destination: SocketAddr, + pub socket: NetworkSocketMetadata, + pub policy_generation: u64, + pub timing: MediationTimingWire, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DnsQueryWire { + pub request: Vec, + pub transport: DnsTransport, + pub identity: BinaryIdentityWire, + pub timing: MediationTimingWire, +} + +pub fn encode_json(value: &T) -> io::Result> { + let payload = serde_json::to_vec(value) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + if payload.len() > MAX_METADATA_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "mediation metadata frame exceeds limit", + )); + } + Ok(payload) +} + +pub fn decode_json(payload: &[u8]) -> io::Result { + serde_json::from_slice(payload) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} + +pub async fn write_frame( + writer: &mut W, + kind: MediationFrameKind, + stream_id: u64, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_FRAME_BYTES.max(MAX_METADATA_BYTES) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "mediation frame exceeds limit", + )); + } + let length = u32::try_from(payload.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "mediation frame too large"))?; + let mut header = [0_u8; HEADER_BYTES]; + header[0] = kind as u8; + header[1..9].copy_from_slice(&stream_id.to_be_bytes()); + header[9..13].copy_from_slice(&length.to_be_bytes()); + writer.write_all(&header).await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_frame( + reader: &mut R, +) -> io::Result> { + let mut header = [0_u8; HEADER_BYTES]; + if reader.read(&mut header[..1]).await? == 0 { + return Ok(None); + } + reader.read_exact(&mut header[1..]).await?; + let kind = MediationFrameKind::try_from(header[0])?; + let stream_id = u64::from_be_bytes(header[1..9].try_into().expect("fixed header")); + let length = u32::from_be_bytes(header[9..13].try_into().expect("fixed header")) as usize; + if length > MAX_FRAME_BYTES.max(MAX_METADATA_BYTES) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "mediation frame exceeds limit", + )); + } + let mut payload = vec![0_u8; length]; + reader.read_exact(&mut payload).await?; + Ok(Some(MediationFrame { + kind, + stream_id, + payload, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn frame_round_trip_preserves_binary_payload() { + let (mut writer, mut reader) = tokio::io::duplex(128); + let send = tokio::spawn(async move { + write_frame( + &mut writer, + MediationFrameKind::NetworkData, + 42, + &[0, 1, 2, 255], + ) + .await + .unwrap(); + }); + let frame = read_frame(&mut reader).await.unwrap().unwrap(); + assert_eq!(frame.kind, MediationFrameKind::NetworkData); + assert_eq!(frame.stream_id, 42); + assert_eq!(frame.payload, vec![0, 1, 2, 255]); + send.await.unwrap(); + } +} diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs new file mode 100644 index 0000000000..09583d788a --- /dev/null +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -0,0 +1,2447 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned remote boundary. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +#[cfg(target_os = "linux")] +use std::mem::size_of; +#[cfg(target_os = "linux")] +use std::os::fd::{FromRawFd as _, IntoRawFd as _}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use crate::AgentSpec; +use crate::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, + BoundaryTerminal, ConfirmedBoundary, DnsMediationSource, ExecSession, ExecSpec, + IsolationBackend, LoopbackTarget, MediatedDnsQuery, MediationTiming, NetworkMediationSource, + NetworkOpenResult, PendingNetworkOpen, ProcessAttachment, ReadyBoundary, RunningBoundary, + SandboxContext, VerifiedTopologyDescriptor, +}; +use async_trait::async_trait; +use hyper_util::rt::TokioIo; +use openshell_core::proto::isolation::v1::{ + BoundaryChunk, isolation_boundary_client::IsolationBoundaryClient, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; +use tokio_stream::wrappers::ReceiverStream; + +use crate::boundary_protocol::{ + AgentSpecWire, BoundaryClientTls, BoundaryTopology, BoundaryTransport, DnsQueryResultWire, + ExecSpecWire, ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, + ResponseEnvelope, STREAM_DNS_ACK, STREAM_DNS_RESPONSE, STREAM_EXIT, STREAM_STDERR, + STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, decode_frame, + encode_frame, read_stream_frame, validate_resource_claims, write_stream_frame, +}; +use crate::mediation::{self, DnsQueryWire, MediationFrame, MediationFrameKind}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// How long one control call keeps retrying boundary connect attempts. Boot-time +/// callers retry whole calls above this; past boot, exhausting this window +/// means the remote boundary (or its launcher) is gone rather than still starting. +const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; + +/// Host-side remote boundary implementation registered with the supervisor. +#[derive(Debug)] +pub struct RemoteIsolationBackend { + backend_name: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +impl RemoteIsolationBackend { + pub fn new( + backend_name: impl Into, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + ) -> Self { + Self { + backend_name: backend_name.into(), + ca_file_paths, + provider_credentials, + } + } +} + +#[async_trait] +impl IsolationBackend for RemoteIsolationBackend { + fn backend_name(&self) -> &str { + &self.backend_name + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: BoundaryTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox, &self.backend_name)?; + let host_gateway_ip = topology.host_gateway_ip; + let resource_claims = topology.resource_claims.clone(); + let generation = topology.generation.clone(); + let session_epoch = topology.session_epoch.clone(); + let driver_fence = topology.driver_fence.clone(); + let client = Arc::new(BoundaryClient::new(topology)); + let response = client + .call_idempotent(Request::Attach { + policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())), + resource_claims: resource_claims.clone(), + }) + .await?; + let Response::Attached { snapshot } = response else { + return Err(unexpected_response("attached", &response)); + }; + if snapshot.generation != generation { + return Err(BackendError::Confirm( + "sandbox session snapshot generation does not match topology".to_string(), + )); + } + Ok(Box::new(RemoteBound { + client: client.clone(), + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(RemoteNetworkMediation { + client: client.clone(), + }), + dns_mediation: Arc::new(RemoteDnsMediation { client }), + host_gateway_ip, + ca_file_paths: self.ca_file_paths.clone(), + provider_credentials: self.provider_credentials.clone(), + identity: sandbox.identity, + generation, + session_epoch, + resource_claims, + driver_fence, + })) + } +} + +fn validate_topology( + topology: &BoundaryTopology, + sandbox: &SandboxContext, + backend_name: &str, +) -> Result<(), BackendError> { + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.generation.is_empty() || topology.session_epoch.is_empty() { + return Err(BackendError::Descriptor( + "boundary generation and session epoch must not be empty".to_string(), + )); + } + if topology.workload_identity != sandbox.identity { + return Err(BackendError::Descriptor( + "topology workload identity does not match admitted sandbox identity".to_string(), + )); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "boundary bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + validate_resource_claims(&topology.resource_claims)?; + topology.driver_fence.validate_for_backend(backend_name)?; + let tls = match &topology.transport { + BoundaryTransport::Unix { socket_path, tls } => { + validate_socket_path(socket_path)?; + tls + } + BoundaryTransport::TlsTcp { address, tls } => { + validate_tcp_address(*address)?; + tls + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + tls, + } => { + if *guest_cid < 3 { + return Err(BackendError::Descriptor( + "boundary CID must be at least 3".to_string(), + )); + } + validate_control_port(*control_port)?; + tls + } + }; + validate_client_tls(tls)?; + Ok(()) +} + +fn validate_tcp_address(address: std::net::SocketAddr) -> Result<(), BackendError> { + if address.port() == 0 || address.ip().is_unspecified() { + Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )) + } else { + Ok(()) + } +} + +fn validate_client_tls(tls: &BoundaryClientTls) -> Result<(), BackendError> { + rustls::pki_types::ServerName::try_from(tls.server_name.clone()).map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {:?} is invalid: {error}", + tls.server_name + )) + })?; + tls_client_config(tls).map(|_| ()) +} + +fn tls_client_config(tls: &BoundaryClientTls) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificates = rustls_pemfile::certs(&mut tls.ca_certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse boundary TLS CA certificate: {error}")) + })?; + if certificates.is_empty() { + return Err(BackendError::Descriptor( + "boundary TLS CA certificate PEM contains no certificates".to_string(), + )); + } + let mut roots = rustls::RootCertStore::empty(); + for certificate in certificates { + roots.add(certificate).map_err(|error| { + BackendError::Descriptor(format!("load boundary TLS CA certificate: {error}")) + })?; + } + let certificate_chain = rustls_pemfile::certs(&mut tls.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse supervisor TLS certificate: {error}")) + })?; + if certificate_chain.is_empty() { + return Err(BackendError::Descriptor( + "supervisor TLS certificate PEM contains no certificates".to_string(), + )); + } + let private_key = rustls_pemfile::private_key(&mut tls.private_key_pem.as_bytes()) + .map_err(|error| { + BackendError::Descriptor(format!("parse supervisor TLS private key: {error}")) + })? + .ok_or_else(|| { + BackendError::Descriptor( + "supervisor TLS private-key PEM contains no private key".to_string(), + ) + })?; + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_client_auth_cert(certificate_chain, private_key) + .map_err(|error| { + BackendError::Descriptor(format!("build supervisor mutual-TLS config: {error}")) + }) +} + +fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { + if path.is_absolute() { + Ok(()) + } else { + Err(BackendError::Descriptor( + "boundary control Unix socket path must be absolute".to_string(), + )) + } +} + +fn validate_control_port(port: u32) -> Result<(), BackendError> { + if port == 0 { + Err(BackendError::Descriptor( + "boundary control port must be nonzero".to_string(), + )) + } else { + Ok(()) + } +} + +struct RemoteBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, + dns_mediation: Arc, + host_gateway_ip: Option, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + identity: crate::contract::ResolvedWorkloadIdentity, + generation: String, + session_epoch: String, + resource_claims: std::collections::BTreeMap, + driver_fence: crate::contract::DriverFenceEvidence, +} + +#[async_trait] +impl BoundBoundary for RemoteBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + fn dns_mediation_source(&self) -> Option> { + Some(self.dns_mediation.clone()) + } + + fn host_gateway_ip(&self) -> Option { + self.host_gateway_ip + } + + async fn confirm(self: Box) -> Result { + let response = self.client.call_idempotent(Request::Confirm).await?; + let Response::Confirmed { evidence } = response else { + return Err(unexpected_response("confirmed_with_evidence", &response)); + }; + evidence.validate(&self.identity)?; + if evidence.generation != self.generation + || evidence.session_epoch != self.session_epoch + || evidence.resource_claims != self.resource_claims + || evidence.driver_fence != self.driver_fence + { + return Err(BackendError::Confirm( + "sandbox confirmation generation, session, resource claims, or driver fence do not match topology" + .to_string(), + )); + } + Ok(ConfirmedBoundary::new( + Box::new(RemoteReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + ca_file_paths: self.ca_file_paths, + provider_credentials: self.provider_credentials, + }), + *evidence, + )) + } +} + +struct RemoteReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +#[async_trait] +impl ReadyBoundary for RemoteReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let ca_paths = self + .ca_file_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { + let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) + })?; + let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; + (Some(ca_cert), Some(ca_bundle)) + } else { + (None, None) + }; + let (provider_env_revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + }) + .await?; + let Response::Started { + process_id, + provider_env_revision, + } = response + else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(RemoteProcess { + client: self.client.clone(), + process_id, + }); + Ok(Box::new(RemoteRunning { + process, + exec: Arc::new(RemoteExec { + client: self.client.clone(), + provider_credentials: self.provider_credentials, + boundary_revision: tokio::sync::Mutex::new(provider_env_revision), + }), + port_forward: Arc::new(RemotePortForward { + client: self.client, + }), + })) + } +} + +struct RemoteRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for RemoteRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct RemoteProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for RemoteProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await + .map_err(|error| match error { + // A wait that can no longer reach the boundary leaf means the + // boundary is gone, not that a retry could still observe the + // exit status; report boundary loss per the contract. + BackendError::Unavailable(message) => { + BackendError::Terminated(format!("boundary lost during wait: {message}")) + } + error => error, + })?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +async fn open_process_attachment( + client: Arc, + process_id: String, +) -> Result { + let (stream, response) = client + .call_stream(Request::AttachProcess { + process_id: process_id.clone(), + }) + .await?; + let Response::ProcessAttached { + terminal: has_terminal, + } = response + else { + return Err(unexpected_response("process_attached", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_process_responses( + network_reader, + stdout_pump, + stderr_pump, + )); + let terminal: Option> = if has_terminal { + let terminal: Arc = Arc::new(RemoteTerminal { client, process_id }); + Some(terminal) + } else { + None + }; + let stderr: Option = if has_terminal { + None + } else { + let stderr: BoundaryOutput = Box::new(stderr); + Some(stderr) + }; + Ok(ProcessAttachment { + stdin: Box::new(stdin), + stdout: Box::new(stdout), + stderr, + terminal, + }) +} + +async fn pump_process_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_EXIT, _)) | None) | Err(_) => return, + Ok(Some((_channel, _))) => return, + } + } +} + +struct RemoteExec { + client: Arc, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + boundary_revision: tokio::sync::Mutex, +} + +#[async_trait] +impl BoundaryExec for RemoteExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let mut boundary_revision = self.boundary_revision.lock().await; + for _ in 0..3 { + let (revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::UpdateProviderEnvironment { + expected_revision: *boundary_revision, + revision, + provider_env, + }) + .await?; + let Response::ProviderEnvironmentUpdated { + revision: effective_revision, + } = response + else { + return Err(unexpected_response( + "provider_environment_updated", + &response, + )); + }; + *boundary_revision = effective_revision; + if effective_revision == revision { + return open_exec_session(self.client.clone(), spec).await; + } + } + Err(BackendError::Process( + "boundary provider environment changed concurrently during reconciliation".to_string(), + )) + } +} + +struct RemotePortForward { + client: Arc, +} + +#[async_trait] +impl BoundaryPortForward for RemotePortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + let (stream, response) = self + .client + .call_stream(Request::PortForward { + host: target.host(), + port: target.port(), + }) + .await?; + match response { + Response::PortConnected => Ok(stream), + response => Err(unexpected_response("port_connected", &response)), + } + } +} + +struct RemoteExecProcess { + client: Arc, + process_id: String, + exit: Arc, +} + +struct RemoteExit { + result: std::sync::Mutex>>, + changed: Notify, +} + +impl RemoteExit { + fn new() -> Self { + Self { + result: std::sync::Mutex::new(None), + changed: Notify::new(), + } + } + + fn set(&self, result: Result) { + let mut current = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + *current = Some(result); + self.changed.notify_waiters(); + } + } + + async fn wait(&self) -> Result { + loop { + let changed = self.changed.notified(); + let result = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Terminated); + } + changed.await; + } + } +} + +#[async_trait] +impl BoundaryProcess for RemoteExecProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + + async fn wait(&self) -> Result { + self.exit.wait().await + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + expect_response( + self.client + .call_idempotent(Request::ExecSignal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?, + "signaled", + ) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct RemoteTerminal { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryTerminal for RemoteTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Resize { + process_id: self.process_id.clone(), + cols, + rows, + }) + .await?; + if matches!(response, Response::Resized) { + Ok(()) + } else { + Err(unexpected_response("resized", &response)) + } + } +} + +async fn open_exec_session( + client: Arc, + spec: ExecSpec, +) -> Result { + let (stream, response) = client + .call_stream_idempotent(Request::Exec { + spec: ExecSpecWire::from(spec), + }) + .await?; + let Response::ExecStarted { process_id, pty } = response else { + return Err(unexpected_response("exec_started", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + let exit = Arc::new(RemoteExit::new()); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_exec_responses( + network_reader, + stdout_pump, + stderr_pump, + exit.clone(), + )); + + let process: Arc = Arc::new(RemoteExecProcess { + client: client.clone(), + process_id: process_id.clone(), + exit, + }); + let terminal: Option> = if pty { + Some(Arc::new(RemoteTerminal { client, process_id })) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if pty { None } else { Some(Box::new(stderr)) }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) +} + +async fn pump_exec_input( + mut input: tokio::io::DuplexStream, + mut network: tokio::io::WriteHalf, +) { + let mut buffer = vec![0; 16 * 1024]; + loop { + match input.read(&mut buffer).await { + Ok(0) => { + let _ = write_stream_frame(&mut network, STREAM_STDIN_CLOSED, &[]).await; + return; + } + Ok(read) => { + if write_stream_frame(&mut network, STREAM_STDIN, &buffer[..read]) + .await + .is_err() + { + return; + } + } + Err(_) => return, + } + } +} + +async fn pump_exec_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, + exit: Arc, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stdout consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stderr consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_EXIT, payload))) => { + let result = serde_json::from_slice::(&payload) + .map(BoundaryExitStatus::from) + .map_err(|error| format!("decode boundary exec exit: {error}")); + exit.set(result); + return; + } + Ok(Some((channel, _))) => { + exit.set(Err(format!( + "boundary exec returned unexpected stream channel {channel}" + ))); + return; + } + Ok(None) => { + exit.set(Err( + "boundary exec stream closed before exit status".to_string() + )); + return; + } + Err(error) => { + exit.set(Err(format!("read boundary exec stream: {error}"))); + return; + } + } + } +} + +/// Pulls boundary proxy connections over independent HTTP/2 streams. +/// +/// DNS and UDP control messages share the compact persistent mediation +/// session, but TCP byte streams use HTTP/2's native multiplexing. Nesting all +/// TCP connections inside one application-level writer creates avoidable +/// head-of-line blocking during concurrent TLS handshakes. +struct RemoteNetworkMediation { + client: Arc, +} + +#[async_trait] +impl NetworkMediationSource for RemoteNetworkMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptNetwork).await?; + let Response::NetworkConnected { + identity, + destination, + socket, + policy_generation, + timing, + } = response + else { + return Err(unexpected_response("network_connected", &response)); + }; + let (result, completion) = tokio::sync::oneshot::channel(); + let (proxy_stream, transport_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(complete_network_open(stream, transport_stream, completion)); + Ok(PendingNetworkOpen { + stream: Box::new(proxy_stream), + binary_identity: identity.into_result(), + destination, + socket, + policy_generation, + timing: MediationTiming { + sandbox_notification_to_queue: Duration::from_micros( + timing.notification_to_queue_us, + ), + sandbox_queue_wait: Duration::from_micros(timing.queue_wait_us), + supervisor_received_at: Instant::now(), + }, + result, + }) + } +} + +/// Pulls sandbox DNS wire exchanges over authenticated control streams. +struct RemoteDnsMediation { + client: Arc, +} + +#[async_trait] +impl DnsMediationSource for RemoteDnsMediation { + async fn accept(&self) -> Result { + if self.client.topology.multiplexed { + loop { + let session = self.client.mediation_session().await?; + match session.accept_dns().await { + Ok(query) => return Ok(query), + Err(BackendError::Unavailable(_)) if !session.is_healthy() => {} + Err(error) => return Err(error), + } + } + } + let (stream, response) = self.client.open_exchange(Request::AcceptDns).await?; + let Response::DnsQuery { + request, + transport, + identity, + timing, + } = response + else { + return Err(unexpected_response("dns_query", &response)); + }; + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + tokio::spawn(complete_dns_query(stream, response_rx)); + Ok(MediatedDnsQuery { + request, + transport, + binary_identity: identity.into_result(), + timing: MediationTiming { + sandbox_notification_to_queue: Duration::from_micros( + timing.notification_to_queue_us, + ), + sandbox_queue_wait: Duration::from_micros(timing.queue_wait_us), + supervisor_received_at: Instant::now(), + }, + response: response_tx, + }) + } +} + +async fn complete_dns_query( + mut boundary: BoundaryDuplexStream, + response: tokio::sync::oneshot::Receiver, BackendError>>, +) { + let result = match response.await { + Ok(Ok(response)) => DnsQueryResultWire::Response(response), + Ok(Err(error)) => DnsQueryResultWire::Error(error.to_string()), + Err(_) => DnsQueryResultWire::Error("DNS mediation was cancelled".to_string()), + }; + let payload = match serde_json::to_vec(&result) { + Ok(payload) => payload, + Err(error) => { + tracing::warn!(%error, "encode mediated DNS response failed: {error}"); + return; + } + }; + if let Err(error) = write_stream_frame(&mut boundary, STREAM_DNS_RESPONSE, &payload).await { + tracing::warn!(%error, "write mediated DNS response failed: {error}"); + return; + } + match tokio::time::timeout(REQUEST_TIMEOUT, read_stream_frame(&mut boundary)).await { + Ok(Ok(Some((STREAM_DNS_ACK, payload)))) if payload.is_empty() => {} + Ok(Ok(Some((channel, _)))) => { + tracing::warn!(channel, "unexpected mediated DNS acknowledgement channel"); + } + Ok(Ok(None)) => tracing::warn!("boundary closed before acknowledging DNS response"), + Ok(Err(error)) => { + tracing::warn!(%error, "read mediated DNS acknowledgement failed: {error}"); + } + Err(_) => tracing::warn!("timed out waiting for mediated DNS acknowledgement"), + } +} + +async fn complete_network_open( + mut boundary: BoundaryDuplexStream, + mut transport: tokio::io::DuplexStream, + completion: tokio::sync::oneshot::Receiver, +) { + let decision = completion.await.unwrap_or(NetworkOpenResult::Denied { + errno: cancellation_errno(), + }); + let Ok(payload) = serde_json::to_vec(&decision) else { + return; + }; + if write_stream_frame( + &mut boundary, + crate::boundary_protocol::STREAM_NETWORK_DECISION, + &payload, + ) + .await + .is_err() + { + return; + } + if matches!(decision, NetworkOpenResult::RelayReady) { + let _ = tokio::io::copy_bidirectional(&mut boundary, &mut transport).await; + } +} + +const fn cancellation_errno() -> i32 { + #[cfg(unix)] + { + libc::ECANCELED + } + #[cfg(not(unix))] + { + 125 + } +} + +const MEDIATION_EVENT_QUEUE: usize = 256; +struct OutboundMediationFrame { + kind: MediationFrameKind, + stream_id: u64, + payload: Vec, +} + +type MediationRoutes = + Arc>>>; + +struct ClientMediationSession { + dns: tokio::sync::Mutex>, + healthy: Arc, +} + +impl ClientMediationSession { + fn start(stream: BoundaryDuplexStream) -> Arc { + let (dns_tx, dns_rx) = tokio::sync::mpsc::channel(MEDIATION_EVENT_QUEUE); + let healthy = Arc::new(AtomicBool::new(true)); + let session = Arc::new(Self { + dns: tokio::sync::Mutex::new(dns_rx), + healthy: healthy.clone(), + }); + tokio::spawn(async move { + if let Err(error) = run_client_mediation(stream, dns_tx).await { + tracing::debug!(%error, "persistent mediation session ended"); + } + healthy.store(false, Ordering::Release); + }); + session + } + + fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::Acquire) + } + + async fn accept_dns(&self) -> Result { + self.dns.lock().await.recv().await.ok_or_else(|| { + BackendError::Unavailable("persistent DNS mediation session ended".to_string()) + }) + } +} + +async fn run_client_mediation( + stream: BoundaryDuplexStream, + dns_tx: tokio::sync::mpsc::Sender, +) -> std::io::Result<()> { + let (mut reader, mut writer) = tokio::io::split(stream); + let (outbound_tx, mut outbound_rx) = + tokio::sync::mpsc::channel::(MEDIATION_EVENT_QUEUE); + let routes = Arc::new(tokio::sync::Mutex::new(HashMap::new())); + let writer_task = async { + while let Some(frame) = outbound_rx.recv().await { + mediation::write_frame(&mut writer, frame.kind, frame.stream_id, &frame.payload) + .await?; + } + Ok::<(), std::io::Error>(()) + }; + let reader_task = async { + while let Some(frame) = mediation::read_frame(&mut reader).await? { + dispatch_client_mediation_frame(frame, &dns_tx, &outbound_tx, &routes).await?; + } + Ok::<(), std::io::Error>(()) + }; + tokio::pin!(writer_task); + tokio::pin!(reader_task); + let result = tokio::select! { + result = &mut writer_task => result, + result = &mut reader_task => result, + }; + routes.lock().await.clear(); + result +} + +async fn dispatch_client_mediation_frame( + frame: MediationFrame, + dns_tx: &tokio::sync::mpsc::Sender, + outbound: &tokio::sync::mpsc::Sender, + routes: &MediationRoutes, +) -> std::io::Result<()> { + match frame.kind { + MediationFrameKind::DnsQuery => { + let query: DnsQueryWire = mediation::decode_json(&frame.payload)?; + let (response, completion) = + tokio::sync::oneshot::channel::, BackendError>>(); + let outbound = outbound.clone(); + tokio::spawn(async move { + let response = match completion.await { + Ok(Ok(response)) => DnsQueryResultWire::Response(response), + Ok(Err(error)) => DnsQueryResultWire::Error(error.to_string()), + Err(_) => DnsQueryResultWire::Error( + "supervisor dropped the mediated DNS query".to_string(), + ), + }; + if let Ok(payload) = mediation::encode_json(&response) { + let _ = outbound + .send(OutboundMediationFrame { + kind: MediationFrameKind::DnsResponse, + stream_id: frame.stream_id, + payload, + }) + .await; + } + }); + dns_tx + .send(MediatedDnsQuery { + request: query.request, + transport: query.transport, + binary_identity: query.identity.into_result(), + timing: MediationTiming { + sandbox_notification_to_queue: Duration::from_micros( + query.timing.notification_to_queue_us, + ), + sandbox_queue_wait: Duration::from_micros(query.timing.queue_wait_us), + supervisor_received_at: Instant::now(), + }, + response, + }) + .await + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "DNS mediation consumer stopped", + ) + })?; + } + MediationFrameKind::StreamClosed => { + let route = routes.lock().await.get(&frame.stream_id).cloned(); + if let Some(route) = route { + let _ = route.send(frame).await; + } + } + MediationFrameKind::NetworkOpen + | MediationFrameKind::NetworkDecision + | MediationFrameKind::NetworkData + | MediationFrameKind::NetworkEof + | MediationFrameKind::DnsResponse => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "unexpected supervisor-bound mediation frame {:?}", + frame.kind + ), + )); + } + } + Ok(()) +} + +struct BoundaryClient { + topology: BoundaryTopology, + grpc_channel: tokio::sync::Mutex>, + mediation: tokio::sync::Mutex>>, +} + +impl BoundaryClient { + fn new(topology: BoundaryTopology) -> Self { + Self { + topology, + grpc_channel: tokio::sync::Mutex::new(None), + mediation: tokio::sync::Mutex::new(None), + } + } + + async fn call_idempotent(&self, request: Request) -> Result { + let envelope = self.prepare_request(request)?; + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable( + "boundary idempotent control request timed out while waiting for remote boundary boot".to_string(), + ) + })? + } + + async fn call_wait(&self, request: Request) -> Result { + const WAIT_RECONNECT_ATTEMPTS: usize = 3; + let envelope = self.prepare_request(request)?; + for attempt in 1..=WAIT_RECONNECT_ATTEMPTS { + match self.exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) if attempt < WAIT_RECONNECT_ATTEMPTS => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + unreachable!("bounded wait reconnect loop always returns") + } + + async fn call_stream( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + tokio::time::timeout(REQUEST_TIMEOUT, self.open_exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary stream request timed out".to_string()) + })? + } + + async fn call_stream_idempotent( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let envelope = self.prepare_request(request)?; + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.open_exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable("boundary idempotent stream request timed out".to_string()) + })? + } + + #[cfg(test)] + async fn exchange(&self, request: Request) -> Result { + let (_, response) = self.open_exchange(request).await?; + Ok(response) + } + + fn prepare_request(&self, request: Request) -> Result { + RequestEnvelope::new( + self.topology.boundary_id.clone(), + self.topology.bootstrap_token.clone(), + request, + ) + .map_err(|error| BackendError::Process(format!("encode control request: {error}"))) + } + + async fn open_exchange( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let envelope = self.prepare_request(request)?; + self.open_exchange_envelope(&envelope).await + } + + async fn exchange_envelope( + &self, + envelope: &RequestEnvelope, + ) -> Result { + let (_, response) = self.open_exchange_envelope(envelope).await?; + Ok(response) + } + + async fn open_exchange_envelope( + &self, + envelope: &RequestEnvelope, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let request_id = envelope.request_id.clone(); + let mut stream = if self.topology.multiplexed { + self.open_grpc_stream(GrpcStreamKind::Exchange).await? + } else { + self.connect_boundary().await? + }; + let frame = encode_frame(envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write boundary control request: {error}")) + })?; + // `tokio-rustls` may retain part of a large plaintext frame in its + // internal TLS buffer. Flush before waiting for the response so the + // synchronous boundary reader can receive the complete request. + stream.flush().await.map_err(|error| { + BackendError::Unavailable(format!("flush boundary control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "boundary control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "boundary response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + let response = match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + }?; + Ok((stream, response)) + } + + async fn open_grpc_stream( + &self, + kind: GrpcStreamKind, + ) -> Result { + let channel = self.grpc_channel().await?; + open_grpc_client_stream(channel, kind).await + } + + async fn mediation_session(&self) -> Result, BackendError> { + let mut state = self.mediation.lock().await; + if let Some(session) = state.as_ref() + && session.is_healthy() + { + return Ok(session.clone()); + } + for attempt in 0..=40 { + match self.open_mediation_session().await { + Ok(session) => { + *state = Some(session.clone()); + return Ok(session); + } + Err(BackendError::Denied(message)) + if message.contains("already active") && attempt < 40 => + { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + unreachable!("bounded mediation reconnect loop always returns") + } + + async fn open_mediation_session(&self) -> Result, BackendError> { + let mut stream = self.open_grpc_stream(GrpcStreamKind::Mediate).await?; + let envelope = self.prepare_request(Request::OpenMediation)?; + let request_id = envelope.request_id.clone(); + let frame = encode_frame(&envelope) + .map_err(|error| BackendError::Process(format!("encode mediation attach: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write mediation attach: {error}")) + })?; + stream.flush().await.map_err(|error| { + BackendError::Unavailable(format!("flush mediation attach: {error}")) + })?; + let response = + crate::boundary_protocol::read_frame_async::<_, ResponseEnvelope>(&mut stream) + .await + .map_err(|error| { + BackendError::Unavailable(format!("read mediation attach: {error}")) + })?; + if response.request_id != request_id { + return Err(BackendError::Process( + "mediation attach response ID did not match request".to_string(), + )); + } + match response.response { + Response::MediationReady => {} + Response::Error { kind, message } => return Err(guest_error(&kind, message)), + response => return Err(unexpected_response("mediation_ready", &response)), + } + Ok(ClientMediationSession::start(stream)) + } + + async fn grpc_channel(&self) -> Result { + let mut state = self.grpc_channel.lock().await; + if let Some(channel) = state.as_ref() { + return Ok(channel.clone()); + } + let topology = self.topology.clone(); + let endpoint = + tonic::transport::Endpoint::from_static("http://boundary.openshell.internal") + .initial_stream_window_size(16 * 1024 * 1024) + .initial_connection_window_size(16 * 1024 * 1024) + .http2_keep_alive_interval(Duration::from_secs(10)) + .keep_alive_while_idle(true); + let channel = endpoint + .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + let topology = topology.clone(); + async move { + connect_boundary_with_retry(&topology) + .await + .map(TokioIo::new) + .map_err(|error| std::io::Error::other(error.to_string())) + } + })) + .await + .map_err(|error| { + BackendError::Unavailable(format!("start boundary gRPC channel: {error}")) + })?; + *state = Some(channel.clone()); + Ok(channel) + } + + async fn connect_boundary(&self) -> Result { + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match self.connect_boundary_once().await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } + + async fn connect_boundary_once(&self) -> Result { + connect_boundary_once(&self.topology).await + } +} + +async fn connect_boundary_with_retry( + topology: &BoundaryTopology, +) -> Result { + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match connect_boundary_once(topology).await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } +} + +async fn connect_boundary_once( + topology: &BoundaryTopology, +) -> Result { + let (stream, tls): (BoundaryDuplexStream, &BoundaryClientTls) = match &topology.transport { + BoundaryTransport::Unix { socket_path, tls } => { + let stream = UnixStream::connect(socket_path).await.map_err(|error| { + BackendError::Unavailable(format!( + "connect to mapped boundary control socket {}: {error}", + socket_path.display() + )) + })?; + (Box::new(stream), tls) + } + BoundaryTransport::TlsTcp { address, tls } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TLS endpoint {address}: {error}" + )) + })?; + enable_boundary_tcp_keepalive(&stream); + (Box::new(stream), tls) + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + tls, + } => (connect_host_vsock(*guest_cid, *control_port)?, tls), + }; + let server_name = + rustls::pki_types::ServerName::try_from(tls.server_name.clone()).map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {:?} is invalid: {error}", + tls.server_name + )) + })?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(tls_client_config(tls)?)); + let stream = connector + .connect(server_name, stream) + .await + .map_err(|error| { + BackendError::Unavailable(format!("authenticate sandbox channel: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[derive(Clone, Copy)] +enum GrpcStreamKind { + Exchange, + Mediate, +} + +async fn open_grpc_client_stream( + channel: tonic::transport::Channel, + kind: GrpcStreamKind, +) -> Result { + let (application, bridge) = tokio::io::duplex(256 * 1024); + let (reader, writer) = tokio::io::split(bridge); + let (outbound, outbound_rx) = tokio::sync::mpsc::channel::(64); + tokio::spawn(pump_to_grpc(reader, outbound)); + let mut client = IsolationBoundaryClient::new(channel) + .max_decoding_message_size(64 * 1024) + .max_encoding_message_size(64 * 1024); + let request = ReceiverStream::new(outbound_rx); + let response = match kind { + GrpcStreamKind::Exchange => client.exchange(request).await, + GrpcStreamKind::Mediate => client.mediate(request).await, + } + .map_err(|error| BackendError::Unavailable(format!("open boundary gRPC stream: {error}")))?; + tokio::spawn(pump_from_grpc(response.into_inner(), writer)); + Ok(Box::new(application)) +} + +async fn pump_to_grpc(mut reader: R, sender: tokio::sync::mpsc::Sender) +where + R: tokio::io::AsyncRead + Unpin, +{ + let mut buffer = vec![0_u8; 16 * 1024]; + loop { + let read = match reader.read(&mut buffer).await { + Ok(read) => read, + Err(error) => { + tracing::debug!(%error, "boundary gRPC request reader ended"); + return; + } + }; + if read == 0 { + return; + } + if sender + .send(BoundaryChunk { + data: buffer[..read].to_vec(), + }) + .await + .is_err() + { + return; + } + } +} + +async fn pump_from_grpc(mut stream: tonic::Streaming, mut writer: W) +where + W: tokio::io::AsyncWrite + Unpin, +{ + loop { + match stream.message().await { + Ok(Some(chunk)) => { + if let Err(error) = writer.write_all(&chunk.data).await { + tracing::debug!(%error, "boundary gRPC response writer ended"); + return; + } + } + Ok(None) => { + let _ = writer.shutdown().await; + return; + } + Err(error) => { + tracing::debug!(%error, "boundary gRPC response stream ended"); + return; + } + } + } +} + +fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(30)) + .with_interval(Duration::from_secs(10)); + let _ = socket2::SockRef::from(stream).set_tcp_keepalive(&keepalive); +} + +#[cfg(target_os = "linux")] +fn connect_host_vsock( + guest_cid: u32, + control_port: u32, +) -> Result { + let fd = unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(BackendError::Unavailable(format!( + "create host vsock: {}", + std::io::Error::last_os_error() + ))); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }; + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address family: {error}")) + })?; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: control_port, + svm_cid: guest_cid, + svm_zero: [0; 4], + }; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address length: {error}")) + })?; + let result = unsafe { + libc::connect( + std::os::fd::AsRawFd::as_raw_fd(&fd), + (&raw const address).cast::(), + address_length, + ) + }; + if result != 0 { + return Err(BackendError::Unavailable(format!( + "connect host vsock CID {guest_cid} port {control_port}: {}", + std::io::Error::last_os_error() + ))); + } + let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + BackendError::Unavailable(format!("set host vsock nonblocking: {error}")) + })?; + let stream = UnixStream::from_std(stream).map_err(|error| { + BackendError::Unavailable(format!("register host vsock with Tokio: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[cfg(not(target_os = "linux"))] +fn connect_host_vsock( + _guest_cid: u32, + _control_port: u32, +) -> Result { + Err(BackendError::Unavailable( + "host AF_VSOCK transport is supported only on Linux".to_string(), + )) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached { .. }, "attached") + | (Response::Confirmed { .. }, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected boundary response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("boundary process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::pin::Pin; + use std::task::{Context, Poll}; + + use super::*; + use crate::boundary_protocol::generate_boundary_mutual_tls_material; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_core::proto::isolation::v1::{ + BoundaryChunk, + isolation_boundary_server::{IsolationBoundary, IsolationBoundaryServer}, + }; + + fn test_driver_fence() -> crate::contract::DriverFenceEvidence { + crate::contract::DriverFenceEvidence::Vm { + generation: "test-generation".to_string(), + network_device_count: 0, + } + } + + #[tokio::test] + async fn boundary_tcp_connections_enable_keepalive() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let connected = tokio::spawn(async move { tokio::net::TcpStream::connect(address).await }); + let (_server, _) = listener.accept().await.unwrap(); + let client = connected.await.unwrap().unwrap(); + + enable_boundary_tcp_keepalive(&client); + + assert!(socket2::SockRef::from(&client).keepalive().unwrap()); + } + + #[derive(Clone)] + struct TestGrpcBoundary { + wait_for_half_close: bool, + requests: Arc, + } + + type TestGrpcStream = Pin< + Box> + Send + 'static>, + >; + + #[tonic::async_trait] + impl IsolationBoundary for TestGrpcBoundary { + type ExchangeStream = TestGrpcStream; + type MediateStream = TestGrpcStream; + + async fn exchange( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + let mut inbound = request.into_inner(); + let wait_for_half_close = self.wait_for_half_close; + let requests = self.requests.clone(); + let (outbound, outbound_rx) = tokio::sync::mpsc::channel(1); + tokio::spawn(async move { + let mut frame = Vec::new(); + loop { + match inbound.message().await { + Ok(Some(chunk)) => { + frame.extend_from_slice(&chunk.data); + if !wait_for_half_close && complete_control_frame(&frame) { + break; + } + } + Ok(None) => break, + Err(error) => { + let _ = outbound.send(Err(error)).await; + return; + } + } + } + requests.fetch_add(1, Ordering::AcqRel); + let response = if complete_control_frame(&frame) { + let envelope: RequestEnvelope = match decode_frame(&frame) { + Ok(envelope) => envelope, + Err(error) => { + let _ = outbound + .send(Err(tonic::Status::invalid_argument(error.to_string()))) + .await; + return; + } + }; + match encode_frame(&ResponseEnvelope { + request_id: envelope.request_id, + response: Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + }, + }) { + Ok(response) => response, + Err(error) => { + let _ = outbound + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } + } else { + b"complete response".to_vec() + }; + let _ = outbound.send(Ok(BoundaryChunk { data: response })).await; + }); + Ok(tonic::Response::new(Box::pin(ReceiverStream::new( + outbound_rx, + )))) + } + + async fn mediate( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + self.exchange(request).await + } + } + + fn complete_control_frame(frame: &[u8]) -> bool { + frame.len() >= 4 + && frame.len() + >= 4 + usize::try_from(u32::from_be_bytes( + frame[..4].try_into().expect("frame header"), + )) + .expect("frame length") + } + + #[tokio::test] + async fn grpc_stream_preserves_response_after_request_half_close() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let service = TestGrpcBoundary { + wait_for_half_close: true, + requests, + }; + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + tonic::transport::Server::builder() + .add_service(IsolationBoundaryServer::new(service)) + .serve_with_incoming(tokio_stream::iter([Ok::<_, std::io::Error>(stream)])) + .await + .unwrap(); + }); + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut stream = open_grpc_client_stream(channel, GrpcStreamKind::Exchange) + .await + .unwrap(); + stream.write_all(b"finite request").await.unwrap(); + stream.shutdown().await.unwrap(); + let mut response = [0_u8; 17]; + stream.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"complete response"); + drop(stream); + server.abort(); + } + + #[tokio::test] + async fn remote_dns_exchange_returns_supervisor_response() { + let socket_path = std::env::temp_dir().join(format!( + "openshell-dns-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let certificate = test_certificate(); + let server_config = certificate.server_config.clone(); + let listener = tokio::net::UnixListener::bind(&socket_path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .unwrap(); + let declared = stream.read_u32().await.unwrap() as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&u32::try_from(declared).unwrap().to_be_bytes()); + stream.read_exact(&mut frame[4..]).await.unwrap(); + let request: RequestEnvelope = decode_frame(&frame).unwrap(); + assert_eq!(request.request, Request::AcceptDns); + let response = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response: Response::DnsQuery { + request: vec![1, 2, 3], + transport: crate::contract::DnsTransport::Udp, + identity: crate::boundary_protocol::BinaryIdentityWire { + binary_path: Some(PathBuf::from("/usr/bin/dig")), + binary_digest: Some("a".repeat(64)), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: None, + }, + timing: crate::boundary_protocol::MediationTimingWire::default(), + }, + }) + .unwrap(); + stream.write_all(&response).await.unwrap(); + let (channel, payload) = read_stream_frame(&mut stream).await.unwrap().unwrap(); + assert_eq!(channel, STREAM_DNS_RESPONSE); + assert_eq!( + serde_json::from_slice::(&payload).unwrap(), + DnsQueryResultWire::Response(vec![4, 5, 6]) + ); + write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) + .await + .unwrap(); + }); + let client = Arc::new(BoundaryClient::new(BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: socket_path.clone(), + tls: certificate.client_tls, + }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "a".repeat(32), + })); + let source = RemoteDnsMediation { client }; + let query = source.accept().await.unwrap(); + assert_eq!(query.request, [1, 2, 3]); + assert_eq!(query.transport, crate::contract::DnsTransport::Udp); + assert_eq!( + query.binary_identity.unwrap().binary_path, + PathBuf::from("/usr/bin/dig") + ); + query.response.send(Ok(vec![4, 5, 6])).unwrap(); + server.await.unwrap(); + let _ = std::fs::remove_file(socket_path); + } + + struct TestCertificate { + client_tls: BoundaryClientTls, + server_config: Arc, + } + + fn test_certificate() -> TestCertificate { + let _ = rustls::crypto::ring::default_provider().install_default(); + let material = generate_boundary_mutual_tls_material().expect("generate test material"); + let certificates = rustls_pemfile::certs(&mut material.sandbox_certificate_pem.as_bytes()) + .collect::, _>>() + .expect("parse server certificate"); + let private_key = + rustls_pemfile::private_key(&mut material.sandbox_private_key_pem.as_bytes()) + .expect("parse server private key") + .expect("server private key"); + let client_ca = rustls_pemfile::certs(&mut material.ca_certificate_pem.as_bytes()) + .collect::, _>>() + .expect("parse client CA"); + let mut client_roots = rustls::RootCertStore::empty(); + for certificate in client_ca { + client_roots.add(certificate).expect("add client CA"); + } + let verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .expect("build client verifier"); + let server_config = rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(certificates, private_key) + .expect("build test TLS server config"); + TestCertificate { + client_tls: BoundaryClientTls { + server_name: material.server_name, + ca_certificate_pem: material.ca_certificate_pem, + certificate_chain_pem: material.supervisor_certificate_pem, + private_key_pem: material.supervisor_private_key_pem, + }, + server_config: Arc::new(server_config), + } + } + + fn tls_topology( + address: std::net::SocketAddr, + tls: BoundaryClientTls, + token: &str, + ) -> BoundaryTopology { + BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { address, tls }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: token.to_string(), + } + } + + async fn spawn_tls_boundary( + certificate: Arc, + expected_token: String, + ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test TLS boundary"); + let address = listener.local_addr().expect("read test listener address"); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(certificate) + .accept(stream) + .await + else { + return; + }; + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let response = if request.boundary_id == "sandbox-1" + && request.bootstrap_token == expected_token + { + Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + } + } else { + Response::Error { + kind: "denied".to_string(), + message: "control authentication failed".to_string(), + } + }; + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + (address, task) + } + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + identity: crate::contract::ResolvedWorkloadIdentity::new( + 10_001, + 10_001, + Vec::new(), + "test".to_string(), + "sha256:test".to_string(), + ) + .expect("identity"), + } + } + + fn test_confirmation_evidence() -> crate::contract::SandboxConfirmEvidence { + crate::contract::SandboxConfirmEvidence { + generation: "test-generation".to_string(), + identity: sandbox().identity, + capabilities: crate::contract::CapabilityEvidence { + inheritable: 0, + permitted: 0, + effective: 0, + bounding: 0, + ambient: 0, + }, + no_new_privileges: true, + sandbox_dumpable: false, + child_dumpable: true, + core_limit_zero: true, + native_architecture: std::env::consts::ARCH.to_string(), + kernel_release: "test".to_string(), + seccomp: crate::contract::SeccompEvidence { + new_listener: true, + notification_round_trip: true, + id_validation: true, + addfd_send: true, + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: true, + task_memory_write: true, + cancellation: true, + }, + landlock_abi: 1, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + authenticated_supervisor: true, + session_epoch: "test-session".to_string(), + driver_fence: test_driver_fence(), + resource_claims: std::collections::BTreeMap::new(), + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + tls: test_certificate().client_tls, + }, + multiplexed: false, + host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = BoundaryTopology { + boundary_id: "other".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + tls: test_certificate().client_tls, + }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_rejects_an_unspecified_tcp_target() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { + address: "0.0.0.0:5500".parse().expect("valid address"), + tls: test_certificate().client_tls, + }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_accepts_a_concrete_tcp_target() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { + address: "10.42.0.7:5500".parse().expect("valid address"), + tls: test_certificate().client_tls, + }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + validate_topology(&topology, &sandbox(), "vm").expect("TCP topology should be valid"); + } + + #[test] + fn topology_rejects_invalid_tls_configuration() { + let topology = tls_topology( + "127.0.0.1:5500".parse().expect("valid address"), + BoundaryClientTls { + server_name: "not a dns name!".to_string(), + ca_certificate_pem: "not a certificate".to_string(), + certificate_chain_pem: "not a certificate".to_string(), + private_key_pem: "not a key".to_string(), + }, + "0123456789abcdef0123456789abcdef", + ); + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[tokio::test] + async fn tls_tcp_round_trip_verifies_server_certificate() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + &"a".repeat(32), + )); + + assert_eq!( + client + .exchange(Request::Confirm) + .await + .expect("TLS request"), + Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + } + ); + server.await.expect("TLS test server"); + } + + struct TestTlsIo(tokio_rustls::server::TlsStream); + + impl tokio::io::AsyncRead for TestTlsIo { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_read(context, buffer) + } + } + + impl tokio::io::AsyncWrite for TestTlsIo { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + Pin::new(&mut self.0).poll_write(context, buffer) + } + + fn poll_flush( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_flush(context) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(context) + } + } + + impl tonic::transport::server::Connected for TestTlsIo { + type ConnectInfo = (); + + fn connect_info(&self) -> Self::ConnectInfo {} + } + + #[tokio::test] + async fn grpc_session_reuses_one_tls_connection_for_concurrent_requests() { + const REQUESTS: usize = 8; + let certificate = test_certificate(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind gRPC test boundary"); + let address = listener.local_addr().expect("gRPC listener address"); + let server_config = certificate.server_config; + let accepted = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let accepted_by_server = accepted.clone(); + let handled = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let service = TestGrpcBoundary { + wait_for_half_close: false, + requests: handled.clone(), + }; + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept TLS session"); + accepted_by_server.fetch_add(1, Ordering::AcqRel); + let stream = tokio_rustls::TlsAcceptor::from(server_config.clone()) + .accept(stream) + .await + .expect("authenticate gRPC TLS session"); + let service = service.clone(); + tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(IsolationBoundaryServer::new(service)) + .serve_with_incoming(tokio_stream::iter([Ok::<_, std::io::Error>( + TestTlsIo(stream), + )])) + .await + .expect("serve test gRPC connection"); + }); + } + }); + let client = Arc::new(BoundaryClient::new(BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { + address, + tls: certificate.client_tls, + }, + multiplexed: true, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "a".repeat(32), + })); + let mut requests = Vec::new(); + for _ in 0..REQUESTS { + let client = client.clone(); + requests.push(tokio::spawn(async move { + let response = client + .exchange(Request::Confirm) + .await + .expect("gRPC confirm request"); + assert!(matches!(response, Response::Confirmed { .. })); + })); + } + for request in requests { + request.await.expect("gRPC client request"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(handled.load(Ordering::Acquire), REQUESTS); + assert_eq!(accepted.load(Ordering::Acquire), 1); + server.abort(); + } + + #[tokio::test] + async fn tls_tcp_flushes_large_control_requests_before_reading_response() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + &"a".repeat(32), + )); + let context = sandbox(); + + assert!(matches!( + client + .exchange(Request::StartAgent { + sandbox_id: context.sandbox_id, + spec: AgentSpecWire::from(context.agent), + policy: Box::new(SandboxPolicyWire::from(context.policy)), + ca_cert: Some(vec![b'c'; 16 * 1024]), + ca_bundle: Some(vec![b'b'; 256 * 1024]), + provider_env_revision: 0, + provider_env: HashMap::new(), + }) + .await + .expect("large TLS request"), + Response::Confirmed { .. } + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_unix_flushes_large_control_requests_before_reading_response() { + let socket_path = std::env::temp_dir().join(format!( + "openshell-large-control-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_nanos() + )); + let certificate = test_certificate(); + let server_config = certificate.server_config; + let listener = tokio::net::UnixListener::bind(&socket_path).expect("bind test socket"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .expect("accept TLS session"); + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response: Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + }, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + let context = sandbox(); + let client = BoundaryClient::new(BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: context.identity.clone(), + transport: BoundaryTransport::Unix { + socket_path: socket_path.clone(), + tls: certificate.client_tls, + }, + multiplexed: false, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "a".repeat(32), + }); + + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(2), + client.exchange(Request::StartAgent { + sandbox_id: context.sandbox_id, + spec: AgentSpecWire::from(context.agent), + policy: Box::new(SandboxPolicyWire::from(context.policy)), + ca_cert: Some(vec![b'c'; 16 * 1024]), + ca_bundle: Some(vec![b'b'; 256 * 1024]), + provider_env_revision: 0, + provider_env: HashMap::new(), + }) + ) + .await + .expect("large Unix TLS request timed out") + .expect("large Unix TLS request"), + Response::Confirmed { .. } + )); + server.await.expect("TLS test server"); + let _ = std::fs::remove_file(socket_path); + } + + #[tokio::test] + async fn tls_tcp_preserves_boundary_token_authentication() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary( + certificate.server_config, + "expected-token-expected-token-12".to_string(), + ) + .await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + "incorrect-token-incorrect-token", + )); + + assert!(matches!( + client.exchange(Request::Confirm).await, + Err(BackendError::Denied(_)) + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_rejects_an_untrusted_server_certificate() { + let presented = test_certificate(); + let trusted = test_certificate(); + let (address, server) = spawn_tls_boundary(presented.server_config, "a".repeat(32)).await; + let client = + BoundaryClient::new(tls_topology(address, trusted.client_tls, &"a".repeat(32))); + + assert!(matches!( + client.connect_boundary_once().await, + Err(BackendError::Unavailable(_)) + )); + // The server observes the client's fatal alert and may fail its accept; + // completing the task is sufficient for this rejection test. + let _ = server.await; + } +} diff --git a/crates/openshell-isolation-interface/tests/backend_conformance.rs b/crates/openshell-isolation-interface/tests/backend_conformance.rs index 8f813279c6..6f43048308 100644 --- a/crates/openshell-isolation-interface/tests/backend_conformance.rs +++ b/crates/openshell-isolation-interface/tests/backend_conformance.rs @@ -109,6 +109,7 @@ impl NetworkMediationSource for MockSource { process_generation: 1, }, policy_generation: 1, + timing: MediationTiming::default(), result, }) } @@ -134,6 +135,7 @@ impl NetworkMediationSource for UnattributedSource { process_generation: 1, }, policy_generation: 1, + timing: MediationTiming::default(), result, }) } @@ -375,11 +377,50 @@ fn confirmation_evidence() -> SandboxConfirmEvidence { tcp_deny_round_trip: true, authenticated_supervisor: true, session_epoch: "epoch-1".to_string(), - direct_egress_blocked: true, + driver_fence: DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + }, resource_claims: BTreeMap::new(), } } +#[test] +fn driver_fence_evidence_is_backend_specific_and_fail_closed() { + let docker = DriverFenceEvidence::Docker { + container_id: "sha256:container".to_string(), + network_mode: "none".to_string(), + unexpected_networks: Vec::new(), + }; + let kubernetes = DriverFenceEvidence::Kubernetes { + network_policy_uid: "policy-uid".to_string(), + network_policy_resource_version: "42".to_string(), + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + }; + let vm = DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + }; + + assert!(docker.validate_for_backend("docker").is_ok()); + assert!( + kubernetes + .validate_for_backend("kubernetes-proxy-pod") + .is_ok() + ); + assert!(vm.validate_for_backend("vm").is_ok()); + assert!(docker.validate_for_backend("vm").is_err()); + + let drifted = DriverFenceEvidence::Docker { + container_id: "sha256:container".to_string(), + network_mode: "bridge".to_string(), + unexpected_networks: vec!["bridge".to_string()], + }; + assert!(drifted.validate_for_backend("docker").is_err()); +} + /// The backend-independent supervisor sequence. Identical for every backend: /// this is the proof that adding a backend needs no supervisor lifecycle change. async fn drive( @@ -766,7 +807,7 @@ fn workload_identity_rejects_root_and_normalizes_groups() { let identity = ResolvedWorkloadIdentity::new( 1000, 1001, - vec![1003, 1002, 1003], + vec![1003, 1001, 1002, 1003], "policy".into(), "digest".into(), ) diff --git a/proto/isolation_boundary.proto b/proto/isolation_boundary.proto new file mode 100644 index 0000000000..333b8567bc --- /dev/null +++ b/proto/isolation_boundary.proto @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.isolation.v1; + +// IsolationBoundary multiplexes the protected sandbox-to-supervisor protocol +// over one authenticated gRPC connection. Each Exchange call is an independent +// HTTP/2 stream; Mediate is the persistent DNS and UDP data plane. +service IsolationBoundary { + rpc Exchange(stream BoundaryChunk) returns (stream BoundaryChunk); + rpc Mediate(stream BoundaryChunk) returns (stream BoundaryChunk); +} + +// BoundaryChunk carries an ordered fragment of the existing versioned +// isolation protocol. Message boundaries are not semantically significant. +message BoundaryChunk { + bytes data = 1; +} From 222a34c78a084d3180d494d524fed5e19a078bb5 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 16:50:11 -0700 Subject: [PATCH 2/7] fix(isolation): harden signals and separate process status from transport Signed-off-by: Drew Newberry --- crates/openshell-binary-identity/src/lib.rs | 60 ++- .../src/boundary_protocol.rs | 18 - .../src/contract.rs | 18 +- .../src/linux/child_seccomp.rs | 7 +- .../src/linux/mod.rs | 1 + .../src/linux/process_signal.rs | 211 ++++++++ .../src/linux/seccomp_notify.rs | 10 +- .../src/linux/socket_registry.rs | 2 +- .../src/linux/workload_launcher.rs | 2 +- .../src/mediation.rs | 38 +- .../src/remote.rs | 487 ++++++------------ .../tests/backend_conformance.rs | 2 +- 12 files changed, 449 insertions(+), 407 deletions(-) create mode 100644 crates/openshell-isolation-interface/src/linux/process_signal.rs diff --git a/crates/openshell-binary-identity/src/lib.rs b/crates/openshell-binary-identity/src/lib.rs index 6e5c347a8b..eb06934eaf 100644 --- a/crates/openshell-binary-identity/src/lib.rs +++ b/crates/openshell-binary-identity/src/lib.rs @@ -15,22 +15,20 @@ use openshell_isolation_interface::contract::{BinaryIdentity, ResolveError}; #[cfg(target_os = "linux")] use std::collections::HashMap; #[cfg(target_os = "linux")] -use std::sync::{Mutex, OnceLock}; +use std::sync::{Arc, Mutex}; #[cfg(target_os = "linux")] const EXECUTABLE_DIGEST_CACHE_CAPACITY: usize = 1_024; -#[cfg(target_os = "linux")] -static EXECUTABLE_DIGEST_CACHE: OnceLock>> = - OnceLock::new(); - /// Resolves executable identity from a Linux procfs process identifier. /// /// The configured scope bounds ancestry and cmdline collection to the observed /// PID namespace or a known workload process tree. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] pub struct ProcfsIdentityResolver { ancestry_scope: AncestryScope, + #[cfg(target_os = "linux")] + cache: Arc>>, } #[derive(Clone, Copy, Debug)] @@ -49,29 +47,33 @@ impl ProcfsIdentityResolver { /// Build a resolver that discovers a nested PID namespace's init process /// and never reports host-runtime ancestors outside that namespace. #[must_use] - pub const fn for_pid_namespace() -> Self { + pub fn for_pid_namespace() -> Self { Self { ancestry_scope: AncestryScope::PidNamespace, + #[cfg(target_os = "linux")] + cache: Arc::new(Mutex::new(HashMap::new())), } } /// Build a resolver bounded by the workload's trusted process-tree root. #[must_use] - pub const fn for_process_tree(ancestor_root: u32) -> Self { + pub fn for_process_tree(ancestor_root: u32) -> Self { Self { ancestry_scope: AncestryScope::ProcessTree(ancestor_root), + #[cfg(target_os = "linux")] + cache: Arc::new(Mutex::new(HashMap::new())), } } /// Resolve the identity for an authoritative process ID. - pub fn resolve(self, pid: u32) -> Result { + pub fn resolve(&self, pid: u32) -> Result { #[cfg(target_os = "linux")] { let ancestor_root = match self.ancestry_scope { AncestryScope::PidNamespace => nested_pid_namespace_init(pid), AncestryScope::ProcessTree(root) => Some(root), }; - resolve_linux_process(pid, ancestor_root) + resolve_linux_process(pid, ancestor_root, &self.cache) } #[cfg(not(target_os = "linux"))] @@ -94,11 +96,12 @@ impl ProcfsIdentityResolver { fn resolve_linux_process( pid: u32, ancestor_root: Option, + cache: &Mutex>, ) -> Result { let (snapshot, mut executable) = open_process_snapshot(pid)?; let binary_path = snapshot.binary_path.clone(); let executable_key = snapshot.executable_cache_key(); - let cached_digest = cached_executable_digest(executable_key); + let cached_digest = cached_executable_digest(cache, executable_key); let binary_digest = cached_digest.map_or_else(|| hash_executable(pid, &mut executable), Ok)?; let ancestor_processes = collect_ancestor_processes(&snapshot, ancestor_root); let ancestors = ancestor_processes @@ -128,7 +131,7 @@ fn resolve_linux_process( validate_process_snapshot(ancestor.pid, ancestor)?; } if cached_digest.is_none() { - cache_executable_digest(executable_key, binary_digest); + cache_executable_digest(cache, executable_key, binary_digest); } Ok(BinaryIdentity { @@ -184,13 +187,11 @@ impl ProcessSnapshot { } #[cfg(target_os = "linux")] -fn executable_digest_cache() -> &'static Mutex> { - EXECUTABLE_DIGEST_CACHE.get_or_init(|| Mutex::new(HashMap::new())) -} - -#[cfg(target_os = "linux")] -fn cached_executable_digest(key: ExecutableCacheKey) -> Option { - executable_digest_cache() +fn cached_executable_digest( + cache: &Mutex>, + key: ExecutableCacheKey, +) -> Option { + cache .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&key) @@ -198,8 +199,12 @@ fn cached_executable_digest(key: ExecutableCacheKey) -> Option { } #[cfg(target_os = "linux")] -fn cache_executable_digest(key: ExecutableCacheKey, digest: Sha256Digest) { - let mut cache = executable_digest_cache() +fn cache_executable_digest( + cache: &Mutex>, + key: ExecutableCacheKey, + digest: Sha256Digest, +) { + let mut cache = cache .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if cache.len() >= EXECUTABLE_DIGEST_CACHE_CAPACITY { @@ -435,6 +440,19 @@ fn cmdline_absolute_paths(cmdline: &[u8]) -> Vec { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn resolver_cache_is_owned_and_only_explicit_clones_share_it() { + let first = ProcfsIdentityResolver::for_pid_namespace(); + let shared = first.clone(); + let separate = ProcfsIdentityResolver::for_pid_namespace(); + assert!(Arc::ptr_eq(&first.cache, &shared.cache)); + assert!(!Arc::ptr_eq(&first.cache, &separate.cache)); + first.resolve(std::process::id()).unwrap(); + assert!(!shared.cache.lock().unwrap().is_empty()); + assert!(separate.cache.lock().unwrap().is_empty()); + } + #[test] fn resolves_current_process_from_live_executable() { let identity = ProcfsIdentityResolver::for_pid_namespace() diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index 310afacda6..05a1af4f99 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -37,10 +37,6 @@ pub const STREAM_EXIT: u8 = 3; pub const STREAM_STDIN_CLOSED: u8 = 4; /// Supervisor decision for a staged seccomp-mediated TCP open. pub const STREAM_NETWORK_DECISION: u8 = 5; -/// Supervisor response for one sandbox-local DNS relay exchange. -pub const STREAM_DNS_RESPONSE: u8 = 6; -/// Boundary acknowledgement that a mediated DNS response was committed. -pub const STREAM_DNS_ACK: u8 = 7; pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; /// Control-side endpoint for a driver-provisioned boundary. /// Supervisor-side mutual-TLS identity for one sandbox generation. @@ -202,8 +198,6 @@ pub struct BoundaryTopology { pub workload_identity: crate::contract::ResolvedWorkloadIdentity, /// Driver-provisioned control endpoint. pub transport: BoundaryTransport, - /// Multiplex logical exchanges over one authenticated gRPC connection. - pub multiplexed: bool, /// Trusted dial target for well-known host-gateway aliases, when the /// network supervisor cannot use the boundary's resolver view. #[serde(default)] @@ -226,7 +220,6 @@ impl fmt::Debug for BoundaryTopology { .field("generation", &self.generation) .field("session_epoch", &"") .field("transport", &self.transport) - .field("multiplexed", &self.multiplexed) .field("host_gateway_ip", &self.host_gateway_ip) .field("resource_claims", &self.resource_claims) .field("driver_fence", &self.driver_fence) @@ -265,8 +258,6 @@ pub struct BoundaryConfig { pub bootstrap_token: String, /// Driver-provisioned listener. pub listener: BoundaryListener, - /// Serve the protected protocol as multiplexed gRPC streams. - pub multiplexed: bool, /// Immutable coordinates the boundary requires from the control-side /// topology descriptor before accepting attachment. #[serde(default)] @@ -298,7 +289,6 @@ impl fmt::Debug for BoundaryConfig { .field("session_epoch", &"") .field("bootstrap_token", &"") .field("listener", &self.listener) - .field("multiplexed", &self.multiplexed) .field("resource_claims", &self.resource_claims) .field("resource_claim_files", &self.resource_claim_files) .field("workload_identity", &self.workload_identity) @@ -501,7 +491,6 @@ pub enum Request { /// plane. OpenMediation, AcceptNetwork, - AcceptDns, } impl Request { @@ -610,7 +599,6 @@ impl fmt::Debug for Request { .finish(), Self::OpenMediation => formatter.write_str("OpenMediation"), Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), - Self::AcceptDns => formatter.write_str("AcceptDns"), } } } @@ -660,12 +648,6 @@ pub enum Response { policy_generation: u64, timing: MediationTimingWire, }, - DnsQuery { - request: Vec, - transport: crate::contract::DnsTransport, - identity: BinaryIdentityWire, - timing: MediationTimingWire, - }, Error { kind: String, message: String, diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index 9e7f3480eb..74fdea736d 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -436,6 +436,11 @@ pub enum DriverFenceEvidence { network_mode: String, unexpected_networks: Vec, }, + Podman { + container_id: String, + network_mode: String, + unexpected_networks: Vec, + }, Kubernetes { network_policy_uid: String, network_policy_resource_version: String, @@ -454,6 +459,7 @@ impl DriverFenceEvidence { pub const fn backend_name(&self) -> &'static str { match self { Self::Docker { .. } => "docker", + Self::Podman { .. } => "podman", Self::Kubernetes { .. } => "kubernetes-proxy-pod", Self::Vm { .. } => "vm", } @@ -473,6 +479,16 @@ impl DriverFenceEvidence { && network_mode == "none" && unexpected_networks.is_empty() } + Self::Podman { + container_id, + network_mode, + unexpected_networks, + } => { + backend_name == "podman" + && !container_id.is_empty() + && network_mode == "none" + && unexpected_networks.is_empty() + } Self::Kubernetes { network_policy_uid, network_policy_resource_version, @@ -551,7 +567,7 @@ impl SandboxConfirmEvidence { && self.seccomp.task_memory_read && self.seccomp.task_memory_write && self.seccomp.cancellation - && self.landlock_abi > 0 + && self.landlock_abi >= 3 && self.landlock_allow_deny && self.udp_dns_round_trip && self.tcp_dns_round_trip diff --git a/crates/openshell-isolation-interface/src/linux/child_seccomp.rs b/crates/openshell-isolation-interface/src/linux/child_seccomp.rs index 21d5005e9d..b73fbf35c8 100644 --- a/crates/openshell-isolation-interface/src/linux/child_seccomp.rs +++ b/crates/openshell-isolation-interface/src/linux/child_seccomp.rs @@ -92,9 +92,10 @@ impl ChildHardeningProgram { /// /// `sandbox_tgid` is the sandbox PID as visible from its workload namespace. /// The filter blocks all direct thread-targeting through `tkill`, and blocks -/// process-directed operations that name the trusted sandbox leader. Worker -/// threads share that TGID and are therefore covered by `tgkill` and the -/// process-level APIs. +/// process-directed operations that name the trusted sandbox leader. The +/// ordinary workload listener must additionally mediate `kill` and +/// `rt_sigqueueinfo`: Linux accepts nonleader TIDs for those operations, so a +/// static TGID comparison alone cannot protect future sandbox worker threads. pub fn prepare(sandbox_tgid: u32) -> io::Result { if sandbox_tgid == 0 { return Err(io::Error::new( diff --git a/crates/openshell-isolation-interface/src/linux/mod.rs b/crates/openshell-isolation-interface/src/linux/mod.rs index ceba35e1f1..bad3d329dc 100644 --- a/crates/openshell-isolation-interface/src/linux/mod.rs +++ b/crates/openshell-isolation-interface/src/linux/mod.rs @@ -9,6 +9,7 @@ pub mod child_seccomp; pub mod landlock; pub mod proc_fd; +pub mod process_signal; pub mod seccomp_notify; pub mod socket_registry; pub mod task_memory; diff --git a/crates/openshell-isolation-interface/src/linux/process_signal.rs b/crates/openshell-isolation-interface/src/linux/process_signal.rs new file mode 100644 index 0000000000..a671dfb4de --- /dev/null +++ b/crates/openshell-isolation-interface/src/linux/process_signal.rs @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process-directed workload signals delivered through retained pidfds. +//! +//! Linux accepts a worker TID for `kill()`, so scalar seccomp checks against the +//! sandbox leader do not suffice. Resolve the thread group, exclude the live +//! sandbox, and retain the target before delivery. Never continue an inspected +//! numeric PID: it could be reused by a newly created sandbox worker. + +#![allow(unsafe_code)] + +use std::io; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +use crate::linux::seccomp_notify::{Notification, NotificationListener}; +use crate::linux::task_memory; + +/// Emulate one positive-target kill or `rt_sigqueueinfo` notification. +/// +/// The sandbox and workload must use the same procfs/PID namespace. The +/// sandbox TGID remains live throughout delivery, so an excluded numeric TGID +/// cannot be reused. Group/broadcast signaling remains deliberately denied. +/// Plain kill is broker-originated; queued signals retain their supplied +/// siginfo. Kernel signal permission and siginfo checks still apply. +pub fn mediate_process_signal( + listener: &NotificationListener, + notification: Notification, + sandbox_tgid: u32, +) -> io::Result<()> { + listener.validate_id(notification.id)?; + let target = scalar_int(notification.args[0]); + let signal = scalar_int(notification.args[1]); + if target <= 0 || !(0..=64).contains(&signal) { + return Err(io::Error::from_raw_os_error(if target <= 0 { + libc::EPERM + } else { + libc::EINVAL + })); + } + let target = u32::try_from(target).map_err(|_| io::Error::from_raw_os_error(libc::ESRCH))?; + let retained = retain_signal_target(target, sandbox_tgid)?; + // SAFETY: all-zero siginfo consists of valid integer/pointer fields. A + // queued operation copies the entire object once before it is consumed. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let info_ptr = match i64::from(notification.syscall) { + libc::SYS_kill => std::ptr::null(), + libc::SYS_rt_sigqueueinfo => { + // SAFETY: bytes exclusively spans the live siginfo object; all bit + // patterns are valid and task_memory requires a complete copy. + let bytes = unsafe { + std::slice::from_raw_parts_mut( + (&raw mut info).cast::(), + size_of::(), + ) + }; + task_memory::read_exact(notification.tid, notification.args[2], bytes)?; + // Positive/kernel-origin and SI_TKILL codes cannot be impersonated. + if info.si_code >= 0 || info.si_code == libc::SI_TKILL { + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + &raw const info + } + _ => return Err(io::Error::from_raw_os_error(libc::ENOSYS)), + }; + listener.validate_id(notification.id)?; + // SAFETY: retained owns a live pidfd; info is null or a complete trusted + // copy. The kernel targets that process object, never a reused numeric PID. + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + retained.as_raw_fd(), + signal, + info_ptr, + 0, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + listener.respond_value(notification.id, 0) +} + +fn scalar_int(value: u64) -> i32 { + let bytes = value.to_ne_bytes(); + #[cfg(target_endian = "little")] + let scalar = [bytes[0], bytes[1], bytes[2], bytes[3]]; + #[cfg(target_endian = "big")] + let scalar = [bytes[4], bytes[5], bytes[6], bytes[7]]; + i32::from_ne_bytes(scalar) +} + +fn retain_signal_target(tid: u32, sandbox_tgid: u32) -> io::Result { + if sandbox_tgid == 0 { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + let status = std::fs::read_to_string(format!("/proc/{tid}/status"))?; + let target_group = status + .lines() + .find_map(|line| { + line.strip_prefix("Tgid:") + .and_then(|value| value.trim().parse::().ok()) + }) + .ok_or_else(|| io::Error::from_raw_os_error(libc::ESRCH))?; + if target_group == sandbox_tgid || target_group == 0 { + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + // SAFETY: pidfd_open takes only scalar arguments and returns a new owned FD. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, target_group, 0) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = i32::try_from(fd).map_err(|_| io::Error::other("pidfd does not fit RawFd"))?; + // SAFETY: successful pidfd_open transferred this descriptor to the caller. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filtered_worker_cannot_signal_the_sandbox_through_a_tid() { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + let listener = crate::linux::seccomp_notify::install_workload_listener().unwrap(); + sender.send(listener).unwrap(); + // SAFETY: gettid has no arguments; signal zero checks permission + // without delivering a signal to the disposable test thread. + let tid = unsafe { libc::syscall(libc::SYS_gettid) }; + let result = unsafe { libc::syscall(libc::SYS_kill, tid, 0) }; + (result, io::Error::last_os_error().raw_os_error()) + }); + let listener = receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let notification = listener.receive().unwrap(); + let error = + mediate_process_signal(&listener, notification, std::process::id()).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(libc::EPERM)); + listener + .respond_errno(notification.id, libc::EPERM) + .unwrap(); + assert_eq!(worker.join().unwrap(), (-1, Some(libc::EPERM))); + } + + #[test] + fn retained_child_can_be_signaled_without_numeric_pid_delivery() { + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let retained = retain_signal_target(child.id(), std::process::id()).unwrap(); + // SAFETY: this pidfd owns the disposable child launched by this test. + assert_eq!( + unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + retained.as_raw_fd(), + libc::SIGTERM, + std::ptr::null::(), + 0, + ) + }, + 0 + ); + assert!(!child.wait().unwrap().success()); + assert_eq!( + unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + retained.as_raw_fd(), + 0, + std::ptr::null::(), + 0, + ) + }, + -1 + ); + assert_eq!(io::Error::last_os_error().raw_os_error(), Some(libc::ESRCH)); + } + + #[test] + fn rejects_live_sandbox_worker_tid() { + let sandbox_tgid = std::process::id(); + std::thread::spawn(move || { + // SAFETY: gettid has no arguments or side effects. + let tid = u32::try_from(unsafe { libc::syscall(libc::SYS_gettid) }).unwrap(); + assert_ne!(tid, sandbox_tgid); + assert_eq!( + retain_signal_target(tid, sandbox_tgid) + .unwrap_err() + .raw_os_error(), + Some(libc::EPERM) + ); + }) + .join() + .unwrap(); + } + + #[test] + fn rejects_leader_and_preserves_scalar_pid_semantics() { + let pid = std::process::id(); + assert_eq!( + retain_signal_target(pid, pid).unwrap_err().raw_os_error(), + Some(libc::EPERM) + ); + assert_eq!(scalar_int(u64::MAX), -1); + assert_eq!(scalar_int(1 << 32 | 0x7b), 123); + } +} diff --git a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs index 081224abac..c5e8e9bd9c 100644 --- a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs +++ b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs @@ -361,6 +361,8 @@ pub fn install_workload_listener() -> io::Result { libc::SYS_sendmmsg, libc::SYS_getpeername, libc::SYS_setsockopt, + libc::SYS_kill, + libc::SYS_rt_sigqueueinfo, ]) } @@ -496,14 +498,18 @@ fn probe_task_memory_copy() -> io::Result<()> { let source = 0x1122_3344_5566_7788_u64; let tid = std::process::id(); let mut source_bytes = [0_u8; size_of::()]; - super::task_memory::read_exact(tid, std::ptr::addr_of!(source) as u64, &mut source_bytes)?; + crate::linux::task_memory::read_exact( + tid, + std::ptr::addr_of!(source) as u64, + &mut source_bytes, + )?; let mut copied = u64::from_ne_bytes(source_bytes); if copied != source { return Err(io::Error::other("task-memory probe read wrong value")); } let replacement = 0xaabb_ccdd_eeff_0011_u64; - super::task_memory::write_exact( + crate::linux::task_memory::write_exact( tid, std::ptr::addr_of_mut!(copied) as u64, &replacement.to_ne_bytes(), diff --git a/crates/openshell-isolation-interface/src/linux/socket_registry.rs b/crates/openshell-isolation-interface/src/linux/socket_registry.rs index 966612144e..81a71920e3 100644 --- a/crates/openshell-isolation-interface/src/linux/socket_registry.rs +++ b/crates/openshell-isolation-interface/src/linux/socket_registry.rs @@ -13,7 +13,7 @@ use std::os::fd::{AsRawFd, BorrowedFd, OwnedFd, RawFd}; use rustix::fs::fstat; -use super::proc_fd; +use crate::linux::proc_fd; /// Stable identity for one mediated socket within a listener generation. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] diff --git a/crates/openshell-isolation-interface/src/linux/workload_launcher.rs b/crates/openshell-isolation-interface/src/linux/workload_launcher.rs index 9197cf28f0..8602d9df80 100644 --- a/crates/openshell-isolation-interface/src/linux/workload_launcher.rs +++ b/crates/openshell-isolation-interface/src/linux/workload_launcher.rs @@ -16,7 +16,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use std::thread; -use super::seccomp_notify::{NotificationListener, install_workload_listener}; +use crate::linux::seccomp_notify::{NotificationListener, install_workload_listener}; type LaunchJob = Box; diff --git a/crates/openshell-isolation-interface/src/mediation.rs b/crates/openshell-isolation-interface/src/mediation.rs index 8d809c7807..317de7282b 100644 --- a/crates/openshell-isolation-interface/src/mediation.rs +++ b/crates/openshell-isolation-interface/src/mediation.rs @@ -7,14 +7,13 @@ //! transport so one busy connection cannot head-of-line block another. use std::io; -use std::net::SocketAddr; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; use crate::boundary_protocol::{BinaryIdentityWire, MediationTimingWire}; -use crate::contract::{DnsTransport, NetworkSocketMetadata}; +use crate::contract::DnsTransport; const HEADER_BYTES: usize = 13; const MAX_METADATA_BYTES: usize = 256 * 1024; @@ -23,13 +22,8 @@ const MAX_FRAME_BYTES: usize = MAX_METADATA_BYTES; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum MediationFrameKind { - NetworkOpen = 1, - NetworkDecision = 2, - NetworkData = 3, - StreamClosed = 4, DnsQuery = 5, DnsResponse = 6, - NetworkEof = 7, } impl TryFrom for MediationFrameKind { @@ -37,13 +31,8 @@ impl TryFrom for MediationFrameKind { fn try_from(value: u8) -> Result { match value { - 1 => Ok(Self::NetworkOpen), - 2 => Ok(Self::NetworkDecision), - 3 => Ok(Self::NetworkData), - 4 => Ok(Self::StreamClosed), 5 => Ok(Self::DnsQuery), 6 => Ok(Self::DnsResponse), - 7 => Ok(Self::NetworkEof), _ => Err(io::Error::new( io::ErrorKind::InvalidData, format!("unknown mediation frame kind {value}"), @@ -59,15 +48,6 @@ pub struct MediationFrame { pub payload: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct NetworkOpenWire { - pub identity: BinaryIdentityWire, - pub destination: SocketAddr, - pub socket: NetworkSocketMetadata, - pub policy_generation: u64, - pub timing: MediationTimingWire, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DnsQueryWire { pub request: Vec, @@ -125,8 +105,16 @@ pub async fn read_frame( } reader.read_exact(&mut header[1..]).await?; let kind = MediationFrameKind::try_from(header[0])?; - let stream_id = u64::from_be_bytes(header[1..9].try_into().expect("fixed header")); - let length = u32::from_be_bytes(header[9..13].try_into().expect("fixed header")) as usize; + let stream_id = u64::from_be_bytes( + header[1..9] + .try_into() + .map_err(|_| io::Error::other("invalid stream ID header"))?, + ); + let length = u32::from_be_bytes( + header[9..13] + .try_into() + .map_err(|_| io::Error::other("invalid payload length header"))?, + ) as usize; if length > MAX_FRAME_BYTES.max(MAX_METADATA_BYTES) { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -152,7 +140,7 @@ mod tests { let send = tokio::spawn(async move { write_frame( &mut writer, - MediationFrameKind::NetworkData, + MediationFrameKind::DnsResponse, 42, &[0, 1, 2, 255], ) @@ -160,7 +148,7 @@ mod tests { .unwrap(); }); let frame = read_frame(&mut reader).await.unwrap().unwrap(); - assert_eq!(frame.kind, MediationFrameKind::NetworkData); + assert_eq!(frame.kind, MediationFrameKind::DnsResponse); assert_eq!(frame.stream_id, 42); assert_eq!(frame.payload, vec![0, 1, 2, 255]); send.await.unwrap(); diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 09583d788a..3410601376 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -5,6 +5,7 @@ #![allow(unsafe_code)] +#[cfg(test)] use std::collections::HashMap; #[cfg(target_os = "linux")] use std::mem::size_of; @@ -31,15 +32,14 @@ use openshell_core::proto::isolation::v1::{ }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::UnixStream; -use tokio::sync::Notify; use tokio_stream::wrappers::ReceiverStream; use crate::boundary_protocol::{ AgentSpecWire, BoundaryClientTls, BoundaryTopology, BoundaryTransport, DnsQueryResultWire, - ExecSpecWire, ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, - ResponseEnvelope, STREAM_DNS_ACK, STREAM_DNS_RESPONSE, STREAM_EXIT, STREAM_STDERR, - STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, decode_frame, - encode_frame, read_stream_frame, validate_resource_claims, write_stream_frame, + ExecSpecWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope, + STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, + SandboxPolicyWire, SignalWire, decode_frame, encode_frame, read_stream_frame, + validate_resource_claims, write_stream_frame, }; use crate::mediation::{self, DnsQueryWire, MediationFrame, MediationFrameKind}; @@ -608,47 +608,6 @@ impl BoundaryPortForward for RemotePortForward { struct RemoteExecProcess { client: Arc, process_id: String, - exit: Arc, -} - -struct RemoteExit { - result: std::sync::Mutex>>, - changed: Notify, -} - -impl RemoteExit { - fn new() -> Self { - Self { - result: std::sync::Mutex::new(None), - changed: Notify::new(), - } - } - - fn set(&self, result: Result) { - let mut current = self - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if current.is_none() { - *current = Some(result); - self.changed.notify_waiters(); - } - } - - async fn wait(&self) -> Result { - loop { - let changed = self.changed.notified(); - let result = self - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone(); - if let Some(result) = result { - return result.map_err(BackendError::Terminated); - } - changed.await; - } - } } #[async_trait] @@ -658,7 +617,12 @@ impl BoundaryProcess for RemoteExecProcess { } async fn wait(&self) -> Result { - self.exit.wait().await + RemoteProcess { + client: self.client.clone(), + process_id: self.process_id.clone(), + } + .wait() + .await } async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { @@ -718,19 +682,16 @@ async fn open_exec_session( let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); - let exit = Arc::new(RemoteExit::new()); tokio::spawn(pump_exec_input(stdin_pump, network_writer)); - tokio::spawn(pump_exec_responses( + tokio::spawn(pump_process_responses( network_reader, stdout_pump, stderr_pump, - exit.clone(), )); let process: Arc = Arc::new(RemoteExecProcess { client: client.clone(), process_id: process_id.clone(), - exit, }); let terminal: Option> = if pty { Some(Arc::new(RemoteTerminal { client, process_id })) @@ -773,56 +734,9 @@ async fn pump_exec_input( } } -async fn pump_exec_responses( - mut network: tokio::io::ReadHalf, - mut stdout: tokio::io::DuplexStream, - mut stderr: tokio::io::DuplexStream, - exit: Arc, -) { - loop { - match read_stream_frame(&mut network).await { - Ok(Some((STREAM_STDOUT, payload))) => { - if stdout.write_all(&payload).await.is_err() { - exit.set(Err("boundary exec stdout consumer closed".to_string())); - return; - } - } - Ok(Some((STREAM_STDERR, payload))) => { - if stderr.write_all(&payload).await.is_err() { - exit.set(Err("boundary exec stderr consumer closed".to_string())); - return; - } - } - Ok(Some((STREAM_EXIT, payload))) => { - let result = serde_json::from_slice::(&payload) - .map(BoundaryExitStatus::from) - .map_err(|error| format!("decode boundary exec exit: {error}")); - exit.set(result); - return; - } - Ok(Some((channel, _))) => { - exit.set(Err(format!( - "boundary exec returned unexpected stream channel {channel}" - ))); - return; - } - Ok(None) => { - exit.set(Err( - "boundary exec stream closed before exit status".to_string() - )); - return; - } - Err(error) => { - exit.set(Err(format!("read boundary exec stream: {error}"))); - return; - } - } - } -} - /// Pulls boundary proxy connections over independent HTTP/2 streams. /// -/// DNS and UDP control messages share the compact persistent mediation +/// DNS control messages share the compact persistent mediation /// session, but TCP byte streams use HTTP/2's native multiplexing. Nesting all /// TCP connections inside one application-level writer creates avoidable /// head-of-line blocking during concurrent TLS handshakes. @@ -873,74 +787,14 @@ struct RemoteDnsMediation { #[async_trait] impl DnsMediationSource for RemoteDnsMediation { async fn accept(&self) -> Result { - if self.client.topology.multiplexed { - loop { - let session = self.client.mediation_session().await?; - match session.accept_dns().await { - Ok(query) => return Ok(query), - Err(BackendError::Unavailable(_)) if !session.is_healthy() => {} - Err(error) => return Err(error), - } + loop { + let session = self.client.mediation_session().await?; + match session.accept_dns().await { + Ok(query) => return Ok(query), + Err(BackendError::Unavailable(_)) if !session.is_healthy() => {} + Err(error) => return Err(error), } } - let (stream, response) = self.client.open_exchange(Request::AcceptDns).await?; - let Response::DnsQuery { - request, - transport, - identity, - timing, - } = response - else { - return Err(unexpected_response("dns_query", &response)); - }; - let (response_tx, response_rx) = tokio::sync::oneshot::channel(); - tokio::spawn(complete_dns_query(stream, response_rx)); - Ok(MediatedDnsQuery { - request, - transport, - binary_identity: identity.into_result(), - timing: MediationTiming { - sandbox_notification_to_queue: Duration::from_micros( - timing.notification_to_queue_us, - ), - sandbox_queue_wait: Duration::from_micros(timing.queue_wait_us), - supervisor_received_at: Instant::now(), - }, - response: response_tx, - }) - } -} - -async fn complete_dns_query( - mut boundary: BoundaryDuplexStream, - response: tokio::sync::oneshot::Receiver, BackendError>>, -) { - let result = match response.await { - Ok(Ok(response)) => DnsQueryResultWire::Response(response), - Ok(Err(error)) => DnsQueryResultWire::Error(error.to_string()), - Err(_) => DnsQueryResultWire::Error("DNS mediation was cancelled".to_string()), - }; - let payload = match serde_json::to_vec(&result) { - Ok(payload) => payload, - Err(error) => { - tracing::warn!(%error, "encode mediated DNS response failed: {error}"); - return; - } - }; - if let Err(error) = write_stream_frame(&mut boundary, STREAM_DNS_RESPONSE, &payload).await { - tracing::warn!(%error, "write mediated DNS response failed: {error}"); - return; - } - match tokio::time::timeout(REQUEST_TIMEOUT, read_stream_frame(&mut boundary)).await { - Ok(Ok(Some((STREAM_DNS_ACK, payload)))) if payload.is_empty() => {} - Ok(Ok(Some((channel, _)))) => { - tracing::warn!(channel, "unexpected mediated DNS acknowledgement channel"); - } - Ok(Ok(None)) => tracing::warn!("boundary closed before acknowledging DNS response"), - Ok(Err(error)) => { - tracing::warn!(%error, "read mediated DNS acknowledgement failed: {error}"); - } - Err(_) => tracing::warn!("timed out waiting for mediated DNS acknowledgement"), } } @@ -988,9 +842,6 @@ struct OutboundMediationFrame { payload: Vec, } -type MediationRoutes = - Arc>>>; - struct ClientMediationSession { dns: tokio::sync::Mutex>, healthy: Arc, @@ -1031,7 +882,6 @@ async fn run_client_mediation( let (mut reader, mut writer) = tokio::io::split(stream); let (outbound_tx, mut outbound_rx) = tokio::sync::mpsc::channel::(MEDIATION_EVENT_QUEUE); - let routes = Arc::new(tokio::sync::Mutex::new(HashMap::new())); let writer_task = async { while let Some(frame) = outbound_rx.recv().await { mediation::write_frame(&mut writer, frame.kind, frame.stream_id, &frame.payload) @@ -1041,7 +891,7 @@ async fn run_client_mediation( }; let reader_task = async { while let Some(frame) = mediation::read_frame(&mut reader).await? { - dispatch_client_mediation_frame(frame, &dns_tx, &outbound_tx, &routes).await?; + dispatch_client_mediation_frame(frame, &dns_tx, &outbound_tx).await?; } Ok::<(), std::io::Error>(()) }; @@ -1051,7 +901,6 @@ async fn run_client_mediation( result = &mut writer_task => result, result = &mut reader_task => result, }; - routes.lock().await.clear(); result } @@ -1059,7 +908,6 @@ async fn dispatch_client_mediation_frame( frame: MediationFrame, dns_tx: &tokio::sync::mpsc::Sender, outbound: &tokio::sync::mpsc::Sender, - routes: &MediationRoutes, ) -> std::io::Result<()> { match frame.kind { MediationFrameKind::DnsQuery => { @@ -1107,17 +955,7 @@ async fn dispatch_client_mediation_frame( ) })?; } - MediationFrameKind::StreamClosed => { - let route = routes.lock().await.get(&frame.stream_id).cloned(); - if let Some(route) = route { - let _ = route.send(frame).await; - } - } - MediationFrameKind::NetworkOpen - | MediationFrameKind::NetworkDecision - | MediationFrameKind::NetworkData - | MediationFrameKind::NetworkEof - | MediationFrameKind::DnsResponse => { + MediationFrameKind::DnsResponse => { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!( @@ -1178,7 +1016,9 @@ impl BoundaryClient { Err(error) => return Err(error), } } - unreachable!("bounded wait reconnect loop always returns") + Err(BackendError::Unavailable( + "boundary wait retry budget exhausted".to_string(), + )) } async fn call_stream( @@ -1250,11 +1090,7 @@ impl BoundaryClient { envelope: &RequestEnvelope, ) -> Result<(BoundaryDuplexStream, Response), BackendError> { let request_id = envelope.request_id.clone(); - let mut stream = if self.topology.multiplexed { - self.open_grpc_stream(GrpcStreamKind::Exchange).await? - } else { - self.connect_boundary().await? - }; + let mut stream = self.open_grpc_stream(GrpcStreamKind::Exchange).await?; let frame = encode_frame(envelope) .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; stream.write_all(&frame).await.map_err(|error| { @@ -1326,7 +1162,9 @@ impl BoundaryClient { Err(error) => return Err(error), } } - unreachable!("bounded mediation reconnect loop always returns") + Err(BackendError::Unavailable( + "boundary mediation retry budget exhausted".to_string(), + )) } async fn open_mediation_session(&self) -> Result, BackendError> { @@ -1390,17 +1228,7 @@ impl BoundaryClient { Ok(channel) } - async fn connect_boundary(&self) -> Result { - let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; - loop { - match self.connect_boundary_once().await { - Ok(stream) => return Ok(stream), - Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), - Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, - } - } - } - + #[cfg(test)] async fn connect_boundary_once(&self) -> Result { connect_boundary_once(&self.topology).await } @@ -1651,7 +1479,7 @@ mod tests { use std::task::{Context, Poll}; use super::*; - use crate::boundary_protocol::generate_boundary_mutual_tls_material; + use crate::boundary_protocol::{ExitStatusWire, generate_boundary_mutual_tls_material}; use openshell_core::policy::{ FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, }; @@ -1683,6 +1511,7 @@ mod tests { #[derive(Clone)] struct TestGrpcBoundary { wait_for_half_close: bool, + expected_token: String, requests: Arc, } @@ -1702,6 +1531,7 @@ mod tests { let mut inbound = request.into_inner(); let wait_for_half_close = self.wait_for_half_close; let requests = self.requests.clone(); + let expected_token = self.expected_token.clone(); let (outbound, outbound_rx) = tokio::sync::mpsc::channel(1); tokio::spawn(async move { let mut frame = Vec::new(); @@ -1733,8 +1563,28 @@ mod tests { }; match encode_frame(&ResponseEnvelope { request_id: envelope.request_id, - response: Response::Confirmed { - evidence: Box::new(test_confirmation_evidence()), + response: if envelope.bootstrap_token != expected_token + || envelope.boundary_id != "sandbox-1" + { + Response::Error { + kind: "denied".to_string(), + message: "control authentication failed".to_string(), + } + } else if matches!(envelope.request, Request::Wait { .. }) { + Response::Exited { + status: ExitStatusWire::Exited(23), + } + } else if matches!(envelope.request, Request::Exec { .. }) { + Response::ExecStarted { + process_id: "test-generation:exec:1".to_string(), + pty: false, + } + } else if matches!(envelope.request, Request::AttachProcess { .. }) { + Response::ProcessAttached { terminal: false } + } else { + Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + } }, }) { Ok(response) => response, @@ -1779,6 +1629,7 @@ mod tests { let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let service = TestGrpcBoundary { wait_for_half_close: true, + expected_token: "a".repeat(32), requests, }; let server = tokio::spawn(async move { @@ -1807,83 +1658,49 @@ mod tests { } #[tokio::test] - async fn remote_dns_exchange_returns_supervisor_response() { - let socket_path = std::env::temp_dir().join(format!( - "openshell-dns-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let certificate = test_certificate(); - let server_config = certificate.server_config.clone(); - let listener = tokio::net::UnixListener::bind(&socket_path).unwrap(); + async fn persistent_dns_exchange_returns_supervisor_response() { + let (client_stream, mut server_stream) = tokio::io::duplex(4096); + let session = ClientMediationSession::start(Box::new(client_stream)); let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut stream = tokio_rustls::TlsAcceptor::from(server_config) - .accept(stream) - .await - .unwrap(); - let declared = stream.read_u32().await.unwrap() as usize; - let mut frame = vec![0_u8; 4 + declared]; - frame[..4].copy_from_slice(&u32::try_from(declared).unwrap().to_be_bytes()); - stream.read_exact(&mut frame[4..]).await.unwrap(); - let request: RequestEnvelope = decode_frame(&frame).unwrap(); - assert_eq!(request.request, Request::AcceptDns); - let response = encode_frame(&ResponseEnvelope { - request_id: request.request_id, - response: Response::DnsQuery { - request: vec![1, 2, 3], - transport: crate::contract::DnsTransport::Udp, - identity: crate::boundary_protocol::BinaryIdentityWire { - binary_path: Some(PathBuf::from("/usr/bin/dig")), - binary_digest: Some("a".repeat(64)), - ancestors: Vec::new(), - cmdline_paths: Vec::new(), - resolve_error: None, - }, - timing: crate::boundary_protocol::MediationTimingWire::default(), + let query = DnsQueryWire { + request: vec![1, 2, 3], + transport: crate::contract::DnsTransport::Udp, + identity: crate::boundary_protocol::BinaryIdentityWire { + binary_path: Some(PathBuf::from("/usr/bin/dig")), + binary_digest: Some("a".repeat(64)), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: None, }, - }) + timing: crate::boundary_protocol::MediationTimingWire::default(), + }; + mediation::write_frame( + &mut server_stream, + MediationFrameKind::DnsQuery, + 42, + &mediation::encode_json(&query).unwrap(), + ) + .await .unwrap(); - stream.write_all(&response).await.unwrap(); - let (channel, payload) = read_stream_frame(&mut stream).await.unwrap().unwrap(); - assert_eq!(channel, STREAM_DNS_RESPONSE); + let reply = mediation::read_frame(&mut server_stream) + .await + .unwrap() + .unwrap(); + assert_eq!(reply.kind, MediationFrameKind::DnsResponse); + assert_eq!(reply.stream_id, 42); assert_eq!( - serde_json::from_slice::(&payload).unwrap(), + mediation::decode_json::(&reply.payload).unwrap(), DnsQueryResultWire::Response(vec![4, 5, 6]) ); - write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) - .await - .unwrap(); }); - let client = Arc::new(BoundaryClient::new(BoundaryTopology { - boundary_id: "sandbox-1".to_string(), - generation: "test-generation".to_string(), - session_epoch: "test-session".to_string(), - workload_identity: sandbox().identity, - transport: BoundaryTransport::Unix { - socket_path: socket_path.clone(), - tls: certificate.client_tls, - }, - multiplexed: false, - host_gateway_ip: None, - resource_claims: std::collections::BTreeMap::new(), - driver_fence: test_driver_fence(), - bootstrap_token: "a".repeat(32), - })); - let source = RemoteDnsMediation { client }; - let query = source.accept().await.unwrap(); + let query = session.accept_dns().await.unwrap(); assert_eq!(query.request, [1, 2, 3]); - assert_eq!(query.transport, crate::contract::DnsTransport::Udp); assert_eq!( query.binary_identity.unwrap().binary_path, PathBuf::from("/usr/bin/dig") ); query.response.send(Ok(vec![4, 5, 6])).unwrap(); server.await.unwrap(); - let _ = std::fs::remove_file(socket_path); } struct TestCertificate { @@ -1937,7 +1754,6 @@ mod tests { session_epoch: "test-session".to_string(), workload_identity: sandbox().identity, transport: BoundaryTransport::TlsTcp { address, tls }, - multiplexed: false, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -1949,49 +1765,36 @@ mod tests { certificate: Arc, expected_token: String, ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind test TLS boundary"); - let address = listener.local_addr().expect("read test listener address"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); let task = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept TLS control client"); - let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(certificate) + let (stream, _) = listener.accept().await.unwrap(); + let Ok(stream) = tokio_rustls::TlsAcceptor::from(certificate) .accept(stream) .await else { return; }; - let declared_u32 = stream.read_u32().await.expect("read request length"); - let declared = declared_u32 as usize; - let mut frame = vec![0_u8; 4 + declared]; - frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); - stream - .read_exact(&mut frame[4..]) - .await - .expect("read request frame"); - let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); - let response = if request.boundary_id == "sandbox-1" - && request.bootstrap_token == expected_token - { - Response::Confirmed { - evidence: Box::new(test_confirmation_evidence()), - } - } else { - Response::Error { - kind: "denied".to_string(), - message: "control authentication failed".to_string(), - } - }; - let frame = encode_frame(&ResponseEnvelope { - request_id: request.request_id, - response, - }) - .expect("encode response"); - stream.write_all(&frame).await.expect("write response"); + serve_test_grpc(Box::new(stream), expected_token).await; }); (address, task) } + async fn serve_test_grpc(stream: BoundaryDuplexStream, expected_token: String) { + let service = TestGrpcBoundary { + wait_for_half_close: false, + expected_token, + requests: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }; + tonic::transport::Server::builder() + .add_service(IsolationBoundaryServer::new(service)) + .serve_with_incoming(tokio_stream::iter([Ok::<_, std::io::Error>(TestTlsIo( + stream, + ))])) + .await + .unwrap(); + } + fn sandbox() -> SandboxContext { SandboxContext { sandbox_id: "sandbox-1".to_string(), @@ -2048,7 +1851,7 @@ mod tests { task_memory_write: true, cancellation: true, }, - landlock_abi: 1, + landlock_abi: 3, landlock_allow_deny: true, udp_dns_round_trip: true, tcp_dns_round_trip: true, @@ -2072,7 +1875,6 @@ mod tests { socket_path: PathBuf::from("/tmp/vsock.sock"), tls: test_certificate().client_tls, }, - multiplexed: false, host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2094,7 +1896,6 @@ mod tests { socket_path: PathBuf::from("/tmp/vsock.sock"), tls: test_certificate().client_tls, }, - multiplexed: false, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2117,7 +1918,6 @@ mod tests { address: "0.0.0.0:5500".parse().expect("valid address"), tls: test_certificate().client_tls, }, - multiplexed: false, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2140,7 +1940,6 @@ mod tests { address: "10.42.0.7:5500".parse().expect("valid address"), tls: test_certificate().client_tls, }, - multiplexed: false, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2186,10 +1985,47 @@ mod tests { evidence: Box::new(test_confirmation_evidence()), } ); - server.await.expect("TLS test server"); + server.abort(); + } + + #[tokio::test] + async fn exec_wait_survives_output_loss_and_reattachment() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = Arc::new(BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + &"a".repeat(32), + ))); + let session = open_exec_session( + client, + ExecSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + env: Vec::new(), + workdir: None, + pty: false, + }, + ) + .await + .unwrap(); + // The test peer closes its I/O stream without an exit frame. Neither + // that loss nor a dropped reader can invalidate the process handle. + drop(session.stdin); + drop(session.stdout); + drop(session.stderr); + let attachment = session.process.attach().await.unwrap(); + drop(attachment); + for _ in 0..2 { + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(23) + ); + } + server.abort(); } - struct TestTlsIo(tokio_rustls::server::TlsStream); + struct TestTlsIo(BoundaryDuplexStream); impl tokio::io::AsyncRead for TestTlsIo { fn poll_read( @@ -2245,6 +2081,7 @@ mod tests { let handled = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let service = TestGrpcBoundary { wait_for_half_close: false, + expected_token: "a".repeat(32), requests: handled.clone(), }; let server = tokio::spawn(async move { @@ -2260,7 +2097,7 @@ mod tests { tonic::transport::Server::builder() .add_service(IsolationBoundaryServer::new(service)) .serve_with_incoming(tokio_stream::iter([Ok::<_, std::io::Error>( - TestTlsIo(stream), + TestTlsIo(Box::new(stream)), )])) .await .expect("serve test gRPC connection"); @@ -2276,7 +2113,6 @@ mod tests { address, tls: certificate.client_tls, }, - multiplexed: true, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2328,7 +2164,7 @@ mod tests { .expect("large TLS request"), Response::Confirmed { .. } )); - server.await.expect("TLS test server"); + server.abort(); } #[tokio::test] @@ -2345,28 +2181,12 @@ mod tests { let server_config = certificate.server_config; let listener = tokio::net::UnixListener::bind(&socket_path).expect("bind test socket"); let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept TLS control client"); - let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + let (stream, _) = listener.accept().await.unwrap(); + let stream = tokio_rustls::TlsAcceptor::from(server_config) .accept(stream) .await - .expect("accept TLS session"); - let declared_u32 = stream.read_u32().await.expect("read request length"); - let declared = declared_u32 as usize; - let mut frame = vec![0_u8; 4 + declared]; - frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); - stream - .read_exact(&mut frame[4..]) - .await - .expect("read request frame"); - let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); - let frame = encode_frame(&ResponseEnvelope { - request_id: request.request_id, - response: Response::Confirmed { - evidence: Box::new(test_confirmation_evidence()), - }, - }) - .expect("encode response"); - stream.write_all(&frame).await.expect("write response"); + .unwrap(); + serve_test_grpc(Box::new(stream), "a".repeat(32)).await; }); let context = sandbox(); let client = BoundaryClient::new(BoundaryTopology { @@ -2378,7 +2198,6 @@ mod tests { socket_path: socket_path.clone(), tls: certificate.client_tls, }, - multiplexed: false, host_gateway_ip: None, resource_claims: std::collections::BTreeMap::new(), driver_fence: test_driver_fence(), @@ -2403,7 +2222,7 @@ mod tests { .expect("large Unix TLS request"), Response::Confirmed { .. } )); - server.await.expect("TLS test server"); + server.abort(); let _ = std::fs::remove_file(socket_path); } @@ -2425,7 +2244,7 @@ mod tests { client.exchange(Request::Confirm).await, Err(BackendError::Denied(_)) )); - server.await.expect("TLS test server"); + server.abort(); } #[tokio::test] diff --git a/crates/openshell-isolation-interface/tests/backend_conformance.rs b/crates/openshell-isolation-interface/tests/backend_conformance.rs index 6f43048308..1542a040db 100644 --- a/crates/openshell-isolation-interface/tests/backend_conformance.rs +++ b/crates/openshell-isolation-interface/tests/backend_conformance.rs @@ -369,7 +369,7 @@ fn confirmation_evidence() -> SandboxConfirmEvidence { task_memory_write: true, cancellation: true, }, - landlock_abi: 1, + landlock_abi: 3, landlock_allow_deny: true, udp_dns_round_trip: true, tcp_dns_round_trip: true, From b404cbfa89de79408481fb441e038875002136b8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:06:40 -0700 Subject: [PATCH 3/7] fix(isolation): validate remote confirmation through public contract Signed-off-by: Drew Newberry --- crates/openshell-isolation-interface/src/remote.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 3410601376..56dbd12ac0 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -303,7 +303,6 @@ impl BoundBoundary for RemoteBound { let Response::Confirmed { evidence } = response else { return Err(unexpected_response("confirmed_with_evidence", &response)); }; - evidence.validate(&self.identity)?; if evidence.generation != self.generation || evidence.session_epoch != self.session_epoch || evidence.resource_claims != self.resource_claims @@ -314,7 +313,7 @@ impl BoundBoundary for RemoteBound { .to_string(), )); } - Ok(ConfirmedBoundary::new( + ConfirmedBoundary::try_new( Box::new(RemoteReady { client: self.client, agent: self.agent, @@ -324,7 +323,8 @@ impl BoundBoundary for RemoteBound { provider_credentials: self.provider_credentials, }), *evidence, - )) + &self.identity, + ) } } From 4dcb473e04173b70c86d169de3d5159f3f76817e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:09:34 -0700 Subject: [PATCH 4/7] fix(isolation): validate wire state and propagate snapshot failures Signed-off-by: Drew Newberry --- architecture/sandbox.md | 20 +++ .../src/provider_credentials.rs | 80 ++++++++---- .../src/boundary_protocol.rs | 117 +++++++++++++----- .../src/contract.rs | 19 ++- .../src/linux/seccomp_notify.rs | 19 ++- .../src/remote.rs | 45 ++++--- docs/reference/support-matrix.mdx | 12 +- docs/security/best-practices.mdx | 16 ++- 8 files changed, 237 insertions(+), 91 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index ba5b3c59b0..75263865e3 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -542,3 +542,23 @@ engine with a gateway policy revision. `Error/MainProcessFailed`. Infrastructure failures also use `Error`, with a distinct condition reason and no fabricated canonical-process result. Runtime restart policies must not replace the canonical process. + +## Shared Boundary Primitives + +`openshell-isolation-interface` owns the common boundary protocol and Linux +mechanisms. Drivers provide the protected transport and immutable resource +identity; they do not implement their own process or network protocol. All +remote traffic uses one mutually authenticated gRPC connection. Independent +streams carry process control, exec output, and TCP bytes; a persistent +`Mediate` stream carries DNS queries and supervisor-produced answers. There is +no alternate raw-TLS application protocol or general UDP framing. + +The shared process-signal mediator resolves each positive target PID or TID to +its thread-group leader, excludes the sandbox leader, retains a pidfd, and sends +the signal through that descriptor. It never continues the original numeric-PID +syscall after inspection. This prevents TID aliases or PID reuse from turning an +agent signal into a signal to the sandbox. Ordinary mediated `kill` reports the +broker as its sender, not the original calling agent's `SI_USER` identity. +Queued signals preserve permitted application siginfo payloads; they cannot +forge kernel-generated or `SI_TKILL` codes. Programs requiring original sender +identity must account for this mediation boundary. diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index d9055fa319..050ff83183 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -359,7 +359,11 @@ impl ProviderCredentialState { /// here so SDKs can read them at startup. /// 3. Everything else stays as placeholders for proxy-time resolution. pub fn child_env_with_gcp_resolved(&self) -> HashMap { - self.child_env_snapshot_with_gcp_resolved().1 + let inner = self + .inner + .read() + .expect("provider credential state poisoned"); + Self::resolve_child_env_snapshot(&inner).1 } /// Return the current revision and its workload-facing environment from @@ -369,13 +373,21 @@ impl ProviderCredentialState { /// revision must describe the exact environment sent across the boundary, /// so callers must not obtain the two values through separate lock /// acquisitions. - pub fn child_env_snapshot_with_gcp_resolved(&self) -> (u64, HashMap) { - use crate::google_cloud; - + pub fn child_env_snapshot_with_gcp_resolved( + &self, + ) -> std::io::Result<(u64, HashMap)> { let inner = self .inner .read() - .expect("provider credential state poisoned"); + .map_err(|_| std::io::Error::other("provider credential state poisoned"))?; + Ok(Self::resolve_child_env_snapshot(&inner)) + } + + fn resolve_child_env_snapshot( + inner: &ProviderCredentialStateInner, + ) -> (u64, HashMap) { + use crate::google_cloud; + let mut env = inner.current.child_env.clone(); let has_gcp_metadata = env.contains_key("GCE_METADATA_HOST") @@ -438,13 +450,13 @@ impl ProviderCredentialState { expected_revision: u64, revision: u64, mut child_env: HashMap, - ) -> u64 { + ) -> std::io::Result { let mut inner = self .inner .write() - .expect("provider credential state poisoned"); + .map_err(|_| std::io::Error::other("provider credential state poisoned"))?; if revision == inner.current.revision || expected_revision != inner.current.revision { - return inner.current.revision; + return Ok(inner.current.revision); } for key in &inner.suppressed_keys { @@ -462,7 +474,7 @@ impl ProviderCredentialState { inner.static_credential_bindings.clear(); inner.known_static_credential_keys.clear(); inner.static_credential_identity_epochs.clear(); - revision + Ok(revision) } /// Return the GCP token placeholder and its remaining lifetime in seconds. @@ -2173,32 +2185,58 @@ mod tests { ); assert_eq!( - state.compare_and_install_child_env_snapshot( - 4, - 6, - HashMap::from([("TOKEN".to_string(), "six".to_string())]), - ), + state + .compare_and_install_child_env_snapshot( + 4, + 6, + HashMap::from([("TOKEN".to_string(), "six".to_string())]), + ) + .unwrap(), 6 ); assert_eq!( - state.compare_and_install_child_env_snapshot( - 4, - 5, - HashMap::from([("TOKEN".to_string(), "stale".to_string())]), - ), + state + .compare_and_install_child_env_snapshot( + 4, + 5, + HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ) + .unwrap(), 6 ); assert_eq!( - state.compare_and_install_child_env_snapshot(6, 2, HashMap::new()), + state + .compare_and_install_child_env_snapshot(6, 2, HashMap::new()) + .unwrap(), 2, "opaque revisions may move numerically backwards" ); - let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); + let (revision, env) = state.child_env_snapshot_with_gcp_resolved().unwrap(); assert_eq!(revision, 2); assert!(env.is_empty(), "an empty snapshot must revoke the old env"); } + #[test] + fn poisoned_environment_update_returns_error_without_recovering_state() { + let state = ProviderCredentialState::from_child_env_snapshot(4, HashMap::new()); + let poison = state.clone(); + assert!( + std::thread::spawn(move || { + let _guard = poison.inner.write().unwrap(); + panic!("poison state during mutation"); + }) + .join() + .is_err() + ); + assert!( + state + .compare_and_install_child_env_snapshot(4, 5, HashMap::new()) + .is_err() + ); + assert!(state.child_env_snapshot_with_gcp_resolved().is_err()); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index 05a1af4f99..37d2b27494 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -5,8 +5,9 @@ //! //! Drivers choose and provision the transport, but they do not redefine the //! process lifecycle, streaming, identity, or authentication messages. The -//! control and boundary roles exchange these length-delimited JSON frames over -//! a private Unix socket, authenticated TCP connection, or virtio-vsock stream. +//! supervisor and sandbox exchange these length-delimited JSON frames inside +//! gRPC streams on one mutually authenticated connection. The driver chooses +//! the underlying private Unix socket, TCP connection, or virtio-vsock stream. use std::fmt; use std::io; @@ -649,7 +650,7 @@ pub enum Response { timing: MediationTimingWire, }, Error { - kind: String, + kind: BoundaryErrorKind, message: String, }, } @@ -701,31 +702,41 @@ pub enum DnsQueryResultWire { Error(String), } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BoundaryErrorKind { + Invalid, + Denied, + Unavailable, + Terminated, + Process, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct BinaryIdentityWire { - pub binary_path: Option, - pub binary_digest: Option, - pub ancestors: Vec, - pub cmdline_paths: Vec, - pub resolve_error: Option, +#[serde(tag = "result", rename_all = "snake_case", deny_unknown_fields)] +pub enum BinaryIdentityWire { + Resolved { + binary_path: PathBuf, + binary_digest: Option, + ancestors: Vec, + cmdline_paths: Vec, + }, + Failed { + message: String, + }, } impl From> for BinaryIdentityWire { fn from(identity: Result) -> Self { match identity { - Ok(identity) => Self { - binary_path: Some(identity.binary_path), - binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + Ok(identity) => Self::Resolved { + binary_path: identity.binary_path, + binary_digest: identity.binary_digest, ancestors: identity.ancestors, cmdline_paths: identity.cmdline_paths, - resolve_error: None, }, - Err(error) => Self { - binary_path: None, - binary_digest: None, - ancestors: Vec::new(), - cmdline_paths: Vec::new(), - resolve_error: Some(error.to_string()), + Err(error) => Self::Failed { + message: error.to_string(), }, } } @@ -733,22 +744,20 @@ impl From> for BinaryIdentityWire { impl BinaryIdentityWire { pub fn into_result(self) -> Result { - if let Some(error) = self.resolve_error { - return Err(ResolveError::Failed(error)); + match self { + Self::Resolved { + binary_path, + binary_digest, + ancestors, + cmdline_paths, + } => Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors, + cmdline_paths, + }), + Self::Failed { message } => Err(ResolveError::Failed(message)), } - let binary_path = self.binary_path.ok_or_else(|| { - ResolveError::Failed("boundary identity omitted binary path".to_string()) - })?; - let binary_digest = self - .binary_digest - .map(|digest| digest.parse::()) - .transpose()?; - Ok(BinaryIdentity { - binary_path, - binary_digest, - ancestors: self.ancestors, - cmdline_paths: self.cmdline_paths, - }) } } @@ -1087,6 +1096,46 @@ pub enum FrameError { mod tests { use super::*; + #[test] + fn binary_identity_wire_rejects_ambiguous_or_invalid_shapes() { + for encoded in [ + r#"{"result":"resolved","ancestors":[],"cmdline_paths":[]}"#, + r#"{"result":"resolved","binary_path":"/bin/tool","binary_digest":"invalid","ancestors":[],"cmdline_paths":[]}"#, + r#"{"result":"failed","message":"unavailable","binary_path":"/bin/tool"}"#, + ] { + assert!(serde_json::from_str::(encoded).is_err()); + } + let identity = BinaryIdentityWire::from(Ok(BinaryIdentity { + binary_path: PathBuf::from("/bin/tool"), + binary_digest: Some("a".repeat(64).parse().unwrap()), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + })); + let encoded = serde_json::to_vec(&identity).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + identity + ); + } + + #[test] + fn boundary_error_kind_rejects_unknown_values() { + assert!(serde_json::from_str::(r#""unkown""#).is_err()); + for kind in [ + BoundaryErrorKind::Invalid, + BoundaryErrorKind::Denied, + BoundaryErrorKind::Unavailable, + BoundaryErrorKind::Terminated, + BoundaryErrorKind::Process, + ] { + let encoded = serde_json::to_vec(&kind).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + kind + ); + } + } + #[test] fn request_round_trips_and_redacts_token() { let request = RequestEnvelope { diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index 74fdea736d..03248417ea 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -255,7 +255,7 @@ pub struct SandboxContext { } /// The agent workload to run inside the boundary. -pub use crate::AgentSpec; +use crate::AgentSpec; /// Maps backend name to its implementation. This is the only lookup by name; /// supervisor lifecycle never branches on a concrete backend, and resolution @@ -840,9 +840,24 @@ pub struct BinaryIdentity { /// A SHA-256 digest, kept typed so the identity field is not coupled to its /// textual encoding or forced to repeat the algorithm in its name. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] pub struct Sha256Digest([u8; 32]); +impl TryFrom for Sha256Digest { + type Error = ResolveError; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(value: Sha256Digest) -> Self { + value.to_string() + } +} + impl Sha256Digest { /// Return the raw digest bytes. #[must_use] diff --git a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs index c5e8e9bd9c..da2aa6d1c6 100644 --- a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs +++ b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs @@ -300,7 +300,7 @@ impl NotificationListener { srcfd, newfd: 0, newfd_flags: if close_on_exec { - u32::try_from(libc::O_CLOEXEC).expect("O_CLOEXEC fits u32") + u32::try_from(libc::O_CLOEXEC).map_err(io::Error::other)? } else { 0 }, @@ -460,7 +460,7 @@ fn probe_addfd_send() -> io::Result<()> { size_of::(), ) }; - let word_size = isize::try_from(size_of::()).expect("u64 size fits isize"); + let word_size = isize::try_from(size_of::()).map_err(io::Error::other)?; if read != word_size || value != 7 { return Err(io::Error::other("injected eventfd was not usable")); } @@ -560,7 +560,7 @@ fn probe_connected_sendto_fast_path() -> io::Result<()> { 0, ) }; - if sent != isize::try_from(direct.len()).expect("probe length fits isize") { + if sent != isize::try_from(direct.len()).map_err(io::Error::other)? { return Err(io::Error::last_os_error()); } @@ -573,17 +573,14 @@ fn probe_connected_sendto_fast_path() -> io::Result<()> { payload.len(), ) }; - if read != isize::try_from(payload.len()).expect("probe length fits isize") - || &payload != direct - { + if read != isize::try_from(payload.len()).map_err(io::Error::other)? || &payload != direct { return Err(io::Error::other( "connected sendto fast path did not relay data", )); } let destination = libc::sockaddr_un { - sun_family: libc::sa_family_t::try_from(libc::AF_UNIX) - .expect("AF_UNIX fits sa_family_t"), + sun_family: libc::sa_family_t::try_from(libc::AF_UNIX).map_err(io::Error::other)?, sun_path: [0; 108], }; // SAFETY: all pointers refer to live values. This deliberately @@ -596,7 +593,7 @@ fn probe_connected_sendto_fast_path() -> io::Result<()> { 0, std::ptr::addr_of!(destination).cast(), libc::socklen_t::try_from(size_of::()) - .expect("sockaddr family size fits socklen_t"), + .map_err(io::Error::other)?, ) }; if result != -1 || io::Error::last_os_error().raw_os_error() != Some(libc::EACCES) { @@ -677,8 +674,8 @@ fn receive_probe_notification(listener: &NotificationListener) -> io::Result Err(guest_error(&kind, message)), + Response::Error { kind, message } => Err(guest_error(kind, message)), response => Ok(response), }?; Ok((stream, response)) @@ -1192,7 +1199,7 @@ impl BoundaryClient { } match response.response { Response::MediationReady => {} - Response::Error { kind, message } => return Err(guest_error(&kind, message)), + Response::Error { kind, message } => return Err(guest_error(kind, message)), response => return Err(unexpected_response("mediation_ready", &response)), } Ok(ClientMediationSession::start(stream)) @@ -1251,6 +1258,7 @@ async fn connect_boundary_once( topology: &BoundaryTopology, ) -> Result { let (stream, tls): (BoundaryDuplexStream, &BoundaryClientTls) = match &topology.transport { + #[cfg(unix)] BoundaryTransport::Unix { socket_path, tls } => { let stream = UnixStream::connect(socket_path).await.map_err(|error| { BackendError::Unavailable(format!( @@ -1260,6 +1268,12 @@ async fn connect_boundary_once( })?; (Box::new(stream), tls) } + #[cfg(not(unix))] + BoundaryTransport::Unix { .. } => { + return Err(BackendError::Unavailable( + "Unix boundary transport requires a Unix host".to_string(), + )); + } BoundaryTransport::TlsTcp { address, tls } => { let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) .await @@ -1461,14 +1475,15 @@ fn unexpected_response(expected: &str, response: &Response) -> BackendError { )) } -fn guest_error(kind: &str, message: String) -> BackendError { +fn guest_error(kind: crate::boundary_protocol::BoundaryErrorKind, message: String) -> BackendError { + use crate::boundary_protocol::BoundaryErrorKind; let message = format!("boundary process leaf: {message}"); match kind { - "invalid" => BackendError::Descriptor(message), - "denied" => BackendError::Denied(message), - "unavailable" => BackendError::Unavailable(message), - "terminated" => BackendError::Terminated(message), - _ => BackendError::Process(message), + BoundaryErrorKind::Invalid => BackendError::Descriptor(message), + BoundaryErrorKind::Denied => BackendError::Denied(message), + BoundaryErrorKind::Unavailable => BackendError::Unavailable(message), + BoundaryErrorKind::Terminated => BackendError::Terminated(message), + BoundaryErrorKind::Process => BackendError::Process(message), } } @@ -1567,7 +1582,7 @@ mod tests { || envelope.boundary_id != "sandbox-1" { Response::Error { - kind: "denied".to_string(), + kind: crate::boundary_protocol::BoundaryErrorKind::Denied, message: "control authentication failed".to_string(), } } else if matches!(envelope.request, Request::Wait { .. }) { @@ -1665,12 +1680,11 @@ mod tests { let query = DnsQueryWire { request: vec![1, 2, 3], transport: crate::contract::DnsTransport::Udp, - identity: crate::boundary_protocol::BinaryIdentityWire { - binary_path: Some(PathBuf::from("/usr/bin/dig")), - binary_digest: Some("a".repeat(64)), + identity: crate::boundary_protocol::BinaryIdentityWire::Resolved { + binary_path: PathBuf::from("/usr/bin/dig"), + binary_digest: Some("a".repeat(64).parse().unwrap()), ancestors: Vec::new(), cmdline_paths: Vec::new(), - resolve_error: None, }, timing: crate::boundary_protocol::MediationTimingWire::default(), }; @@ -2167,6 +2181,7 @@ mod tests { server.abort(); } + #[cfg(unix)] #[tokio::test] async fn tls_unix_flushes_large_control_requests_before_reading_response() { let socket_path = std::env::temp_dir().join(format!( diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index cfa5b69549..7671866deb 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -84,12 +84,18 @@ To override the default image references, use Helm values: ## Kernel Requirements -OpenShell enforces sandbox isolation through two Linux kernel security modules: +The sandbox boundary requires the following Linux kernel facilities, including +when it runs inside a container or microVM: | Module | Requirement | Details | | -------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [Landlock LSM](https://docs.kernel.org/security/landlock.html) | Recommended | Enforces filesystem access restrictions at the kernel level. The `best_effort` compatibility mode uses the highest Landlock ABI the host kernel supports. The `hard_requirement` mode fails sandbox creation if the required ABI is unavailable. | -| seccomp | Required | Filters dangerous system calls. Available on all modern Linux kernels (3.17+). | +| [Landlock LSM](https://docs.kernel.org/security/landlock.html) | Required | ABI 3 or newer, introduced in Linux 6.2, with Landlock enabled. The mandatory baseline protects private channel and bootstrap files, including against truncation. A filesystem policy's `best_effort` setting never disables this baseline. | +| seccomp | Required | Nested user-notification filters and atomic `SECCOMP_IOCTL_NOTIF_ADDFD` with `SECCOMP_ADDFD_FLAG_SEND`, usable under the runtime's existing seccomp profile without added capabilities. The sandbox actively probes these operations before admitting the workload. | + +A kernel version alone does not establish support. A disabled Landlock LSM or a +runtime profile that blocks the required seccomp operations causes launch to +fail closed. An upstream Linux 6.2 or newer kernel provides the required +Landlock ABI; distribution backports must pass the same active qualification. On macOS, these kernel modules run inside the Docker Desktop Linux VM, not on the host kernel. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 1c541f8f8e..f18e9ede68 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -159,14 +159,20 @@ Paths listed in `read_only` receive read-only access. Paths listed in `read_write` receive full access. All other paths are inaccessible. -Landlock setup runs in two phases. The parent supervisor probes the kernel ABI and opens the configured path file descriptors before forking. The child then applies the ruleset with `restrict_self()` after privilege drop. At startup, OpenShell emits the selected ABI version and the applied read-only and read-write rule counts so you can confirm what the kernel accepted. +The sandbox installs a mandatory Landlock baseline before running agent code. +It requires ABI 3 or newer, introduced in Linux 6.2, so denied writes include +file truncation. The baseline hides the private channel and bootstrap root from +the agent by granting access only to explicitly opened, permitted root children; +it never grants the filesystem root or follows a root-level symlink when +constructing that allowlist. Startup fails if this baseline cannot be enforced. +The configured filesystem policy adds restrictions on top of the baseline. | Aspect | Detail | |---|---| -| Default | `compatibility: best_effort`. Uses the highest kernel ABI available. Missing paths are skipped. If the kernel does not support Landlock or any configured path cannot be opened, the sandbox continues without those restrictions and emits a High-severity OCSF `DetectionFinding`. | -| What you can change | Set `compatibility: hard_requirement` to abort sandbox startup if Landlock is unavailable or any configured path cannot be opened. | -| Risk if relaxed | On kernels without Landlock (pre-5.13), or when all paths fail to open, the sandbox runs without kernel-level filesystem restrictions. The agent can access any file the process user can access. | -| Recommendation | Use `best_effort` for development. Use `hard_requirement` in environments where any gap in filesystem isolation is unacceptable. Treat High-severity Landlock findings as a signal to investigate the host kernel or the image. Run on Ubuntu 22.04+ or any kernel 5.13+ for Landlock support. | +| Default | The mandatory baseline always requires Landlock ABI 3 or newer. The additional filesystem policy defaults to `compatibility: best_effort`, which may skip unavailable policy paths and emits a High-severity OCSF `DetectionFinding` when that policy cannot be fully applied. | +| What you can change | Set `compatibility: hard_requirement` to reject startup when the additional filesystem policy cannot be applied. Neither setting relaxes the private-file baseline. | +| Risk if relaxed | A skipped additional policy can expose otherwise permitted image or workspace files to the agent. It cannot grant access to files excluded by the mandatory baseline. | +| Recommendation | Require Linux 6.2 or newer with Landlock enabled, or a qualified distribution backport providing ABI 3. Use `hard_requirement` where all configured filesystem restrictions must apply. Investigate High-severity Landlock findings rather than treating them as a supported degraded security mode. | ### Read-Only vs Read-Write Paths From ece0ef7d634ffc95b8cce271f65d6a58e5a74f74 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:12:52 -0700 Subject: [PATCH 5/7] test(isolation): import owned agent specification explicitly Signed-off-by: Drew Newberry --- .../openshell-isolation-interface/tests/backend_conformance.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/openshell-isolation-interface/tests/backend_conformance.rs b/crates/openshell-isolation-interface/tests/backend_conformance.rs index 1542a040db..e138eaec11 100644 --- a/crates/openshell-isolation-interface/tests/backend_conformance.rs +++ b/crates/openshell-isolation-interface/tests/backend_conformance.rs @@ -16,6 +16,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use async_trait::async_trait; +use openshell_isolation_interface::AgentSpec; use openshell_isolation_interface::contract::*; use tokio::sync::oneshot; From d672ae195f920a0a961851c3cf4117bcc52bded1 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 21:17:51 -0700 Subject: [PATCH 6/7] docs(isolation): describe mediated DNS channel Signed-off-by: Drew Newberry --- proto/isolation_boundary.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/isolation_boundary.proto b/proto/isolation_boundary.proto index 333b8567bc..fa5706ca1f 100644 --- a/proto/isolation_boundary.proto +++ b/proto/isolation_boundary.proto @@ -7,7 +7,7 @@ package openshell.isolation.v1; // IsolationBoundary multiplexes the protected sandbox-to-supervisor protocol // over one authenticated gRPC connection. Each Exchange call is an independent -// HTTP/2 stream; Mediate is the persistent DNS and UDP data plane. +// HTTP/2 stream; Mediate is the persistent DNS exchange channel. service IsolationBoundary { rpc Exchange(stream BoundaryChunk) returns (stream BoundaryChunk); rpc Mediate(stream BoundaryChunk) returns (stream BoundaryChunk); From ce8745c7dddd41d4116fabaa3123fbe38b5e925b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 21:24:17 -0700 Subject: [PATCH 7/7] fix(isolation): bound mediation attach without nested retries Signed-off-by: Drew Newberry --- .../src/remote.rs | 73 ++++++++++++++----- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index c7a0ec2fc7..d883d84894 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -1155,23 +1155,15 @@ impl BoundaryClient { { return Ok(session.clone()); } - for attempt in 0..=40 { - match self.open_mediation_session().await { - Ok(session) => { - *state = Some(session.clone()); - return Ok(session); - } - Err(BackendError::Denied(message)) - if message.contains("already active") && attempt < 40 => - { - tokio::time::sleep(Duration::from_millis(25)).await; - } - Err(error) => return Err(error), - } - } - Err(BackendError::Unavailable( - "boundary mediation retry budget exhausted".to_string(), - )) + // The boundary owns exclusive-lease retirement and bounds replacement + // waiting. Never multiply that deadline with message-matching retries. + let session = tokio::time::timeout(REQUEST_TIMEOUT, self.open_mediation_session()) + .await + .map_err(|_| { + BackendError::Unavailable("boundary mediation attach timed out".to_string()) + })??; + *state = Some(session.clone()); + Ok(session) } async fn open_mediation_session(&self) -> Result, BackendError> { @@ -1585,6 +1577,11 @@ mod tests { kind: crate::boundary_protocol::BoundaryErrorKind::Denied, message: "control authentication failed".to_string(), } + } else if matches!(envelope.request, Request::OpenMediation) { + Response::Error { + kind: crate::boundary_protocol::BoundaryErrorKind::Denied, + message: "a mediation session is already active".to_string(), + } } else if matches!(envelope.request, Request::Wait { .. }) { Response::Exited { status: ExitStatusWire::Exited(23), @@ -1637,6 +1634,48 @@ mod tests { .expect("frame length") } + #[tokio::test] + async fn mediation_denial_is_not_retried_or_cached_as_a_session() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let service = TestGrpcBoundary { + wait_for_half_close: false, + expected_token: "a".repeat(32), + requests: requests.clone(), + }; + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + tonic::transport::Server::builder() + .add_service(IsolationBoundaryServer::new(service)) + .serve_with_incoming(tokio_stream::iter([Ok::<_, std::io::Error>(stream)])) + .await + .unwrap(); + }); + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .unwrap() + .connect() + .await + .unwrap(); + let client = BoundaryClient::new(tls_topology( + address, + test_certificate().client_tls, + &"a".repeat(32), + )); + *client.grpc_channel.lock().await = Some(channel); + // A caller may try again later, but each call makes exactly one + // bounded attach attempt and preserves the server's typed denial. + for expected_requests in 1..=2 { + assert!(matches!( + client.mediation_session().await, + Err(BackendError::Denied(_)) + )); + assert_eq!(requests.load(Ordering::Acquire), expected_requests); + assert!(client.mediation.lock().await.is_none()); + } + server.abort(); + } + #[tokio::test] async fn grpc_stream_preserves_response_after_request_half_close() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();