diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index d79b7366d..94e88e8a1 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -36,6 +36,12 @@ template resource limits. Docker and Podman apply them as runtime limits. Kubernetes mirrors each limit into the matching request. VM accepts the fields but currently ignores them. +The CLI `--runtime-class` flag sets `SandboxTemplate.runtime_class_name`. The +Kubernetes driver maps this onto pod `spec.runtimeClassName` (used to request +sandboxed runtimes such as Kata Containers or gVisor); other drivers ignore it. +When omitted on GPU sandboxes, the Kubernetes driver still falls back to +`nvidia`. + VM runtime state paths are derived only from driver-validated sandbox IDs matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a private `run/` directory plus Unix peer UID/PID checks. Standalone diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 917c8faa1..73fbbd320 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1148,6 +1148,7 @@ enum DoctorCommands { } #[derive(Subcommand, Debug)] +#[allow(clippy::large_enum_variant)] enum SandboxCommands { /// Create a sandbox. #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] @@ -1216,6 +1217,15 @@ enum SandboxCommands { #[arg(long)] memory: Option, + /// Request a Kubernetes `RuntimeClass` for the sandbox pod (for example: + /// `kata`, `kata-containers`, `gvisor`). + /// + /// Only honored by the Kubernetes driver. On GPU sandboxes the driver + /// defaults to `nvidia` when this flag is omitted; passing this flag + /// overrides that default. + #[arg(long, value_name = "NAME")] + runtime_class: Option, + /// Provider names to attach to this sandbox. #[arg(long = "provider")] providers: Vec, @@ -2518,6 +2528,7 @@ async fn main() -> Result<()> { gpu_device, cpu, memory, + runtime_class, providers, policy, forward, @@ -2586,6 +2597,7 @@ async fn main() -> Result<()> { gpu_device.as_deref(), cpu.as_deref(), memory.as_deref(), + runtime_class.as_deref(), editor, &providers, policy.as_deref(), @@ -4168,6 +4180,23 @@ mod tests { } } + #[test] + fn sandbox_create_runtime_class_flag_parses() { + let cli = + Cli::try_parse_from(["openshell", "sandbox", "create", "--runtime-class", "kata"]) + .expect("sandbox create --runtime-class should parse"); + + match cli.command { + Some(Commands::Sandbox { + command: Some(SandboxCommands::Create { runtime_class, .. }), + .. + }) => { + assert_eq!(runtime_class.as_deref(), Some("kata")); + } + other => panic!("expected SandboxCommands::Create, got: {other:?}"), + } + } + #[test] fn service_expose_accepts_positional_target_port_and_service() { let cli = Cli::try_parse_from([ diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index b92be199e..2a60ac93c 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1685,6 +1685,7 @@ pub async fn sandbox_create( gpu_device: Option<&str>, cpu: Option<&str>, memory: Option<&str>, + runtime_class: Option<&str>, editor: Option, providers: &[String], policy: Option<&str>, @@ -1754,10 +1755,11 @@ pub async fn sandbox_create( let policy = load_sandbox_policy(policy)?; let resource_limits = build_sandbox_resource_limits(cpu, memory)?; - let template = if image.is_some() || resource_limits.is_some() { + let template = if image.is_some() || resource_limits.is_some() || runtime_class.is_some() { Some(SandboxTemplate { image: image.unwrap_or_default(), resources: resource_limits, + runtime_class_name: runtime_class.unwrap_or_default().to_string(), ..SandboxTemplate::default() }) } else { diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 3ed43b2fc..106a33fa4 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -789,6 +789,7 @@ async fn sandbox_create_keeps_command_sessions_by_default() { None, None, None, + None, &[], None, None, @@ -830,6 +831,7 @@ async fn sandbox_create_sends_cpu_and_memory_limits_only() { Some("500m"), Some("2Gi"), None, + None, &[], None, None, @@ -906,6 +908,7 @@ async fn sandbox_create_does_not_infer_command_providers_when_v2_enabled() { None, None, None, + None, &[], None, None, @@ -962,6 +965,7 @@ async fn sandbox_create_returns_vm_error_without_waiting_for_timeout() { None, None, None, + None, &[], None, None, @@ -1014,6 +1018,7 @@ async fn sandbox_create_keeps_waiting_while_vm_progress_arrives() { None, None, None, + None, &[], None, None, @@ -1058,6 +1063,7 @@ async fn sandbox_create_times_out_when_only_logs_arrive() { None, None, None, + None, &[], None, None, @@ -1098,6 +1104,7 @@ async fn sandbox_create_deletes_command_sessions_with_no_keep() { None, None, None, + None, &[], None, None, @@ -1142,6 +1149,7 @@ async fn sandbox_create_deletes_shell_sessions_with_no_keep() { None, None, None, + None, &[], None, None, @@ -1186,6 +1194,7 @@ async fn sandbox_create_keeps_sandbox_with_hidden_keep_flag() { None, None, None, + None, &[], None, None, @@ -1230,6 +1239,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() { None, None, None, + None, &[], None, Some(openshell_core::forward::ForwardSpec::new(forward_port)), diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index 512abfd3d..ac2575f98 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -55,6 +55,19 @@ For Docker-backed sandboxes, GPU injection uses Docker CDI. If you enable Docker CDI after the gateway starts, restart the gateway so OpenShell can detect the updated Docker daemon capability. +### Runtime Class + +For Kubernetes-backed gateways, use `--runtime-class` to request a sandboxed +container runtime (for example, Kata Containers or gVisor) for the sandbox pod: + +```shell +openshell sandbox create --runtime-class kata -- claude +``` + +The value is passed through to the pod `spec.runtimeClassName`. The cluster +administrator must install and register the named RuntimeClass; OpenShell does +not provision it. Other compute drivers ignore this flag. + ### Custom Containers Use `--from` to create a sandbox from the base image, another pre-built sandbox name, a local directory, or a container image: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 26957baab..d36c968a9 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -71,6 +71,11 @@ name = "user_namespaces" path = "tests/user_namespaces.rs" required-features = ["e2e-kubernetes"] +[[test]] +name = "runtime_class" +path = "tests/runtime_class.rs" +required-features = ["e2e-kubernetes"] + [[test]] name = "host_gateway_alias" path = "tests/host_gateway_alias.rs" diff --git a/e2e/rust/tests/runtime_class.rs b/e2e/rust/tests/runtime_class.rs new file mode 100644 index 000000000..8c4e1e8e5 --- /dev/null +++ b/e2e/rust/tests/runtime_class.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes")] + +//! E2E test: verify `--runtime-class` propagates to the sandbox pod spec. +//! +//! Registers a `RuntimeClass` whose handler is `runc` (already present in +//! k3d/containerd) so the pod still schedules, runs `openshell sandbox create +//! --runtime-class `, and asserts the resulting pod has +//! `spec.runtimeClassName` set to the requested value. + +use std::process::Stdio; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use openshell_e2e::harness::binary::openshell_cmd; + +const SANDBOX_NAMESPACE: &str = "openshell"; +const RUNTIME_CLASS_NAME: &str = "openshell-e2e-runtime-class"; + +const RUNTIME_CLASS_MANIFEST: &str = r"apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: openshell-e2e-runtime-class +handler: runc +"; + +async fn kubectl(args: &[&str]) -> Result { + let output = tokio::process::Command::new("kubectl") + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("failed to run kubectl: {e}"))?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + if !output.status.success() { + return Err(format!("kubectl {args:?} failed: {stdout}{stderr}")); + } + Ok(stdout) +} + +async fn apply_runtime_class() -> Result<(), String> { + let mut child = tokio::process::Command::new("kubectl") + .args(["apply", "-f", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to spawn kubectl apply: {e}"))?; + { + use tokio::io::AsyncWriteExt; + let mut stdin = child.stdin.take().expect("stdin piped"); + stdin + .write_all(RUNTIME_CLASS_MANIFEST.as_bytes()) + .await + .map_err(|e| format!("write kubectl stdin: {e}"))?; + } + let output = child + .wait_with_output() + .await + .map_err(|e| format!("kubectl apply wait: {e}"))?; + if !output.status.success() { + return Err(format!( + "kubectl apply -f - failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + Ok(()) +} + +async fn delete_runtime_class() { + let _ = kubectl(&[ + "delete", + "runtimeclass", + RUNTIME_CLASS_NAME, + "--ignore-not-found", + ]) + .await; +} + +async fn delete_sandbox(name: &str) { + let _ = kubectl(&[ + "delete", + "sandbox", + name, + "-n", + SANDBOX_NAMESPACE, + "--ignore-not-found", + ]) + .await; +} + +fn unique_sandbox_name() -> String { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("rtc-e2e-{suffix}") +} + +async fn wait_for_pod(name: &str, timeout: Duration) -> Result<(), String> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(found) = kubectl(&[ + "get", + "pod", + name, + "-n", + SANDBOX_NAMESPACE, + "-o", + "jsonpath={.metadata.name}", + ]) + .await + && !found.trim().is_empty() + { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + Err(format!( + "pod {name} did not appear in namespace {SANDBOX_NAMESPACE} within {timeout:?}" + )) +} + +#[tokio::test] +async fn runtime_class_flag_propagates_to_pod_spec() { + apply_runtime_class() + .await + .expect("register e2e RuntimeClass"); + + let sandbox_name = unique_sandbox_name(); + + let mut create_cmd = openshell_cmd(); + create_cmd + .args([ + "sandbox", + "create", + "--name", + &sandbox_name, + "--runtime-class", + RUNTIME_CLASS_NAME, + "--", + "echo", + "runtime-class-ok", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let create_result = tokio::time::timeout(Duration::from_secs(180), create_cmd.output()).await; + + let wait_err = wait_for_pod(&sandbox_name, Duration::from_secs(60)).await; + + let runtime_class_observed = kubectl(&[ + "get", + "pod", + &sandbox_name, + "-n", + SANDBOX_NAMESPACE, + "-o", + "jsonpath={.spec.runtimeClassName}", + ]) + .await; + + delete_sandbox(&sandbox_name).await; + delete_runtime_class().await; + + let create_output = create_result + .expect("sandbox create did not finish in 180s") + .expect("sandbox create spawn failed"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&create_output.stdout), + String::from_utf8_lossy(&create_output.stderr), + ); + assert!( + create_output.status.success(), + "sandbox create with --runtime-class failed:\n{combined}", + ); + + wait_err.expect("sandbox pod never appeared"); + + let observed = runtime_class_observed.expect("read pod runtimeClassName"); + assert_eq!( + observed.trim(), + RUNTIME_CLASS_NAME, + "pod {sandbox_name} should have spec.runtimeClassName={RUNTIME_CLASS_NAME}, got '{observed}'", + ); +}